diff --git a/.circleci/config.yml b/.circleci/config.yml index f66e598a..e0d242cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,8 +18,6 @@ jobs: - run: npm install - run: npm test - run: npx tsc - - run: npm run pkg-binaries-linux - - run: ./snyk-api-import-linux help build-test-from-fork: docker: - image: circleci/node:12 diff --git a/.releaserc b/.releaserc index 92e61c16..23d3747e 100644 --- a/.releaserc +++ b/.releaserc @@ -1,89 +1,9 @@ { "prepare": [ "@semantic-release/npm", - { - "//": "adds a file to identify a build as a standalone binary", - "path": "@semantic-release/exec", - "cmd": "echo '' > dist/STANDALONE" - }, - { - "//": "build the macos", - "path": "@semantic-release/exec", - "cmd": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t mac-x64-12.18.2 -o snyk-api-import-macos" - }, - { - "//": "build the linux", - "path": "@semantic-release/exec", - "cmd": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t linux-x64-12.16.2 -o snyk-api-import-linux" - }, - { - "//": "build the alpine", - "path": "@semantic-release/exec", - "cmd": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t alpine-x64-12.9.1 -o snyk-api-import-alpine" - }, - { - "//": "build the windows binaries", - "path": "@semantic-release/exec", - "cmd": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t windows-x64-12.18.2 -o snyk-api-import-win.exe" - }, - { - "//": "shasum all binaries", - "path": "@semantic-release/exec", - "cmd": "shasum -a 256 snyk-api-import-linux > snyk-api-import-linux.sha256 && shasum -a 256 snyk-api-import-macos > snyk-api-import-macos.sha256 && shasum -a 256 snyk-api-import-win.exe > snyk-api-import-win.exe.sha256 && shasum -a 256 snyk-api-import-alpine > snyk-api-import-alpine.sha256" - }, - { - "//": "removes the file we use to identify a build as a standalone binary", - "path": "@semantic-release/exec", - "cmd": "rm dist/STANDALONE" - } ], "publish": [ "@semantic-release/npm", - { - "path": "@semantic-release/github", - "assets": [ - { - "path": "./snyk-api-import-linux", - "name": "snyk-api-import-linux", - "label": "snyk-api-import-linux" - }, - { - "path": "./snyk-api-import-linux.sha256", - "name": "snyk-api-import-linux.sha256", - "label": "snyk-api-import-linux.sha256" - }, - { - "path": "./snyk-api-import-alpine", - "name": "snyk-api-import-alpine", - "label": "snyk-api-import-alpine" - }, - { - "path": "./snyk-api-import-alpine.sha256", - "name": "snyk-api-import-alpine.sha256", - "label": "snyk-api-import-alpine.sha256" - }, - { - "path": "./snyk-api-import-macos", - "name": "snyk-api-import-macos", - "label": "snyk-api-import-macos" - }, - { - "path": "./snyk-api-import-macos.sha256", - "name": "snyk-api-import-macos.sha256", - "label": "snyk-api-import-macos.sha256" - }, - { - "path": "./snyk-api-import-win.exe", - "name": "snyk-api-import-win.exe", - "label": "snyk-api-import-win.exe" - }, - { - "path": "./snyk-api-import-win.exe.sha256", - "name": "snyk-api-import-win.exe.sha256", - "label": "snyk-api-import-win.exe.sha256" - } - ] - } ], "branches": ["master"] } diff --git a/README.md b/README.md index 129c66ec..cf4706c4 100644 --- a/README.md +++ b/README.md @@ -39,33 +39,15 @@ If you need to adjust concurrency you can stop the script, change the concurrenc - [AWS automation example](docs/example-workflows/aws-automation-example.md) # Installation -`snyk-api-import` CLI can be installed through multiple channels. -## Standalone executables (macOS, Linux, Windows) +Snyk snyk-api-import CLI is available as an [npm package](https://www.npmjs.com/package/snyk-api-import). -Use [GitHub Releases](https://github.com/snyk-tech-services/snyk-api-import/releases) to download a standalone executable of Snyk CLI for your platform. - -## More installation methods - -
- Install with npm or Yarn - -### Install with npm or Yarn - -[Snyk snyk-api-import CLI is available as an npm package](https://www.npmjs.com/package/snyk-api-import). If you have Node.js installed locally, you can install it by running: +Ensure you have [node.js installed](https://nodejs.org/en/learn/getting-started/how-to-install-nodejs), the `snyk-api-import` can then be installed by running: ```bash npm install snyk-api-import@latest -g ``` -or if you are using Yarn: - -```bash -yarn global add snyk-api-import -``` - -
- # Usage By default the `import` command will run if no command specified. - `import` - kick off a an API powered import of repos/targets into existing Snyk orgs defined in [import configuration file](./docs/import.md). 100% support available for all project types supported via [Import API](https://snyk.docs.apiary.io/#reference/import-projects/import/import-targets). @@ -109,5 +91,3 @@ The logs can be explored using [Bunyan CLI](http://trentm.com/node-bunyan/bunyan snyk-api-import supports 100% of the same integration types and project sources as the Import API documentation. If an example is not in the docs for your use case please see the API documentation

- - diff --git a/docs/example-workflows/aws-automation-example.md b/docs/example-workflows/aws-automation-example.md index 4d18336b..d0db219f 100644 --- a/docs/example-workflows/aws-automation-example.md +++ b/docs/example-workflows/aws-automation-example.md @@ -45,7 +45,8 @@ To kick off an import, your Lambda function will need your `SNYK_TOKEN`, `SNYK_O ### Suggested architecture for the Lambda function 1. Get the "repository-name" and the "image-tag" from the event -2. Install or download the latest [snyk-api-import tool](https://github.com/snyk-tech-services/snyk-api-import/releases) +2. Install the latest snyk-api-import tool +`npm install snyk-api-import@latest -g` 3. Create a targets file for the tool using your SNYK_ORG_ID, ORG_INTEGRATION_ID as explained [here](https://github.com/snyk-tech-services/snyk-api-import/blob/master/docs/import.md#1-create-the-import-projectsjson-file). The import file shsould have the neccessary fileds as needed by our [import endpoint](https://snyk.docs.apiary.io/#reference/import-projects/import-targets), for example - an import file for a new image will look something like this: ``` { @@ -63,8 +64,8 @@ To kick off an import, your Lambda function will need your `SNYK_TOKEN`, `SNYK_O 4. Set your `SNYK_TOKEN` and your `SNYK_LOG_PATH` as an enviroment variables 5. Kick off an import by running the import command with refernce to the created import file, for example: ``` -DEBUG=snyk* snyk-api-import import --file=path/to/import-targets.json +DEBUG=snyk* npx snyk-api-import import --file=path/to/import-targets.json ``` 6. Wait for the imported targets log file - "imported-targets.log" to be created 7. Check that the log indicates a successful import -8. Done! \ No newline at end of file +8. Done! diff --git a/docs/import-data.md b/docs/import-data.md index 01134436..a351530d 100644 --- a/docs/import-data.md +++ b/docs/import-data.md @@ -38,9 +38,9 @@ This is a util that can help generate the import json data needed by the import - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **Github.com:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github` - - **Github Enterprise Server:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github-enterprise --sourceUrl=https://ghe.custom.com` - - **Github Enterprise Cloud:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github-enterprise` + - **Github.com:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github` + - **Github Enterprise Server:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github-enterprise --sourceUrl=https://ghe.custom.com` + - **Github Enterprise Cloud:** `DEBUG=snyk* GITHUB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=github-enterprise` 4. Use the generated data to feed into [import] command (/import.md) to generate kick off the import. @@ -67,8 +67,8 @@ This is a util that can help generate the import json data needed by the import - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **Gitlab.com:** `DEBUG=snyk* GITLAB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=gitlab ` - - **Hosted Gitlab:** `DEBUG=snyk* GITLAB_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=gitlab --sourceUrl=https://gitlab.custom.com` + - **Gitlab.com:** `DEBUG=snyk* GITLAB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=gitlab ` + - **Hosted Gitlab:** `DEBUG=snyk* GITLAB_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=gitlab --sourceUrl=https://gitlab.custom.com` 4. Use the generated data to feed into [import] command (/import.md) to generate kick off the import. @@ -97,8 +97,8 @@ This is a util that can help generate the import json data needed by the import - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **dev.azure.com:** `DEBUG=snyk* AZURE_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=azure-repos` - - **Hosted Azure:** `DEBUG=snyk* AZURE_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=azure-repos --sourceUrl=https://azure.custom.com` + - **dev.azure.com:** `DEBUG=snyk* AZURE_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=azure-repos` + - **Hosted Azure:** `DEBUG=snyk* AZURE_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=azure-repos --sourceUrl=https://azure.custom.com` 4. Use the generated data to feed into [import] command (/import.md) to generate kick off the import. @@ -127,7 +127,7 @@ This is a util that can help generate the import json data needed by the import - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **Bitbucket Server:** `DEBUG=snyk* BITBUCKET_SERVER_TOKEN=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=bitbucket-server --sourceUrl=https://bitbucket-server.dev.example.com` + - **Bitbucket Server:** `DEBUG=snyk* BITBUCKET_SERVER_TOKEN=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=bitbucket-server --sourceUrl=https://bitbucket-server.dev.example.com` 4. Use the generated data to feed into [import] command (/import.md) to generate kick off the import. @@ -160,7 +160,7 @@ export BITBUCKET_CLOUD_PASSWORD=your_bitbucket_cloud_password - All organization IDs can be found by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) 3. Run the command to generate import data: - - **Bitbucket Cloud:** `DEBUG=snyk* BITBUCKET_CLOUD_USERNAME=*** BITBUCKET_CLOUD_PASSWORD=*** SNYK_TOKEN=*** snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=bitbucket-cloud` + - **Bitbucket Cloud:** `DEBUG=snyk* BITBUCKET_CLOUD_USERNAME=*** BITBUCKET_CLOUD_PASSWORD=*** SNYK_TOKEN=*** npx snyk-api-import import:data --orgsData=path/to/snyk-orgs.json --source=bitbucket-cloud` 4. Use the generated data to feed into [import command](/import.md) to generate kick off the import. diff --git a/docs/import.md b/docs/import.md index 9526ef2c..6fe0147a 100644 --- a/docs/import.md +++ b/docs/import.md @@ -192,9 +192,11 @@ If you have any tests ot fixtures that should be ignored, please set the `exclus - `CONCURRENT_IMPORTS` (optional) defaults to 15 repos at a time, which is the recommended amount to import at once as a max. Just 1 repo may have many projects inside which can trigger a many files at once to be requested from the user's SCM instance and some may have rate limiting in place. This script aims to help reduce the risk of hitting a rate limit. - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1` -## 3. Download & run +## 3. Install and run -Grab a binary from the [releases page](https://github.com/snyk-tech-services/snyk-api-import/releases) and run with `DEBUG=snyk* snyk-api-import-macos import --file=path/to/imported-targets.json` +Install the tool from [npm](https://www.npmjs.com/): +`npm install snyk-api-import@latest -g` +and run with `DEBUG=snyk* npx snyk-api-import import --file=path/to/imported-targets.json` ## 4. Review logs When import is started via Snyk API, many files & targets will be added to an import job. This job when complete will provide logs of what projects could be detected, which failed and any errors that were encountered. For more details see [Import API documentation](https://snyk.docs.apiary.io/#reference/import-projects/import/import-targets) @@ -227,11 +229,11 @@ Note: Command to run: - skip all previously imported into all orgs in a Group: - `snyk-api-import-macos list:imported --integrationType= --groupId=` + `npx snyk-api-import list:imported --integrationType= --groupId=` - skip all previously imported for a specific Organization: - `snyk-api-import-macos list:imported --integrationType= --orgId=` -- a single integration / projects source `snyk-api-import-macos list:imported --integrationType= --groupId=` -- multiple integrations / projects sources `snyk-api-import-macos list:imported --integrationType= --integrationType= --orgId=` + `npx snyk-api-import list:imported --integrationType= --orgId=` +- a single integration / projects source `npx snyk-api-import list:imported --integrationType= --groupId=` +- multiple integrations / projects sources `npx snyk-api-import list:imported --integrationType= --integrationType= --orgId=` Supported integration types: diff --git a/docs/mirror-bitbucket-cloud.md b/docs/mirror-bitbucket-cloud.md index 1416f79b..e2b65ee9 100644 --- a/docs/mirror-bitbucket-cloud.md +++ b/docs/mirror-bitbucket-cloud.md @@ -4,17 +4,17 @@ You will need to configure Bitbucket Cloud username and password and Snyk token Please refer to individual documentation pages for more detailed info, however the general steps are: 1. `export BITBUCKET_CLOUD_USERNAME=***`, `export BITBUCKET_CLOUD_PASSWORD=***` and `export SNYK_TOKEN=***` -2. Generate organization data e.g. `snyk-api-import orgs:data --source=bitbucket-cloud --groupId=` [Full instructions](./orgs.md) -3. Create organizations in Snyk `snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-cloud` [Full instructions](./import-data.md) -5. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) +2. Generate organization data e.g. `npx snyk-api-import orgs:data --source=bitbucket-cloud --groupId=` [Full instructions](./orgs.md) +3. Create organizations in Snyk `npx snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-cloud` [Full instructions](./import-data.md) +5. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) ## Re-importing new repos & orgs only while Mirroring Once initial import is complete you may want to periodically check for new repos and make sure they are added into Snyk. To do this a similar flow to what is described above with a few small changes can be used: 1. `export BITBUCKET_CLOUD_USERNAME=***`, `export BITBUCKET_CLOUD_PASSWORD=***` and `export SNYK_TOKEN=***` -2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `snyk-api-import orgs:data --source=bitbucket-cloud --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) -3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-cloud` [Full instructions](./import-data.md) +2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `npx snyk-api-import orgs:data --source=bitbucket-cloud --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) +3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `npx snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-cloud` [Full instructions](./import-data.md) 5. Optional. Generate the previously imported log to skip all previously imported repos a Group (see full [documentation](./import.md#to-skip-all-previously-imported-targets)): -`snyk-api-import-macos list:imported --integrationType= --groupId=` -6. Run import `DEBUG=*snyk* snyk-api-import import` [Full instructions](./import.md) \ No newline at end of file +`npx snyk-api-import list:imported --integrationType= --groupId=` +6. Run import `DEBUG=*snyk* npx snyk-api-import import` [Full instructions](./import.md) diff --git a/docs/mirror-bitbucket-server.md b/docs/mirror-bitbucket-server.md index 474eedb1..480568a4 100644 --- a/docs/mirror-bitbucket-server.md +++ b/docs/mirror-bitbucket-server.md @@ -4,17 +4,17 @@ You will need to configure both Bitbucket Server token and Snyk token as environ Please refer to individual documentation pages for more detailed info, however the general steps are: 1. `export BITBUCKET_SERVER_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data e.g. `snyk-api-import orgs:data --source=bitbucket-server --groupId=` [Full instructions](./orgs.md) -3. Create organizations in Snyk `snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-server` [Full instructions](./import-data.md) -5. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) +2. Generate organization data e.g. `npx snyk-api-import orgs:data --source=bitbucket-server --groupId=` [Full instructions](./orgs.md) +3. Create organizations in Snyk `npx snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-server` [Full instructions](./import-data.md) +5. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) ## Re-importing new repos & orgs only while Mirroring Once initial import is complete you may want to periodically check for new repos and make sure they are added into Snyk. To do this a similar flow to what is described above with a few small changes can be used: 1. `export BITBUCKET_SERVER_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `snyk-api-import orgs:data --source=bitbucket-server --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) -3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-server` [Full instructions](./import-data.md) +2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `npx snyk-api-import orgs:data --source=bitbucket-server --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) +3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `npx snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=bitbucket-server` [Full instructions](./import-data.md) 5. Optional. Generate the previously imported log to skip all previously imported repos a Group (see full [documentation](./import.md#to-skip-all-previously-imported-targets)): -`snyk-api-import-macos list:imported --integrationType= --groupId=` -6. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) +`npx snyk-api-import list:imported --integrationType= --groupId=` +6. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) diff --git a/docs/mirror-github.md b/docs/mirror-github.md index 51135b06..d3bda9ec 100644 --- a/docs/mirror-github.md +++ b/docs/mirror-github.md @@ -9,22 +9,22 @@ You will need to configure both Github token and Snyk token as environment varia Please refer to individual documentation pages for more detailed info, however the general steps are: 1. `export GITHUB_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data e.g. `snyk-api-import orgs:data --source=github --groupId=` [Full instructions](./orgs.md) -3. Create organizations in Snyk `snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=github` [Full instructions](./import-data.md) -5. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) +2. Generate organization data e.g. `npx snyk-api-import orgs:data --source=github --groupId=` [Full instructions](./orgs.md) +3. Create organizations in Snyk `npx snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=github` [Full instructions](./import-data.md) +5. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) ## Re-importing new repos & organizations only while mirroring Once initial import is complete you may want to periodically check for new repos and make sure they are added into Snyk. To do this a similar flow to what is described above with a few small changes can be used: 1. `export GITHUB_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `snyk-api-import orgs:data --source=github --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) -3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=github` [Full instructions](./import-data.md) +2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `npx snyk-api-import orgs:data --source=github --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) +3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `npx snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=github` [Full instructions](./import-data.md) 5. Generate the previously imported log to skip all previously imported repos in a Group (see full [documentation](./import.md#to-skip-all-previously-imported-targets)): -`snyk-api-import-macos list:imported --integrationType= --groupId=` -6. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) +`npx snyk-api-import list:imported --integrationType= --groupId=` +6. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) ## Syncing previously imported repos For repos already monitored in Snyk use the `sync` command to detect changes and update projects in Snyk. 1. Get a list of Snyk Organizations in the Group by listing all organizations a group admin belongs to via [Snyk Organizations API](https://snyk.docs.apiary.io/#reference/groups/list-all-organizations-in-a-group/list-all-organizations-in-a-group) -2. For every public Organization ID, run `sync` command [Full instructions](./sync.md) \ No newline at end of file +2. For every public Organization ID, run `sync` command [Full instructions](./sync.md) diff --git a/docs/mirror-gitlab.md b/docs/mirror-gitlab.md index b53563c5..18352586 100644 --- a/docs/mirror-gitlab.md +++ b/docs/mirror-gitlab.md @@ -4,17 +4,17 @@ You will need to configure both Gitlab token and Snyk token as environment varia Please refer to individual documentation pages for more detailed info, however the general steps are: 1. `export GITLAB_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data e.g. `snyk-api-import orgs:data --source=gitlab --groupId=` [Full instructions](./orgs.md) -3. Create organizations in Snyk `snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=gitlab --integrationType=gitlab` [Full instructions](./import-data.md) -5. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) +2. Generate organization data e.g. `npx snyk-api-import orgs:data --source=gitlab --groupId=` [Full instructions](./orgs.md) +3. Create organizations in Snyk `npx snyk-api-import orgs:create --file=orgs.json` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=gitlab --integrationType=gitlab` [Full instructions](./import-data.md) +5. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) ## Re-importing new repos & orgs only while Mirroring Once initial import is complete you may want to periodically check for new repos and make sure they are added into Snyk. To do this a similar flow to what is described above with a few small changes can be used: 1. `export GITLAB_TOKEN=***` and `export SNYK_TOKEN=***` -2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `snyk-api-import orgs:data --source=gitlab --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) -3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. -4. Generate import data `snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=gitlab` [Full instructions](./import-data.md) +2. Generate organization data in Snyk and skip any that do not have any repos via `--skipEmptyOrg` `npx snyk-api-import orgs:data --source=gitlab --groupId= --skipEmptyOrg` [Full instructions](./orgs.md) +3. Create organizations in Snyk and this time skip any that have been created already with `--noDuplicateNames` parameter `npx snyk-api-import orgs:create --file=orgs.json --noDuplicateNames` [Full instructions](./orgs.md) will create a `snyk-created-orgs.json` file with Snyk organization ids and integration ids that are needed for import. +4. Generate import data `npx snyk-api-import import:data --orgsData=snyk-created-orgs.json --source=gitlab` [Full instructions](./import-data.md) 5. Optional. Generate the previously imported log to skip all previously imported repos a Group (see full [documentation](./import.md#to-skip-all-previously-imported-targets)): -`snyk-api-import-macos list:imported --integrationType=gitlab --groupId=` -6. Run import `DEBUG=*snyk* snyk-api-import import`[Full instructions](./import.md) +`npx snyk-api-import list:imported --integrationType=gitlab --groupId=` +6. Run import `DEBUG=*snyk* npx snyk-api-import import`[Full instructions](./import.md) diff --git a/docs/orgs.md b/docs/orgs.md index 5e782210..8922a224 100644 --- a/docs/orgs.md +++ b/docs/orgs.md @@ -38,8 +38,8 @@ This is an opinionated util and will assume every organization in Github.com / G ## Github.com / Github Enterprise 1. set the [Github.com personal access token](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) as an environment variable: `export GITHUB_TOKEN=your_personal_access_token` 2. Run the command to generate organization data: - - **Github.com:** `snyk-api-import orgs:data --source=github --groupId=` - - **Github Enterprise:** `snyk-api-import orgs:data --source=github-enterprise --groupId= -- sourceUrl=https://ghe.custom.github.com/` + - **Github.com:** `npx snyk-api-import orgs:data --source=github --groupId=` + - **Github Enterprise:** `npx snyk-api-import orgs:data --source=github-enterprise --groupId= -- sourceUrl=https://ghe.custom.github.com/` This will create the organization data in a file `group--github--orgs.json` @@ -47,8 +47,8 @@ This will create the organization data in a file `group--github-< ## Gitlab.com / Hosted Gitlab 1. set the [Gitlab personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) as an environment variable: `export GITLAB_TOKEN=your_personal_access_token` 2. Run the command to generate organization data: - - **Gitlab:** `snyk-api-import orgs:data --source=gitlab --groupId=` - - **Hosted Gitlab:** `snyk-api-import orgs:data --source=gitlab --groupId= -- sourceUrl=https://gitlab.custom.com` + - **Gitlab:** `npx snyk-api-import orgs:data --source=gitlab --groupId=` + - **Hosted Gitlab:** `npx snyk-api-import orgs:data --source=gitlab --groupId= -- sourceUrl=https://gitlab.custom.com` This will create the organization data in a file `group--gitlab-orgs.json`. Both groups & sub-groups will be [listed](https://docs.gitlab.com/ee/api/groups.html) and then these will become Organizations in Snyk. @@ -57,7 +57,7 @@ This will create the organization data in a file `group--gitlab-o **Please note that Bitbucket Server is a hosted environment and you must provide the custom URL for your Bitbucket Server instance in the command** 1. set the [Bitbucket Server access token](https://www.jetbrains.com/help/youtrack/standalone/integration-with-bitbucket-server.html#enable-youtrack-integration-bbserver) as an environment variable: `export BITBUCKET_SERVER_TOKEN=your_personal_access_token` 2. Run the command to generate organization data: - - `snyk-api-import orgs:data --source=bitbucket-server --groupId= --sourceUrl=https://bitbucket-server.custom.com` + - `npx snyk-api-import orgs:data --source=bitbucket-server --groupId= --sourceUrl=https://bitbucket-server.custom.com` This will create the organization data in a file `group--bitbucket-server-orgs.json` @@ -66,7 +66,7 @@ This will create the organization data in a file `group--bitbucke **Note that the URL for Bitbucket Cloud is https://bitbucket.org/** 1. set the Bitbucket Cloud Username and Password as an environment variables: `export BITBUCKET_CLOUD_USERNAME=your_bitbucket_cloud_username` and `export BITBUCKET_CLOUD_PASSWORD=your_bitbucket_cloud_password` 2. Run the command to generate organization data: - - `snyk-api-import orgs:data --source=bitbucket-cloud --groupId=` + - `npx snyk-api-import orgs:data --source=bitbucket-cloud --groupId=` This will create the organization data in a file `group--bitbucket-cloud-orgs.json` @@ -102,11 +102,11 @@ Use the generated data to feed into Snyk [Orgs API](https://snyk.docs.apiary.io/ ## via `orgs:create` util 1. set the `SNYK_TOKEN` environment variable - your [Snyk api token](https://app.snyk.io/account) 2. Run the command to create Orgs: -`snyk-api-import orgs:create --noDuplicateNames --includeExistingOrgsInOutput --file=group--github--orgs.json` +`npx snyk-api-import orgs:create --noDuplicateNames --includeExistingOrgsInOutput --file=group--github--orgs.json` - Using the `noDuplicateNames` flag (optional) will Skip creating an organization if the given name is already taken within the Group. - Using the `includeExistingOrgsInOutput` flag (optional, default is "true") will Log existing organization information as well as newly created. To set this flag as false, please use "--no-includeExistingOrgsInOutput" in the command, like so: -`snyk-api-import orgs:create --no-includeExistingOrgsInOutput --file=group--github--orgs.json` +`npx snyk-api-import orgs:create --no-includeExistingOrgsInOutput --file=group--github--orgs.json` The file format required for this looks like so: ``` diff --git a/docs/sync.md b/docs/sync.md index 1f48aca6..6228939b 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -95,9 +95,11 @@ The command will produce detailed logs for projects that were `updated` and thos - `SNYK_API` (optional) defaults to `https://snyk.io/api/v1` - `GITHUB_TOKEN` - SCM token that has read level or similar permissions to see information about repos like default branch & can list files in a repo -## 2. Download & run +## 2. Install & run -Grab a binary from the [releases page](https://github.com/snyk-tech-services/snyk-api-import/releases) and run with `DEBUG=snyk* snyk-api-import-macos import --file=path/to/imported-targets.json` +Install the tool from [npm](https://www.npmjs.com/): +`npm install snyk-api-import@latest -g` +and run with `DEBUG=snyk* npx snyk-api-import import --file=path/to/imported-targets.json` ## 3. Review logs @@ -120,39 +122,39 @@ When running `sync` in `--dryRun` mode the logs will have `dryRun` as `true` so ### Github.com In dry-run mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github --exclusionGlobs=**/package.json,logs --dryRun=true` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github --exclusionGlobs=**/package.json,logs --dryRun=true` Live mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github` ### GitHub Enterprise Server In dry-run mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --dryRun=true` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --dryRun=true` Live mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --exclusionGlobs=**/*.yaml,logs` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --sourceUrl=https://custom.ghe.com --exclusionGlobs=**/*.yaml,logs` ### GitHub Enterprise Cloud In dry-run mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --dryRun=true` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --dryRun=true` Live mode: -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise` ### Only syncing Container projects (Dockerfiles) `--snykProduct` can be used to specify to sync projects belonging to Open Source, Container (Dockerfiles) or IaC products which represent files in Git repos. -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=container` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=container` ### Only syncing Open Source + Iac projects (Dockerfiles) -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=open-source --snykProduct=iac` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=open-source --snykProduct=iac` ### Exclude from syncing certain files & directories -`DEBUG=*snyk* SNYK_TOKEN=xxxx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=open-source --snykProduct=iac --exclusionGlobs=**/*.yaml,logs,system-test` +`DEBUG=*snyk* SNYK_TOKEN=xxxx npx snyk-api-import sync --orgPublicId= --source=github-enterprise --snykProduct=open-source --snykProduct=iac --exclusionGlobs=**/*.yaml,logs,system-test` # Known limitations diff --git a/jest.config.js b/jest.config.js index 039da88a..f8cdc48d 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,4 +3,7 @@ module.exports = { testEnvironment: 'node', collectCoverageFrom: ['lib/**/*.ts'], coverageReporters: ['text-summary', 'html'], + moduleNameMapper: { + '^axios$': 'axios/dist/node/axios.cjs', + }, }; diff --git a/package.json b/package.json index fa86f14b..d142cb32 100644 --- a/package.json +++ b/package.json @@ -18,9 +18,7 @@ "build": "tsc", "build-watch": "tsc -w", "prepare": "npm run build", - "snyk-test": "snyk test", - "pkg-binaries-linux": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t linux-x64-12.16.2 -o snyk-api-import-linux", - "pkg-binaries": "npx nexe@3.3.7 dist/index.js -r './dist/**/*.js' -t mac-x64-12.18.2 -o snyk-api-import-macos" + "snyk-test": "snyk test" }, "types": "./dist/index.d.ts", "repository": { @@ -55,7 +53,7 @@ "rimraf": "3.0.2", "simple-git": "3.16.0", "sleep-promise": "8.0.1", - "snyk-request-manager": "1.8.3", + "snyk-request-manager": "1.8.4", "source-map-support": "^0.5.16", "split": "1.0.1", "yargs": "16.2.0"