Skip to content
This repository has been archived by the owner on Jul 12, 2024. It is now read-only.

Ingest the remaining entities and relationships #5

Merged
merged 5 commits into from
Sep 10, 2020
Merged

Ingest the remaining entities and relationships #5

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a71a8c8
Ingest the remaining entities and relationships
Aug 25, 2020
d5048d8
Implement requested changes
creativice Sep 9, 2020
aa95f0a
Refactor user/group permission relationship generation
creativice Sep 9, 2020
4b5515d
Turn ANY, ANY LOCAL, ANY repositories to repository groups
creativice Sep 9, 2020
1330c05
Rename buildEntity -> repositoryEntity in arrow function
creativice Sep 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 26 additions & 12 deletions docs/jupiterone.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,23 +23,37 @@ https://github.com/JupiterOne/sdk/blob/master/docs/integrations/development.md

The following entities are created:

| Resources | Entity `_type` | Entity `_class` |
| ---------- | ------------------------ | --------------- |
| Account | `artifactory_account` | `Account` |
| Group | `artifactory_group` | `UserGroup` |
| User | `artifactory_user` | `User` |
| Repository | `artifactory_repository` | `Repository` |
| Resources | Entity `_type` | Entity `_class` |
| ------------------ | --------------------------------- | ------------------ |
| Account | `artifactory_account` | `Account` |
| AccessToken | `artifactory_access_token` | `Key`, `AccessKey` |
| Group | `artifactory_group` | `UserGroup` |
| User | `artifactory_user` | `User` |
| Repository | `artifactory_repository` | `Repository` |
| ArtifactCodeModule | `artifactory_artifact_codemodule` | `CodeModule` |
| Build | `artifactory_build` | `Configuration` |
| Permission | `artifactory_permission` | `AccessPolicy` |
| PipelineSource | `artifactory_pipeline_source` | `CodeRepo` |

### Relationships

The following relationships are created/mapped:

| Source Entity `_type` | Relationship `_class` | Target Entity `_type` |
| --------------------- | --------------------- | ------------------------ |
| `artifactory_account` | **HAS** | `artifactory_group` |
| `artifactory_account` | **HAS** | `artifactory_user` |
| `artifactory_group` | **HAS** | `artifactory_user` |
| `artifactory_account` | **HAS** | `artifactory_repository` |
| Source Entity `_type` | Relationship `_class` | Target Entity `_type` |
| -------------------------- | --------------------- | --------------------------------- |
| `artifactory_account` | **HAS** | `artifactory_access_token` |
| `artifactory_access_token` | **ASSIGNED** | `artifactory_user` |
| `artifactory_account` | **HAS** | `artifactory_group` |
| `artifactory_account` | **HAS** | `artifactory_user` |
| `artifactory_group` | **HAS** | `artifactory_user` |
| `artifactory_account` | **HAS** | `artifactory_repository` |
| `artifactory_repository` | **HAS** | `artifactory_artifact_codemodule` |
| `artifactory_build` | **CREATED** | `artifactory_artifact_codemodule` |
| `artifactory_permission` | **ASSIGNED** | `artifactory_user` |
| `artifactory_permission` | **ASSIGNED** | `artifactory_group` |
| `artifactory_permission` | **ALLOWS** | `artifactory_repository` |
| `artifactory_permission` | **ALLOWS** | `artifactory_build` |
| `artifactory_account` | **HAS** | `artifactory_pipeline_source` |

<!--
********************************************************************************
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
},
"dependencies": {
"node-fetch": "^2.6.0",
"node-match-path": "^0.4.4",
"type-fest": "^0.16.0"
}
}
207 changes: 204 additions & 3 deletions src/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,17 @@ import {
ArtifactoryRepository,
ArtifactoryPermission,
ArtifactoryPermissionRef,
ArtifactoryAccessToken,
ArtifactoryBuild,
ArtifactoryBuildRef,
ArtifactoryBuildResponse,
ArtifactoryArtifactRef,
ArtifactoryArtifactResponse,
ArtifactoryBuildArtifactsResponse,
ArtifactoryPipelineSource,
ArtifactoryBuildDetailsResponse,
ArtifactoryAccessTokenResponse,
ArtifactoryRepositoryName,
} from './types';

/**
Expand All @@ -25,11 +36,13 @@ export class APIClient {
private readonly clientNamespace: string;
private readonly clientAccessToken: string;
private readonly clientAdminName: string;
private readonly clientPipelineAccessToken: string;

constructor(readonly config: IntegrationConfig) {
this.clientNamespace = config.clientNamespace;
this.clientAccessToken = config.clientAccessToken;
this.clientAdminName = config.clientAdminName;
this.clientPipelineAccessToken = config.clientPipelineAccessToken;
}

private withBaseUri(path: string): string {
Expand All @@ -38,13 +51,18 @@ export class APIClient {

private async request(
uri: string,
method: 'GET' | 'HEAD' = 'GET',
method: 'GET' | 'HEAD' | 'POST' = 'GET',
body?,
headers = {},
): Promise<Response> {
return fetch(uri, {
method,
headers: {
Authorization: `Bearer ${this.clientAccessToken}`,
'Content-Type': 'application/json',
...headers,
},
body,
});
}

Expand Down Expand Up @@ -146,8 +164,31 @@ export class APIClient {
);

const repositories: ArtifactoryRepository[] = await response.json();
const wildcardRepositories: ArtifactoryRepository[] = [
{
url: this.withBaseUri('artifactory/*'),
description: 'ANY',
key: 'ANY' as ArtifactoryRepositoryName,
packageType: 'Generic',
type: 'ANY',
},
{
url: this.withBaseUri('artifactory/*?type=local'),
description: 'ANY LOCAL',
key: 'ANY LOCAL' as ArtifactoryRepositoryName,
packageType: 'Generic',
type: 'LOCAL',
},
{
url: this.withBaseUri('artifactory/*?type=remote'),
description: 'ANY REMOTE',
key: 'ANY REMOTE' as ArtifactoryRepositoryName,
packageType: 'Generic',
type: 'REMOTE',
},
];

for (const repository of repositories) {
for (const repository of [...repositories, ...wildcardRepositories]) {
await iteratee(repository);
}
}
Expand All @@ -161,16 +202,176 @@ export class APIClient {
iteratee: ResourceIteratee<ArtifactoryPermission>,
): Promise<void> {
const response = await this.request(
this.withBaseUri('artifactory/api/security/permissions'),
this.withBaseUri('artifactory/api/v2/security/permissions'),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No clue about this, but should the rest of the API calls also be to api/v2?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are the docs to the v2 api: https://www.jfrog.com/confluence/display/JFROG/Artifactory+REST+API+V2, so it only contains a few extending endpoints. We used the v2 here because the response format looked a bit better.

);

const permissionRefs: ArtifactoryPermissionRef[] = await response.json();

for (const permission of permissionRefs) {
const resp = await this.request(permission.uri);

await iteratee(await resp.json());
}
}

/**
* Iterates each access token in the provider.
*
* @param iteratee receives each resource to produce entities/relationships
*/
public async iterateAccessTokens(
iteratee: ResourceIteratee<ArtifactoryAccessToken>,
): Promise<void> {
const response = await this.request(
this.withBaseUri('artifactory/api/security/token'),
);

const accessTokens: ArtifactoryAccessTokenResponse = await response.json();

for (const accessToken of accessTokens.tokens || []) {
await iteratee(accessToken);
}
}

/**
* Iterates each repository artifact in the provider.
*
* @param iteratee receives each resource to produce entities/relationships
*/
public async iterateRepositoryArtifacts(
key: string,
iteratee: ResourceIteratee<ArtifactoryArtifactRef>,
): Promise<void> {
if (['ANY', 'ANY LOCAL', 'ANY REMOTE'].includes(key)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we capturing these artifacts anywhere in the integration? What's the implication of skipping them here?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The ANY, ANY LOCAL and ANY REMOTE are pretty much like magic values for the permissions targets, we create them manually in the graph so that any permissions can target those instead of creating edges for all the matching nodes. We thought this would make the graph perhaps a bit easier to understand. But would love to get your feedback on this.

That's also the reason we're skipping them: those repositories don't actually exist in the api.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense to me. If these are special, though, I would suggest using a different _class and _type. Probably Group and artifactory_repository_group, respectively. That way you won't have to filter them out when calling iterateRepositoryArtifacts.

return;
}

const response = await this.request(
this.withBaseUri(`artifactory/api/storage/${key}`),
);

await this.recurseArtifacts(await response.json(), iteratee);
}

private async recurseArtifacts(
response: ArtifactoryArtifactResponse,
iteratee: ResourceIteratee<ArtifactoryArtifactRef>,
): Promise<void> {
for (const artifact of response.children || []) {
if (artifact.folder) {
const nextUri = `${response.uri}${artifact.uri}`;
const nextResponse = await this.request(nextUri);

await this.recurseArtifacts(await nextResponse.json(), iteratee);
} else {
await iteratee({
...artifact,
uri: this.withBaseUri(
`artifactory/${response.repo}${response.path}${artifact.uri}`,
),
});
}
}
}

/**
* Iterates each build in the provider.
*
* @param iteratee receives each resource to produce entities/relationships
*/
public async iterateBuilds(
iteratee: ResourceIteratee<ArtifactoryBuild>,
): Promise<void> {
const response = await this.request(
this.withBaseUri('artifactory/api/build'),
);

const jsonResponse: ArtifactoryBuildResponse = await response.json();

// A list of artifactory/api/build/<name>
for (const build of jsonResponse.builds || []) {
const buildList = await this.getBuildList(build);

// A list of artifactory/api/build/<name>/<number>
for (const buildUri of buildList) {
const name = build.uri.split('/')[1];
const number = buildUri.split('/')[1];
const artifacts = await this.getBuildArtifacts(name, number);

if (artifacts.length === 0) {
return;
}

const repository = artifacts[0]
.split(this.withBaseUri('artifactory'))[1]
.split('/')[1];

await iteratee({
name,
number,
repository,
artifacts,
uri: this.withBaseUri(`ui/builds${build.uri}`),
});
}
}
}

private async getBuildList(buildRef: ArtifactoryBuildRef): Promise<string[]> {
const response = await this.request(
this.withBaseUri(`artifactory/api/build${buildRef.uri}`),
);

const jsonResponse: ArtifactoryBuildDetailsResponse = await response.json();

return (jsonResponse.buildsNumbers || []).map((b) => b.uri);
}

private async getBuildArtifacts(
name: string,
number: string,
): Promise<string[]> {
const response = await this.request(
this.withBaseUri('artifactory/api/search/buildArtifacts'),
'POST',
JSON.stringify({
buildName: name,
buildNumber: number,
}),
);

const jsonResponse: ArtifactoryBuildArtifactsResponse = await response.json();

if (jsonResponse.errors) {
return [];
}

return jsonResponse.results.map((r) => r.downloadUri);
}

/**
* Iterates each pipeline source in the provider.
*
* @param iteratee receives each resource to produce entities/relationships
*/
public async iteratePipelineSources(
iteratee: ResourceIteratee<ArtifactoryPipelineSource>,
): Promise<void> {
const response = await this.request(
this.withBaseUri('pipelines/api/v1/pipelinesources'),
'GET',
null,
{
Authorization: `Bearer ${this.clientPipelineAccessToken}`,
},
);

const sources: ArtifactoryPipelineSource[] = await response.json();

for (const source of sources || []) {
await iteratee(source);
}
}
}

export function createAPIClient(config: IntegrationConfig): APIClient {
Expand Down
Loading