Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: keep memory consumption sane when traversing history during fetch #6541

Merged
merged 2 commits into from
Oct 13, 2022
Merged

fix: keep memory consumption sane when traversing history during fetch #6541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7fbf3cd
fix: keep memory consumption sane when traversing history during fetch
davidfirst Oct 13, 2022
a11c236
Merge branch 'master' into fetch-with-deps-memory-fix
davidfirst Oct 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions src/scope/component-ops/traverse-versions.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
* methods here.
*/

import memoize from 'memoizee';
import pMapSeries from 'p-map-series';
import { HeadNotFound, ParentNotFound, VersionNotFound } from '../exceptions';
import { ModelComponent, Version } from '../models';
Expand Down Expand Up @@ -144,18 +145,26 @@ export async function getAllVersionHashesByVersionsObjects(
return allVersionsInfo.map((v) => v.ref).filter((ref) => ref) as Ref[];
}

export async function getAllVersionHashes(options: {
export type GetAllVersionHashesParams = {
modelComponent: ModelComponent;
repo: Repository;
throws?: boolean; // in case objects are missing. by default, it's true
versionObjects?: Version[];
startFrom?: Ref | null; // by default, start from the head
stopAt?: Ref[] | null; // by default, stop when the parents is empty
}): Promise<Ref[]> {
};

export async function getAllVersionHashes(options: GetAllVersionHashesParams): Promise<Ref[]> {
const allVersionsInfo = await getAllVersionsInfo(options);
return allVersionsInfo.map((v) => v.ref).filter((ref) => ref) as Ref[];
}

export const getAllVersionHashesMemoized = memoize(getAllVersionHashes, {
normalizer: (args) => JSON.stringify(args[0]),
promise: true,
maxAge: 1, // 1ms is good. it's only for consecutive calls while this function is still in process. we don't want to cache the results.
});

export async function hasVersionByRef(
modelComponent: ModelComponent,
ref: Ref,
Expand Down
18 changes: 7 additions & 11 deletions src/scope/objects/objects-readable-generator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { Ref, Repository } from '.';
import { Scope } from '..';
import ShowDoctorError from '../../error/show-doctor-error';
import logger from '../../logger/logger';
import { getAllVersionHashes } from '../component-ops/traverse-versions';
import { getAllVersionHashesMemoized } from '../component-ops/traverse-versions';
import { CollectObjectsOpts } from '../component-version';
import { HashMismatch } from '../exceptions';
import { Lane, ModelComponent, Version } from '../models';
Expand Down Expand Up @@ -106,22 +106,18 @@ export class ObjectsReadableGenerator {
type: component.getType(),
};
if (collectParents) {
const parentsObjects: ObjectItem[] = [];
const allParentsHashes = await getAllVersionHashes({
const allParentsHashes = await getAllVersionHashesMemoized({
modelComponent: component,
repo: this.repo,
startFrom: version.hash(),
stopAt: collectParentsUntil ? [collectParentsUntil] : undefined,
});
const missingParentsHashes = allParentsHashes.filter((h) => !h.isEqual(version.hash()));
await Promise.all(
missingParentsHashes.map(async (parentHash) => {
const parentVersion = (await parentHash.load(this.repo)) as Version;
const parentsObj = await collectVersionObjects(parentVersion);
parentsObjects.push(...parentsObj);
})
);
this.pushManyObjects(parentsObjects);
await pMapSeries(missingParentsHashes, async (parentHash) => {
const parentVersion = (await parentHash.load(this.repo)) as Version;
const parentsObj = await collectVersionObjects(parentVersion);
this.pushManyObjects(parentsObj);
});
}
const versionObjects = await collectVersionObjects(version);
this.pushManyObjects(versionObjects);
Expand Down