Skip to content

Commit

Permalink
Merge pull request #374 from eljog/eljog/cache-userenvprob
Browse files Browse the repository at this point in the history 
Cache user env for performance improvement
  • Loading branch information
@samruddhikhandale
samruddhikhandale authored Jan 18, 2023
2 parents 43a1bb3 + 5ab5f1d commit 83c8030
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 10 deletions.
80 changes: 70 additions & 10 deletions src/spec-common/injectHeadless.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ export interface ResolverParameters {
skipFeatureAutoMapping: boolean;
skipPostAttach: boolean;
experimentalImageMetadata: boolean;
containerSessionDataFolder?: string;
skipPersistingCustomizationsFromFeatures: boolean;
}

Expand Down Expand Up @@ -667,22 +668,72 @@ async function patchEtcProfile(params: ResolverParameters, containerProperties:
}
}

async function probeUserEnv(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise<ShellServer>); user?: string }, config?: CommonMergedDevContainerConfig) {
const env = await runUserEnvProbe(params, containerProperties, config, 'cat /proc/self/environ', '\0');
async function probeUserEnv(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log; containerSessionDataFolder?: string }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise<ShellServer>); user?: string }, config?: CommonMergedDevContainerConfig) {
let userEnvProbe = getUserEnvProb(config, params);
if (!userEnvProbe || userEnvProbe === 'none') {
return {};
}

let env = await readUserEnvFromCache(userEnvProbe, params, containerProperties.shellServer);
if (env) {
return env;
}
params.output.write('userEnvProbe: falling back to printenv');
const env2 = await runUserEnvProbe(params, containerProperties, config, 'printenv', '\n');
return env2 || {};

params.output.write('userEnvProbe: not found in cache');
env = await runUserEnvProbe(userEnvProbe, params, containerProperties, 'cat /proc/self/environ', '\0');
if (!env) {
params.output.write('userEnvProbe: falling back to printenv');
env = await runUserEnvProbe(userEnvProbe, params, containerProperties, 'printenv', '\n');
}

if (env) {
await updateUserEnvCache(env, userEnvProbe, params, containerProperties.shellServer);
}

return env || {};
}

async function runUserEnvProbe(params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise<ShellServer>); user?: string }, config: CommonMergedDevContainerConfig | undefined, cmd: string, sep: string) {
let userEnvProbe = config?.userEnvProbe;
params.output.write(`userEnvProbe: ${userEnvProbe || params.defaultUserEnvProbe}${userEnvProbe ? '' : ' (default)'}`);
if (!userEnvProbe) {
userEnvProbe = params.defaultUserEnvProbe;
async function readUserEnvFromCache(userEnvProbe: UserEnvProbe, params: { output: Log; containerSessionDataFolder?: string }, shellServer?: ShellServer) {
if (!shellServer || !params.containerSessionDataFolder) {
return undefined;
}

const cacheFile = getUserEnvCacheFilePath(userEnvProbe, params.containerSessionDataFolder);
try {
if (await isFile(shellServer, cacheFile)) {
const { stdout } = await shellServer.exec(`cat '${cacheFile}'`);
return JSON.parse(stdout);
}
}
catch (e) {
params.output.write(`Failed to read/parse user env cache: ${e}`, LogLevel.Error);
}

return undefined;
}

async function updateUserEnvCache(env: Record<string, string>, userEnvProbe: UserEnvProbe, params: { output: Log; containerSessionDataFolder?: string }, shellServer?: ShellServer) {
if (!shellServer || !params.containerSessionDataFolder) {
return;
}

const cacheFile = getUserEnvCacheFilePath(userEnvProbe, params.containerSessionDataFolder);
try {
await shellServer.exec(`mkdir -p '${path.posix.dirname(cacheFile)}' && cat > '${cacheFile}' << 'envJSON'
${JSON.stringify(env, null, '\t')}
envJSON
`);
}
catch (e) {
params.output.write(`Failed to cache user env: ${e}`, LogLevel.Error);
}
}

function getUserEnvCacheFilePath(userEnvProbe: UserEnvProbe, cacheFolder: string): string {
return path.posix.join(cacheFolder, `env-${userEnvProbe}.json`);
}

async function runUserEnvProbe(userEnvProbe: UserEnvProbe, params: { allowSystemConfigChange: boolean; output: Log }, containerProperties: { shell: string; remoteExec: ExecFunction; installFolder?: string; env?: NodeJS.ProcessEnv; shellServer?: ShellServer; launchRootShellServer?: (() => Promise<ShellServer>); user?: string }, cmd: string, sep: string) {
if (userEnvProbe === 'none') {
return {};
}
Expand Down Expand Up @@ -779,6 +830,15 @@ Merged: ${typeof env.PATH === 'string' ? `'${env.PATH}'` : 'None'}` : ''}`);
}
}

function getUserEnvProb(config: CommonMergedDevContainerConfig | undefined, params: { defaultUserEnvProbe: UserEnvProbe; allowSystemConfigChange: boolean; output: Log }) {
let userEnvProbe = config?.userEnvProbe;
params.output.write(`userEnvProbe: ${userEnvProbe || params.defaultUserEnvProbe}${userEnvProbe ? '' : ' (default)'}`);
if (!userEnvProbe) {
userEnvProbe = params.defaultUserEnvProbe;
}
return userEnvProbe;
}

function mergePaths(shellPath: string, containerPath: string, rootUser: boolean) {
const result = shellPath.split(':');
let insertAt = 0;
Expand Down
2 changes: 2 additions & 0 deletions src/spec-node/devContainers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ export interface ProvisionOptions {
skipFeatureAutoMapping: boolean;
skipPostAttach: boolean;
experimentalImageMetadata: boolean;
containerSessionDataFolder?: string;
skipPersistingCustomizationsFromFeatures: boolean;
dotfiles: {
repository?: string;
Expand Down Expand Up @@ -137,6 +138,7 @@ export async function createDockerParams(options: ProvisionOptions, disposables:
skipFeatureAutoMapping: options.skipFeatureAutoMapping,
skipPostAttach: options.skipPostAttach,
experimentalImageMetadata: options.experimentalImageMetadata,
containerSessionDataFolder: options.containerSessionDataFolder,
skipPersistingCustomizationsFromFeatures: options.skipPersistingCustomizationsFromFeatures,
dotfilesConfiguration: {
repository: options.dotfiles.repository,
Expand Down
6 changes: 6 additions & 0 deletions src/spec-node/devContainersSpecCLI.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ function provisionOptions(y: Argv) {
'dotfiles-repository': { type: 'string', description: 'URL of a dotfiles Git repository (e.g., https://github.com/owner/repository.git)' },
'dotfiles-install-command': { type: 'string', description: 'The command to run after cloning the dotfiles repository. Defaults to run the first file of `install.sh`, `install`, `bootstrap.sh`, `bootstrap`, `setup.sh` and `setup` found in the dotfiles repository`s root folder.' },
'dotfiles-target-path': { type: 'string', default: '~/dotfiles', description: 'The path to clone the dotfiles repository to. Defaults to `~/dotfiles`.' },
'container-session-data-folder': { type: 'string', description: 'Folder to cache CLI data, for example userEnvProb results' },
})
.check(argv => {
const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined;
Expand Down Expand Up @@ -185,6 +186,7 @@ async function provision({
'dotfiles-repository': dotfilesRepository,
'dotfiles-install-command': dotfilesInstallCommand,
'dotfiles-target-path': dotfilesTargetPath,
'container-session-data-folder': containerSessionDataFolder,
}: ProvisionArgs) {

const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : undefined;
Expand Down Expand Up @@ -239,6 +241,7 @@ async function provision({
skipFeatureAutoMapping,
skipPostAttach,
experimentalImageMetadata,
containerSessionDataFolder,
skipPersistingCustomizationsFromFeatures: false,
};

Expand Down Expand Up @@ -528,6 +531,7 @@ function runUserCommandsOptions(y: Argv) {
'dotfiles-repository': { type: 'string', description: 'URL of a dotfiles Git repository (e.g., https://github.com/owner/repository.git)' },
'dotfiles-install-command': { type: 'string', description: 'The command to run after cloning the dotfiles repository. Defaults to run the first file of `install.sh`, `install`, `bootstrap.sh`, `bootstrap`, `setup.sh` and `setup` found in the dotfiles repository`s root folder.' },
'dotfiles-target-path': { type: 'string', default: '~/dotfiles', description: 'The path to clone the dotfiles repository to. Defaults to `~/dotfiles`.' },
'container-session-data-folder': { type: 'string', description: 'Folder to cache CLI data, for example userEnvProb results' },
})
.check(argv => {
const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined;
Expand Down Expand Up @@ -582,6 +586,7 @@ async function doRunUserCommands({
'dotfiles-repository': dotfilesRepository,
'dotfiles-install-command': dotfilesInstallCommand,
'dotfiles-target-path': dotfilesTargetPath,
'container-session-data-folder': containerSessionDataFolder,
}: RunUserCommandsArgs) {
const disposables: (() => Promise<unknown> | undefined)[] = [];
const dispose = async () => {
Expand Down Expand Up @@ -632,6 +637,7 @@ async function doRunUserCommands({
installCommand: dotfilesInstallCommand,
targetPath: dotfilesTargetPath,
},
containerSessionDataFolder,
}, disposables);

const { common } = params;
Expand Down

0 comments on commit 83c8030

Please sign in to comment.