Skip to content

Commit

Permalink
Allow entries with same Job ID to match in different workflows
Browse files Browse the repository at this point in the history
Previously, the workflow name was always included when matching a cache entry for the current job.
This can be overly restrictive when job definitions are shared between different workflows. 
The workflow name is still encoded in the cache entry key, but not in the restore key searching for entries with a matching job.

Fixes #1017
  • Loading branch information
bigdaz authored Jan 4, 2024
1 parent 270f30b commit 32bab5b
Show file tree
Hide file tree
Showing 7 changed files with 19 additions and 80 deletions.
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -259,9 +259,9 @@ This allows the most recent state to always be available in the GitHub actions c
### Finding a matching cache entry

In most cases, no exact match will exist for the cache key. Instead, the Gradle User Home will be restored for the closest matching cache entry, using a set of "restore keys". The entries will be matched with the following precedence:
- An exact match on OS, workflow, job, matrix and Git SHA
- The most recent entry saved for the same OS, workflow, job and matrix values
- The most recent entry saved for the same OS, workflow and job
- An exact match on OS, workflow name, job id, matrix and Git SHA
- The most recent entry saved for the same OS, workflow name, job id and matrix values
- The most recent entry saved for the same OS and job id
- The most recent entry saved for the same OS

Due to branch scoping of cache entries, the above match will be first performed for entries from the same branch, and then for the default ('main') branch.
Expand Down
26 changes: 4 additions & 22 deletions dist/main/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -139505,7 +139505,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
});
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.tryDelete = exports.handleCacheFailure = exports.cacheDebug = exports.saveCache = exports.restoreCache = exports.hashStrings = exports.hashFileNames = exports.getUniqueLabelForJobInstanceValues = exports.getUniqueLabelForJobInstance = exports.getCacheKeyForJob = exports.getCacheKeyPrefix = exports.generateCacheKey = exports.CacheKey = exports.isCacheCleanupEnabled = exports.isCacheDebuggingEnabled = exports.isCacheOverwriteExisting = exports.isCacheWriteOnly = exports.isCacheReadOnly = exports.isCacheDisabled = void 0;
exports.tryDelete = exports.handleCacheFailure = exports.cacheDebug = exports.saveCache = exports.restoreCache = exports.hashStrings = exports.hashFileNames = exports.getCacheKeyPrefix = exports.generateCacheKey = exports.CacheKey = exports.isCacheCleanupEnabled = exports.isCacheDebuggingEnabled = exports.isCacheOverwriteExisting = exports.isCacheWriteOnly = exports.isCacheReadOnly = exports.isCacheDisabled = void 0;
const core = __importStar(__nccwpck_require__(42186));
const cache = __importStar(__nccwpck_require__(27799));
const github = __importStar(__nccwpck_require__(95438));
Expand Down Expand Up @@ -139577,34 +139577,16 @@ function getCacheKeyEnvironment() {
return process.env[CACHE_KEY_OS_VAR] || runnerOs;
}
function getCacheKeyJob() {
return process.env[CACHE_KEY_JOB_VAR] || getCacheKeyForJob(github.context.workflow, github.context.job);
return process.env[CACHE_KEY_JOB_VAR] || github.context.job;
}
function getCacheKeyForJob(workflowName, jobId) {
const sanitizedWorkflow = workflowName.replace(/,/g, '').toLowerCase();
return `${sanitizedWorkflow}-${jobId}`;
}
exports.getCacheKeyForJob = getCacheKeyForJob;
function getCacheKeyJobInstance() {
const override = process.env[CACHE_KEY_JOB_INSTANCE_VAR];
if (override) {
return override;
}
const workflowName = github.context.workflow;
const workflowJobContext = params.getJobMatrix();
return hashStrings([workflowJobContext]);
}
function getUniqueLabelForJobInstance() {
return getUniqueLabelForJobInstanceValues(github.context.workflow, github.context.job, params.getJobMatrix());
}
exports.getUniqueLabelForJobInstance = getUniqueLabelForJobInstance;
function getUniqueLabelForJobInstanceValues(workflow, jobId, matrixJson) {
const matrix = JSON.parse(matrixJson);
const matrixString = Object.values(matrix).join('-');
const label = matrixString ? `${workflow}-${jobId}-${matrixString}` : `${workflow}-${jobId}`;
return sanitize(label);
}
exports.getUniqueLabelForJobInstanceValues = getUniqueLabelForJobInstanceValues;
function sanitize(value) {
return value.replace(/[^a-zA-Z0-9_-]/g, '').toLowerCase();
return hashStrings([workflowName, workflowJobContext]);
}
function getCacheKeyJobExecution() {
return process.env[CACHE_KEY_JOB_EXECUTION_VAR] || github.context.sha;
Expand Down
2 changes: 1 addition & 1 deletion dist/main/index.js.map

Large diffs are not rendered by default.

26 changes: 4 additions & 22 deletions dist/post/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -136958,7 +136958,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
});
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.tryDelete = exports.handleCacheFailure = exports.cacheDebug = exports.saveCache = exports.restoreCache = exports.hashStrings = exports.hashFileNames = exports.getUniqueLabelForJobInstanceValues = exports.getUniqueLabelForJobInstance = exports.getCacheKeyForJob = exports.getCacheKeyPrefix = exports.generateCacheKey = exports.CacheKey = exports.isCacheCleanupEnabled = exports.isCacheDebuggingEnabled = exports.isCacheOverwriteExisting = exports.isCacheWriteOnly = exports.isCacheReadOnly = exports.isCacheDisabled = void 0;
exports.tryDelete = exports.handleCacheFailure = exports.cacheDebug = exports.saveCache = exports.restoreCache = exports.hashStrings = exports.hashFileNames = exports.getCacheKeyPrefix = exports.generateCacheKey = exports.CacheKey = exports.isCacheCleanupEnabled = exports.isCacheDebuggingEnabled = exports.isCacheOverwriteExisting = exports.isCacheWriteOnly = exports.isCacheReadOnly = exports.isCacheDisabled = void 0;
const core = __importStar(__nccwpck_require__(42186));
const cache = __importStar(__nccwpck_require__(27799));
const github = __importStar(__nccwpck_require__(95438));
Expand Down Expand Up @@ -137030,34 +137030,16 @@ function getCacheKeyEnvironment() {
return process.env[CACHE_KEY_OS_VAR] || runnerOs;
}
function getCacheKeyJob() {
return process.env[CACHE_KEY_JOB_VAR] || getCacheKeyForJob(github.context.workflow, github.context.job);
return process.env[CACHE_KEY_JOB_VAR] || github.context.job;
}
function getCacheKeyForJob(workflowName, jobId) {
const sanitizedWorkflow = workflowName.replace(/,/g, '').toLowerCase();
return `${sanitizedWorkflow}-${jobId}`;
}
exports.getCacheKeyForJob = getCacheKeyForJob;
function getCacheKeyJobInstance() {
const override = process.env[CACHE_KEY_JOB_INSTANCE_VAR];
if (override) {
return override;
}
const workflowName = github.context.workflow;
const workflowJobContext = params.getJobMatrix();
return hashStrings([workflowJobContext]);
}
function getUniqueLabelForJobInstance() {
return getUniqueLabelForJobInstanceValues(github.context.workflow, github.context.job, params.getJobMatrix());
}
exports.getUniqueLabelForJobInstance = getUniqueLabelForJobInstance;
function getUniqueLabelForJobInstanceValues(workflow, jobId, matrixJson) {
const matrix = JSON.parse(matrixJson);
const matrixString = Object.values(matrix).join('-');
const label = matrixString ? `${workflow}-${jobId}-${matrixString}` : `${workflow}-${jobId}`;
return sanitize(label);
}
exports.getUniqueLabelForJobInstanceValues = getUniqueLabelForJobInstanceValues;
function sanitize(value) {
return value.replace(/[^a-zA-Z0-9_-]/g, '').toLowerCase();
return hashStrings([workflowName, workflowJobContext]);
}
function getCacheKeyJobExecution() {
return process.env[CACHE_KEY_JOB_EXECUTION_VAR] || github.context.sha;
Expand Down
2 changes: 1 addition & 1 deletion dist/post/index.js.map

Large diffs are not rendered by default.

31 changes: 6 additions & 25 deletions src/cache-utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,10 @@ export function generateCacheKey(cacheName: string): CacheKey {
// At the most general level, share caches for all executions on the same OS
const cacheKeyForEnvironment = `${cacheKeyBase}|${getCacheKeyEnvironment()}`

// Prefer caches that run this job
// Then prefer caches that run job with the same ID
const cacheKeyForJob = `${cacheKeyForEnvironment}|${getCacheKeyJob()}`

// Prefer (even more) jobs that run this job with the same context (matrix)
// Prefer (even more) jobs that run this job in the same workflow with the same context (matrix)
const cacheKeyForJobContext = `${cacheKeyForJob}[${getCacheKeyJobInstance()}]`

// Exact match on Git SHA
Expand All @@ -113,12 +113,7 @@ function getCacheKeyEnvironment(): string {
}

function getCacheKeyJob(): string {
return process.env[CACHE_KEY_JOB_VAR] || getCacheKeyForJob(github.context.workflow, github.context.job)
}

export function getCacheKeyForJob(workflowName: string, jobId: string): string {
const sanitizedWorkflow = workflowName.replace(/,/g, '').toLowerCase()
return `${sanitizedWorkflow}-${jobId}`
return process.env[CACHE_KEY_JOB_VAR] || github.context.job
}

function getCacheKeyJobInstance(): string {
Expand All @@ -127,25 +122,11 @@ function getCacheKeyJobInstance(): string {
return override
}

// By default, we hash the full `matrix` data for the run, to uniquely identify this job invocation
// By default, we hash the workflow name and the full `matrix` data for the run, to uniquely identify this job invocation
// The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml.
const workflowName = github.context.workflow
const workflowJobContext = params.getJobMatrix()
return hashStrings([workflowJobContext])
}

export function getUniqueLabelForJobInstance(): string {
return getUniqueLabelForJobInstanceValues(github.context.workflow, github.context.job, params.getJobMatrix())
}

export function getUniqueLabelForJobInstanceValues(workflow: string, jobId: string, matrixJson: string): string {
const matrix = JSON.parse(matrixJson)
const matrixString = Object.values(matrix).join('-')
const label = matrixString ? `${workflow}-${jobId}-${matrixString}` : `${workflow}-${jobId}`
return sanitize(label)
}

function sanitize(value: string): string {
return value.replace(/[^a-zA-Z0-9_-]/g, '').toLowerCase()
return hashStrings([workflowName, workflowJobContext])
}

function getCacheKeyJobExecution(): string {
Expand Down
6 changes: 0 additions & 6 deletions test/jest/cache-utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,4 @@ describe('cacheUtils-utils', () => {
expect(posixHash).toBe(windowsHash)
})
})
describe('sanitizes workflow name in cache key', () => {
it('with comma', () => {
const cacheKey = cacheUtils.getCacheKeyForJob("Workflow, with,commas", "JOB_ID")
expect(cacheKey).toBe('workflow withcommas-JOB_ID')
})
})
})

0 comments on commit 32bab5b

Please sign in to comment.