From 8f9e092b5f911f180f761767733f862f5752e2c4 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Fri, 2 Jul 2021 17:16:50 +0200 Subject: [PATCH 1/3] feat(changelog): get from jsDelivr filelist if possible --- src/__tests__/changelog.test.ts | 36 ++++++++++++++++++++++++++++++++- src/changelog.ts | 27 ++++++++++++++++++++----- src/jsDelivr/index.ts | 18 +++++++++++------ 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/__tests__/changelog.test.ts b/src/__tests__/changelog.test.ts index 1d3040d43..18528f192 100644 --- a/src/__tests__/changelog.test.ts +++ b/src/__tests__/changelog.test.ts @@ -1,4 +1,5 @@ -import { getChangelogs, baseUrlMap } from '../changelog'; +import { getChangelogs, baseUrlMap, getChangelog } from '../changelog'; +import * as jsDelivr from '../jsDelivr'; jest.mock('got', () => { const gotSnapshotUrls = new Set([ @@ -16,6 +17,12 @@ jest.mock('got', () => { }; }); +const spy = jest + .spyOn(jsDelivr, 'getFilesList') + .mockImplementation((): Promise => { + return Promise.resolve([]); + }); + describe('should test baseUrlMap', () => { it('should work with paths', () => { const bitbucketRepo = { @@ -210,3 +217,30 @@ it('should work with HISTORY.md', async () => { 'https://raw.githubusercontent.com/expressjs/body-parser/master/HISTORY.md' ); }); + +describe('jsDelivr', () => { + it('should early return when finding changelog from jsDelivr', async () => { + spy.mockResolvedValue([ + { name: '/package.json', hash: '', time: '1', size: 1 }, + { name: '/CHANGELOG.md', hash: '', time: '1', size: 1 }, + ]); + + const { changelogFilename } = await getChangelog({ + name: 'foo', + version: '1.0.0', + repository: { + url: '', + host: 'github.com', + user: 'expressjs', + project: 'body-parser', + path: '', + head: 'master', + branch: 'master', + }, + }); + expect(jsDelivr.getFilesList).toHaveBeenCalled(); + expect(changelogFilename).toEqual( + 'https://cdn.jsdelivr.net/npm/foo@1.0.0/CHANGELOG.md' + ); + }); +}); diff --git a/src/changelog.ts b/src/changelog.ts index 6dca9aa67..d73f127d5 100644 --- a/src/changelog.ts +++ b/src/changelog.ts @@ -2,6 +2,7 @@ import race from 'promise-rat-race'; import type { RawPkg, Repo } from './@types/pkg'; import { config } from './config'; +import * as jsDelivr from './jsDelivr/index'; import { datadog } from './utils/datadog'; import { request } from './utils/request'; @@ -43,7 +44,10 @@ const fileOptions = [ 'history.md', 'HISTORY', 'history', + 'RELEASES.md', + 'RELEASES', ]; +const fileRegex = /changelog|changes|history|release/i; async function handledGot(file: string): Promise { const result = await request(file, { method: 'HEAD' }); @@ -76,13 +80,26 @@ async function raceFromPaths(files: string[]): Promise<{ } } -function getChangelog({ - repository, - name, - version, -}: Pick): Promise<{ +export async function getChangelog( + pkg: Pick +): Promise<{ changelogFilename: string | null; }> { + // Do a quick call to jsDelivr + // Only work if the package has published their changelog along with the code + const filesList = await jsDelivr.getFilesList(pkg); + for (const file of filesList) { + if (!fileRegex.test(file.name)) { + // eslint-disable-next-line no-continue + continue; + } + + return { changelogFilename: jsDelivr.getFullURL(pkg, file) }; + } + + const { repository, name, version } = pkg; + + // Rollback to brut-force const unpkgFiles = fileOptions.map( (file) => `${config.unpkgRoot}/${name}@${version}/${file}` ); diff --git a/src/jsDelivr/index.ts b/src/jsDelivr/index.ts index 7ef695288..38f6001ce 100644 --- a/src/jsDelivr/index.ts +++ b/src/jsDelivr/index.ts @@ -6,12 +6,13 @@ import { request } from '../utils/request'; type Hit = { type: 'npm'; name: string; hits: number }; type File = { name: string; hash: string; time: string; size: number }; -const hits = new Map(); + +export const hits = new Map(); /** * Load downloads hits. */ -async function loadHits(): Promise { +export async function loadHits(): Promise { const start = Date.now(); log.info('📦 Loading hits from jsDelivr'); @@ -34,7 +35,7 @@ async function loadHits(): Promise { /** * Get download hits. */ -function getHits(pkgs: Array>): Array<{ +export function getHits(pkgs: Array>): Array<{ jsDelivrHits: number; _searchInternal: { jsDelivrPopularity: number }; }> { @@ -59,7 +60,7 @@ function getHits(pkgs: Array>): Array<{ /** * Get packages files list. */ -async function getAllFilesList( +export async function getAllFilesList( pkgs: Array> ): Promise { const start = Date.now(); @@ -73,7 +74,7 @@ async function getAllFilesList( /** * Get one package files list. */ -async function getFilesList( +export async function getFilesList( pkg: Pick ): Promise { const start = Date.now(); @@ -100,4 +101,9 @@ async function getFilesList( return files; } -export { hits, loadHits, getHits, getAllFilesList, getFilesList }; +export function getFullURL( + pkg: Pick, + file: File +): string { + return `https://cdn.jsdelivr.net/npm/${pkg.name}@${pkg.version}${file.name}`; +} From ecd0fdfe04ecd77c47efbe59174597a4d1dcf79a Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Mon, 5 Jul 2021 11:53:04 +0200 Subject: [PATCH 2/3] improve regex --- src/__tests__/changelog.test.ts | 48 +++++++++++++++++++++++++++++++- src/changelog.ts | 49 +++++++++++++++++++++------------ 2 files changed, 79 insertions(+), 18 deletions(-) diff --git a/src/__tests__/changelog.test.ts b/src/__tests__/changelog.test.ts index 18528f192..74082c6d0 100644 --- a/src/__tests__/changelog.test.ts +++ b/src/__tests__/changelog.test.ts @@ -219,7 +219,7 @@ it('should work with HISTORY.md', async () => { }); describe('jsDelivr', () => { - it('should early return when finding changelog from jsDelivr', async () => { + it('should early return when finding changelog', async () => { spy.mockResolvedValue([ { name: '/package.json', hash: '', time: '1', size: 1 }, { name: '/CHANGELOG.md', hash: '', time: '1', size: 1 }, @@ -243,4 +243,50 @@ describe('jsDelivr', () => { 'https://cdn.jsdelivr.net/npm/foo@1.0.0/CHANGELOG.md' ); }); + + it('should early return when finding changelog in nested file', async () => { + spy.mockResolvedValue([ + { name: '/pkg/CHANGELOG.md', hash: '', time: '1', size: 1 }, + ]); + + const { changelogFilename } = await getChangelog({ + name: 'foo', + version: '1.0.0', + repository: { + url: '', + host: 'github.com', + user: 'expressjs', + project: 'body-parser', + path: '', + head: 'master', + branch: 'master', + }, + }); + expect(jsDelivr.getFilesList).toHaveBeenCalled(); + expect(changelogFilename).toEqual( + 'https://cdn.jsdelivr.net/npm/foo@1.0.0/pkg/CHANGELOG.md' + ); + }); + + it('should not register a file looking like a changelog', async () => { + spy.mockResolvedValue([ + { name: '/dist/changelog.js', hash: '', time: '1', size: 1 }, + ]); + + const { changelogFilename } = await getChangelog({ + name: 'foo', + version: '1.0.0', + repository: { + url: '', + host: 'github.com', + user: 'hello', + project: 'foo', + path: '', + head: 'master', + branch: 'master', + }, + }); + expect(jsDelivr.getFilesList).toHaveBeenCalled(); + expect(changelogFilename).toEqual(null); + }); }); diff --git a/src/changelog.ts b/src/changelog.ts index d73f127d5..81998e523 100644 --- a/src/changelog.ts +++ b/src/changelog.ts @@ -1,3 +1,5 @@ +import path from 'path'; + import race from 'promise-rat-race'; import type { RawPkg, Repo } from './@types/pkg'; @@ -10,21 +12,30 @@ export const baseUrlMap = new Map< string, (opts: Pick) => string >(); -baseUrlMap.set('github.com', ({ user, project, path, branch }): string => { - return `https://raw.githubusercontent.com/${user}/${project}/${ - path ? '' : branch - }${`${path.replace('/tree/', '')}`}`; -}); -baseUrlMap.set('gitlab.com', ({ user, project, path, branch }): string => { - return `https://gitlab.com/${user}/${project}${ - path ? path.replace('tree', 'raw') : `/raw/${branch}` - }`; -}); -baseUrlMap.set('bitbucket.org', ({ user, project, path, branch }): string => { - return `https://bitbucket.org/${user}/${project}${ - path ? path.replace('src', 'raw') : `/raw/${branch}` - }`; -}); +baseUrlMap.set( + 'github.com', + ({ user, project, path: pathName, branch }): string => { + return `https://raw.githubusercontent.com/${user}/${project}/${ + pathName ? '' : branch + }${`${pathName.replace('/tree/', '')}`}`; + } +); +baseUrlMap.set( + 'gitlab.com', + ({ user, project, path: pathName, branch }): string => { + return `https://gitlab.com/${user}/${project}${ + pathName ? pathName.replace('tree', 'raw') : `/raw/${branch}` + }`; + } +); +baseUrlMap.set( + 'bitbucket.org', + ({ user, project, path: pathName, branch }): string => { + return `https://bitbucket.org/${user}/${project}${ + pathName ? pathName.replace('src', 'raw') : `/raw/${branch}` + }`; + } +); const fileOptions = [ 'CHANGELOG.md', @@ -47,7 +58,10 @@ const fileOptions = [ 'RELEASES.md', 'RELEASES', ]; -const fileRegex = /changelog|changes|history|release/i; + +// https://regex101.com/r/zU2gjr/1 +const fileRegex = + /^(((changelogs?)|changes|history|(releases?)))((.(md|markdown))?$)/i; async function handledGot(file: string): Promise { const result = await request(file, { method: 'HEAD' }); @@ -89,7 +103,8 @@ export async function getChangelog( // Only work if the package has published their changelog along with the code const filesList = await jsDelivr.getFilesList(pkg); for (const file of filesList) { - if (!fileRegex.test(file.name)) { + const name = path.basename(file.name); + if (!fileRegex.test(name)) { // eslint-disable-next-line no-continue continue; } From bb53531b6a58caab600b852dda0e8879ed05ca21 Mon Sep 17 00:00:00 2001 From: Samuel Bodin Date: Mon, 5 Jul 2021 14:44:20 +0200 Subject: [PATCH 3/3] Update src/changelog.ts Co-authored-by: Haroen Viaene --- src/changelog.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/changelog.ts b/src/changelog.ts index 81998e523..f94708884 100644 --- a/src/changelog.ts +++ b/src/changelog.ts @@ -114,7 +114,7 @@ export async function getChangelog( const { repository, name, version } = pkg; - // Rollback to brut-force + // Rollback to brute-force the source code const unpkgFiles = fileOptions.map( (file) => `${config.unpkgRoot}/${name}@${version}/${file}` );