From a63b92839881832bb9f124ecba698e344de1d541 Mon Sep 17 00:00:00 2001 From: Samuel Bodin <1637651+bodinsamuel@users.noreply.github.com> Date: Mon, 5 Jul 2021 17:08:19 +0200 Subject: [PATCH] feat(ts): use jsdelivr to check for d.ts --- src/@types/pkg.ts | 2 +- .../__snapshots__/formatPkg.test.ts.snap | 7 - src/__tests__/changelog.test.ts | 119 ++++++------- src/__tests__/formatPkg.test.ts | 35 ---- src/changelog.ts | 29 ++-- src/formatPkg.ts | 14 +- src/jsDelivr/index.ts | 2 +- src/npm/__tests__/index.test.ts | 3 - src/saveDocs.ts | 10 +- src/typescript/index.test.ts | 164 +++++++++--------- src/typescript/index.ts | 41 ++--- src/unpkg/index.test.ts | 17 -- src/unpkg/index.ts | 21 --- 13 files changed, 183 insertions(+), 281 deletions(-) delete mode 100644 src/unpkg/index.test.ts delete mode 100644 src/unpkg/index.ts diff --git a/src/@types/pkg.ts b/src/@types/pkg.ts index 7be7e0078..a44ef6682 100644 --- a/src/@types/pkg.ts +++ b/src/@types/pkg.ts @@ -25,7 +25,7 @@ export interface GithubRepo { export type TsType = | { - ts: 'included' | false | { possible: true; dtsMain: string }; + ts: 'included' | false | { possible: true }; } | { ts: 'definitely-typed'; diff --git a/src/__tests__/__snapshots__/formatPkg.test.ts.snap b/src/__tests__/__snapshots__/formatPkg.test.ts.snap index 05df3e895..a4aa75956 100644 --- a/src/__tests__/__snapshots__/formatPkg.test.ts.snap +++ b/src/__tests__/__snapshots__/formatPkg.test.ts.snap @@ -65,7 +65,6 @@ Object { }, "types": Object { "ts": Object { - "dtsMain": "index.d.ts", "possible": true, }, }, @@ -881,7 +880,6 @@ Are you in trouble? Read through our [contribution guidelines](https://bitbucket }, "types": Object { "ts": Object { - "dtsMain": "dist/bundle.d.ts", "possible": true, }, }, @@ -1022,7 +1020,6 @@ Object { }, "types": Object { "ts": Object { - "dtsMain": "dist/atomic-package.tab.d.ts", "possible": true, }, }, @@ -1154,7 +1151,6 @@ Object { }, "types": Object { "ts": Object { - "dtsMain": "index.d.ts", "possible": true, }, }, @@ -1242,7 +1238,6 @@ index(arr, obj); }, "types": Object { "ts": Object { - "dtsMain": "index.d.ts", "possible": true, }, }, @@ -1418,7 +1413,6 @@ Thank you so much for contributing!! }, "types": Object { "ts": Object { - "dtsMain": "prism.d.ts", "possible": true, }, }, @@ -1520,7 +1514,6 @@ Object { "tags": undefined, "types": Object { "ts": Object { - "dtsMain": "index.d.ts", "possible": true, }, }, diff --git a/src/__tests__/changelog.test.ts b/src/__tests__/changelog.test.ts index 74082c6d0..95222ba78 100644 --- a/src/__tests__/changelog.test.ts +++ b/src/__tests__/changelog.test.ts @@ -1,5 +1,4 @@ import { getChangelogs, baseUrlMap, getChangelog } from '../changelog'; -import * as jsDelivr from '../jsDelivr'; jest.mock('got', () => { const gotSnapshotUrls = new Set([ @@ -17,12 +16,6 @@ jest.mock('got', () => { }; }); -const spy = jest - .spyOn(jsDelivr, 'getFilesList') - .mockImplementation((): Promise => { - return Promise.resolve([]); - }); - describe('should test baseUrlMap', () => { it('should work with paths', () => { const bitbucketRepo = { @@ -116,7 +109,7 @@ it('should handle not found changelog for github', async () => { }, }; - const [{ changelogFilename }] = await getChangelogs([pkg]); + const [{ changelogFilename }] = await getChangelogs([pkg], []); expect(changelogFilename).toBe(null); }); @@ -135,7 +128,7 @@ it('should get changelog for github', async () => { }, }; - const [{ changelogFilename }] = await getChangelogs([pkg]); + const [{ changelogFilename }] = await getChangelogs([pkg], []); expect(changelogFilename).toBe( 'https://raw.githubusercontent.com/algolia/algoliasearch-netlify/master/CHANGELOG.md' ); @@ -148,7 +141,7 @@ it('should get changelog from unpkg if there is no repository field', async () = repository: null, }; - const [{ changelogFilename }] = await getChangelogs([pkg]); + const [{ changelogFilename }] = await getChangelogs([pkg], []); expect(changelogFilename).toBe( 'https://unpkg.com/@atlaskit/button@13.3.7/CHANGELOG.md' @@ -170,7 +163,7 @@ it('should get changelog for gitlab', async () => { }, }; - const [{ changelogFilename }] = await getChangelogs([pkg]); + const [{ changelogFilename }] = await getChangelogs([pkg], []); expect(changelogFilename).toBe( 'https://gitlab.com/janslow/gitlab-fetch/raw/master/CHANGELOG.md' ); @@ -191,7 +184,7 @@ it('should get changelog for bitbucket', async () => { }, }; - const [{ changelogFilename }] = await getChangelogs([pkg]); + const [{ changelogFilename }] = await getChangelogs([pkg], []); expect(changelogFilename).toBe( 'https://bitbucket.org/atlassian/aui/raw/master/changelog.md' ); @@ -212,7 +205,7 @@ it('should work with HISTORY.md', async () => { }, }; - const [{ changelogFilename }] = await getChangelogs([pkg]); + const [{ changelogFilename }] = await getChangelogs([pkg], []); expect(changelogFilename).toBe( 'https://raw.githubusercontent.com/expressjs/body-parser/master/HISTORY.md' ); @@ -220,73 +213,69 @@ it('should work with HISTORY.md', async () => { describe('jsDelivr', () => { it('should early return when finding changelog', async () => { - spy.mockResolvedValue([ - { name: '/package.json', hash: '', time: '1', size: 1 }, - { name: '/CHANGELOG.md', hash: '', time: '1', size: 1 }, - ]); - - const { changelogFilename } = await getChangelog({ - name: 'foo', - version: '1.0.0', - repository: { - url: '', - host: 'github.com', - user: 'expressjs', - project: 'body-parser', - path: '', - head: 'master', - branch: 'master', + const { changelogFilename } = await getChangelog( + { + name: 'foo', + version: '1.0.0', + repository: { + url: '', + host: 'github.com', + user: 'expressjs', + project: 'body-parser', + path: '', + head: 'master', + branch: 'master', + }, }, - }); - expect(jsDelivr.getFilesList).toHaveBeenCalled(); + [ + { name: '/package.json', hash: '', time: '1', size: 1 }, + { name: '/CHANGELOG.md', hash: '', time: '1', size: 1 }, + ] + ); expect(changelogFilename).toEqual( 'https://cdn.jsdelivr.net/npm/foo@1.0.0/CHANGELOG.md' ); }); it('should early return when finding changelog in nested file', async () => { - spy.mockResolvedValue([ - { name: '/pkg/CHANGELOG.md', hash: '', time: '1', size: 1 }, - ]); - - const { changelogFilename } = await getChangelog({ - name: 'foo', - version: '1.0.0', - repository: { - url: '', - host: 'github.com', - user: 'expressjs', - project: 'body-parser', - path: '', - head: 'master', - branch: 'master', + const { changelogFilename } = await getChangelog( + { + name: 'foo', + version: '1.0.0', + repository: { + url: '', + host: 'github.com', + user: 'expressjs', + project: 'body-parser', + path: '', + head: 'master', + branch: 'master', + }, }, - }); - expect(jsDelivr.getFilesList).toHaveBeenCalled(); + [{ name: '/pkg/CHANGELOG.md', hash: '', time: '1', size: 1 }] + ); expect(changelogFilename).toEqual( 'https://cdn.jsdelivr.net/npm/foo@1.0.0/pkg/CHANGELOG.md' ); }); it('should not register a file looking like a changelog', async () => { - spy.mockResolvedValue([ - { name: '/dist/changelog.js', hash: '', time: '1', size: 1 }, - ]); - - const { changelogFilename } = await getChangelog({ - name: 'foo', - version: '1.0.0', - repository: { - url: '', - host: 'github.com', - user: 'hello', - project: 'foo', - path: '', - head: 'master', - branch: 'master', + const { changelogFilename } = await getChangelog( + { + name: 'foo', + version: '1.0.0', + repository: { + url: '', + host: 'github.com', + user: 'hello', + project: 'foo', + path: '', + head: 'master', + branch: 'master', + }, }, - }); - expect(jsDelivr.getFilesList).toHaveBeenCalled(); + [{ name: '/dist/changelog.js', hash: '', time: '1', size: 1 }] + ); expect(changelogFilename).toEqual(null); }); }); diff --git a/src/__tests__/formatPkg.test.ts b/src/__tests__/formatPkg.test.ts index a0419e564..e45ee24f3 100644 --- a/src/__tests__/formatPkg.test.ts +++ b/src/__tests__/formatPkg.test.ts @@ -245,41 +245,6 @@ describe('adds TypeScript information', () => { }) ).toEqual(expect.objectContaining({ types: { ts: 'included' } })); }); - - it('adds types possible if we can find a main file', () => { - expect( - formatPkg({ - ...BASE, - name: 'xxx', - main: 'main.js', - }) - ).toEqual( - expect.objectContaining({ - types: { ts: { possible: true, dtsMain: 'main.d.ts' } }, - }) - ); - - expect( - formatPkg({ - ...BASE, - name: 'xxx', - }) - ).toEqual( - expect.objectContaining({ - types: { ts: { possible: true, dtsMain: 'index.d.ts' } }, - }) - ); - }); - - it('gives up when no main is not js', () => { - expect( - formatPkg({ - ...BASE, - name: 'xxx', - main: 'shell-script.sh', - }) - ).toEqual(expect.objectContaining({ types: { ts: false } })); - }); }); describe('getRepositoryInfo', () => { diff --git a/src/changelog.ts b/src/changelog.ts index f94708884..7e87f3eb2 100644 --- a/src/changelog.ts +++ b/src/changelog.ts @@ -95,23 +95,25 @@ async function raceFromPaths(files: string[]): Promise<{ } export async function getChangelog( - pkg: Pick + pkg: Pick, + filelist: jsDelivr.File[] ): Promise<{ changelogFilename: string | null; }> { - // Do a quick call to jsDelivr - // Only work if the package has published their changelog along with the code - const filesList = await jsDelivr.getFilesList(pkg); - for (const file of filesList) { + for (const file of filelist) { const name = path.basename(file.name); if (!fileRegex.test(name)) { // eslint-disable-next-line no-continue continue; } + datadog.increment('jsdelivr.getChangelog.hit'); + return { changelogFilename: jsDelivr.getFullURL(pkg, file) }; } + datadog.increment('jsdelivr.getChangelog.miss'); + const { repository, name, version } = pkg; // Rollback to brute-force the source code @@ -120,19 +122,19 @@ export async function getChangelog( ); if (repository === null) { - return raceFromPaths(unpkgFiles); + return await raceFromPaths(unpkgFiles); } const user = repository.user || ''; const project = repository.project || ''; const host = repository.host || ''; if (user.length < 1 || project.length < 1) { - return raceFromPaths(unpkgFiles); + return await raceFromPaths(unpkgFiles); } // Check if we know how to handle this host if (!baseUrlMap.has(host)) { - return raceFromPaths(unpkgFiles); + return await raceFromPaths(unpkgFiles); } const baseUrl = baseUrlMap.get(host)!(repository); @@ -141,11 +143,12 @@ export async function getChangelog( [baseUrl.replace(/\/$/, ''), file].join('/') ); - return raceFromPaths([...files, ...unpkgFiles]); + return await raceFromPaths([...files, ...unpkgFiles]); } export async function getChangelogs( - pkgs: Array> + pkgs: Array>, + filelists: jsDelivr.File[][] ): Promise< Array<{ changelogFilename: string | null; @@ -153,7 +156,11 @@ export async function getChangelogs( > { const start = Date.now(); - const all = await Promise.all(pkgs.map(getChangelog)); + const all = await Promise.all( + pkgs.map((pkg, index) => { + return getChangelog(pkg, filelists[index] || []); + }) + ); datadog.timing('changelogs.getChangelogs', Date.now() - start); return all; diff --git a/src/formatPkg.ts b/src/formatPkg.ts index e7cabef70..844222144 100644 --- a/src/formatPkg.ts +++ b/src/formatPkg.ts @@ -531,20 +531,8 @@ function getTypes(pkg: NicePackageType): RawPkg['types'] { return { ts: 'included' }; } - // we only look at the first entry in main here - const main = getMains(pkg)[0]; - if (typeof main === 'string' && main.endsWith('.js')) { - const dtsMain = main.replace(/js$/, 'd.ts'); - return { - ts: { - possible: true, - dtsMain, - }, - }; - } - return { - ts: false, + ts: { possible: true }, }; } diff --git a/src/jsDelivr/index.ts b/src/jsDelivr/index.ts index 38f6001ce..2f20a3301 100644 --- a/src/jsDelivr/index.ts +++ b/src/jsDelivr/index.ts @@ -5,7 +5,7 @@ import { log } from '../utils/log'; import { request } from '../utils/request'; type Hit = { type: 'npm'; name: string; hits: number }; -type File = { name: string; hash: string; time: string; size: number }; +export type File = { name: string; hash: string; time: string; size: number }; export const hits = new Map(); diff --git a/src/npm/__tests__/index.test.ts b/src/npm/__tests__/index.test.ts index f9394a14b..7249fd1e0 100644 --- a/src/npm/__tests__/index.test.ts +++ b/src/npm/__tests__/index.test.ts @@ -183,9 +183,6 @@ describe('getDownloads()', () => { pkg.downloadsLast30Days.toString() ); - // eslint-disable-next-line no-console - console.log('downloads', { jest, angular, holmes }); - expect(jest.length).toBeGreaterThanOrEqual(6); expect(jest.length).toBeLessThanOrEqual(8); diff --git a/src/saveDocs.ts b/src/saveDocs.ts index 4741b7fca..404b63989 100644 --- a/src/saveDocs.ts +++ b/src/saveDocs.ts @@ -57,12 +57,16 @@ export default async function saveDocs({ } async function addMetaData(pkgs: RawPkg[]): Promise { - const [downloads, dependents, changelogs, hits, ts] = await Promise.all([ + const [downloads, dependents, hits, filelists] = await Promise.all([ npm.getDownloads(pkgs), npm.getDependents(pkgs), - getChangelogs(pkgs), jsDelivr.getHits(pkgs), - getTSSupport(pkgs), + jsDelivr.getAllFilesList(pkgs), + ]); + + const [changelogs, ts] = await Promise.all([ + getChangelogs(pkgs, filelists), + getTSSupport(pkgs, filelists), ]); const start = Date.now(); diff --git a/src/typescript/index.test.ts b/src/typescript/index.test.ts index ce55a9c39..a16cb23f5 100644 --- a/src/typescript/index.test.ts +++ b/src/typescript/index.test.ts @@ -1,11 +1,5 @@ -import * as npm from '../npm'; -import { fileExistsInUnpkg } from '../unpkg'; - import * as api from './index'; -jest.mock('../npm'); -jest.mock('../unpkg'); - describe('loadTypesIndex()', () => { it('should download and cache all @types', async () => { expect(api.typesCache).not.toHaveProperty('algoliasearch'); @@ -26,32 +20,41 @@ describe('loadTypesIndex()', () => { }); describe('getTypeScriptSupport()', () => { - it('If types are already calculated - return early', async () => { - const typesSupport = await api.getTypeScriptSupport({ - name: 'Has Types', - types: { ts: 'included' }, - version: '1.0', - }); + it('If types are already calculated - return early', () => { + const typesSupport = api.getTypeScriptSupport( + { + name: 'Has Types', + types: { ts: 'included' }, + version: '1.0', + }, + [] + ); expect(typesSupport).toEqual({ types: { ts: 'included' } }); }); - it('Handles not having any possible TS types', async () => { - const typesSupport = await api.getTypeScriptSupport({ - name: 'my-lib', - types: { ts: false }, - version: '1.0', - }); + it('Handles not having any possible TS types', () => { + const typesSupport = api.getTypeScriptSupport( + { + name: 'my-lib', + types: { ts: false }, + version: '1.0', + }, + [] + ); expect(typesSupport).toEqual({ types: { ts: false } }); }); describe('Definitely Typed', () => { - it('Checks for @types/[name]', async () => { - const atTypesSupport = await api.getTypeScriptSupport({ - name: 'lodash.valuesin', - types: { ts: false }, - version: '1.0', - }); + it('Checks for @types/[name]', () => { + const atTypesSupport = api.getTypeScriptSupport( + { + name: 'lodash.valuesin', + types: { ts: false }, + version: '1.0', + }, + [] + ); expect(atTypesSupport).toEqual({ types: { ts: 'definitely-typed', @@ -60,12 +63,15 @@ describe('getTypeScriptSupport()', () => { }); }); - it('Checks for @types/[scope__name]', async () => { - const atTypesSupport = await api.getTypeScriptSupport({ - name: '@mapbox/geojson-area', - types: { ts: false }, - version: '1.0', - }); + it('Checks for @types/[scope__name]', () => { + const atTypesSupport = api.getTypeScriptSupport( + { + name: '@mapbox/geojson-area', + types: { ts: false }, + version: '1.0', + }, + [] + ); expect(atTypesSupport).toEqual({ types: { ts: 'definitely-typed', @@ -73,11 +79,14 @@ describe('getTypeScriptSupport()', () => { }, }); - const atTypesSupport2 = await api.getTypeScriptSupport({ - name: '@reach/router', - types: { ts: false }, - version: '1.0', - }); + const atTypesSupport2 = api.getTypeScriptSupport( + { + name: '@reach/router', + types: { ts: false }, + version: '1.0', + }, + [] + ); expect(atTypesSupport2).toEqual({ types: { ts: 'definitely-typed', @@ -87,57 +96,44 @@ describe('getTypeScriptSupport()', () => { }); }); - describe('unpkg', () => { - it('Checks for a d.ts resolved version of main', async () => { - // @ts-expect-error - npm.validatePackageExists.mockResolvedValue(false); - // @ts-expect-error - fileExistsInUnpkg.mockResolvedValue(true); + describe('FilesList', () => { + it('should match a correct filesList', () => { + const atTypesSupport = api.getTypeScriptSupport( + { + name: 'doesnotexist', + types: { ts: false }, + version: '1.0', + }, + [ + { name: 'index.js', hash: '', time: '', size: 0 }, + { name: 'index.d.ts', hash: '', time: '', size: 0 }, + ] + ); + expect(atTypesSupport).toEqual({ + types: { + ts: 'included', + }, + }); + }); - const typesSupport = await api.getTypeScriptSupport({ - name: 'my-lib', - types: { ts: { possible: true, dtsMain: 'main.d.ts' } }, - version: '1.0.0', + it('should not match an incorrect filesList', () => { + const atTypesSupport = api.getTypeScriptSupport( + { + name: 'doesnotexist', + types: { ts: false }, + version: '1.0', + }, + [ + { name: 'index.js', hash: '', time: '', size: 0 }, + { name: 'index.ts', hash: '', time: '', size: 0 }, + { name: 'index.md', hash: '', time: '', size: 0 }, + ] + ); + expect(atTypesSupport).toEqual({ + types: { + ts: false, + }, }); - expect(typesSupport).toEqual({ types: { ts: 'included' } }); }); }); - - // TO DO : reup this - // adescribe('FilesList', () => { - // ait('should match a correct filesList', async () => { - // const atTypesSupport = await api.getTypeScriptSupport( - // { - // name: 'doesnotexist', - // types: { ts: false }, - // version: '1.0', - - // }, - // [{ name: 'index.js' }, { name: 'index.d.ts' }] - // ); - // expect(atTypesSupport).toEqual({ - // types: { - // _where: 'filesList', - // ts: 'included', - // }, - // }); - // }); - - // ait('should not match an incorrect filesList', async () => { - // const atTypesSupport = await api.getTypeScriptSupport( - // { - // name: 'doesnotexist', - // types: { ts: false }, - // version: '1.0', - - // }, - // [{ name: 'index.js' }, { name: 'index.ts' }, { name: 'index.md' }] - // ); - // expect(atTypesSupport).toEqual({ - // types: { - // ts: false, - // }, - // }); - // }); - // }); }); diff --git a/src/typescript/index.ts b/src/typescript/index.ts index c660851ea..519ba7a12 100644 --- a/src/typescript/index.ts +++ b/src/typescript/index.ts @@ -1,6 +1,6 @@ import type { RawPkg } from '../@types/pkg'; import { config } from '../config'; -import { fileExistsInUnpkg } from '../unpkg'; +import type { File } from '../jsDelivr'; import { datadog } from '../utils/datadog'; import { log } from '../utils/log'; import { request } from '../utils/request'; @@ -54,9 +54,10 @@ export function unmangle(name: string): string { * - { types: { ts: "@types/module" }} - for definitely typed support * - { types: { ts: "included" }} - for types shipped with the module. */ -export async function getTypeScriptSupport( - pkg: Pick -): Promise> { +export function getTypeScriptSupport( + pkg: Pick, + filelist: File[] +): Pick { // Already calculated in `formatPkg` if (pkg.types.ts === 'included') { return { types: pkg.types }; @@ -73,22 +74,17 @@ export async function getTypeScriptSupport( }; } - if (pkg.types.ts === false) { - return { types: { ts: false } }; - } - - // Do we have a main .d.ts file? - // TO DO: replace this with a list of files check - if (pkg.types.ts !== 'definitely-typed' && pkg.types.ts.possible === true) { - const resolved = await fileExistsInUnpkg( - pkg.name, - pkg.version, - pkg.types.ts.dtsMain - ); - if (resolved) { - return { types: { ts: 'included' } }; + for (const file of filelist) { + if (!file.name.endsWith('.d.ts')) { + // eslint-disable-next-line no-continue + continue; } + + datadog.increment('jsdelivr.getTSSupport.hit'); + + return { types: { ts: 'included' } }; } + datadog.increment('jsdelivr.getTSSupport.miss'); return { types: { ts: false } }; } @@ -97,11 +93,16 @@ export async function getTypeScriptSupport( * Check if packages have Typescript definitions. */ export async function getTSSupport( - pkgs: Array> + pkgs: Array>, + filelists: File[][] ): Promise>> { const start = Date.now(); - const all = await Promise.all(pkgs.map(getTypeScriptSupport)); + const all = await Promise.all( + pkgs.map((pkg, index) => { + return getTypeScriptSupport(pkg, filelists[index] || []); + }) + ); datadog.timing('getTSSupport', Date.now() - start); return all; diff --git a/src/unpkg/index.test.ts b/src/unpkg/index.test.ts deleted file mode 100644 index 36d113aba..000000000 --- a/src/unpkg/index.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { fileExistsInUnpkg } from './index'; - -describe('unpkg', () => { - it('should do a successful HEAD request', async () => { - const exists = await fileExistsInUnpkg('jest', '24.8.0', 'bin/jest.js'); - expect(exists).toBe(true); - }); - - it('should do a failed HEAD request', async () => { - const exists = await fileExistsInUnpkg( - 'jest', - '24.8.0', - 'bin/notexists.js' - ); - expect(exists).toBe(false); - }); -}); diff --git a/src/unpkg/index.ts b/src/unpkg/index.ts deleted file mode 100644 index 94903f8be..000000000 --- a/src/unpkg/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { config } from '../config'; -import { request } from '../utils/request'; - -// make a head request to a route like: -// https://unpkg.com/lodash@4.17.11/_LazyWrapper.js -// to validate the existence of a particular file -export async function fileExistsInUnpkg( - pkg: string, - version: string, - path: string -): Promise { - const uri = `${config.unpkgRoot}/${pkg}@${version}/${path}`; - try { - const response = await request(uri, { - method: 'HEAD', - }); - return response.statusCode === 200; - } catch (e) { - return false; - } -}