diff --git a/src/__tests__/typescript.test.ts b/src/__tests__/typescript.test.ts deleted file mode 100644 index 42e05f811..000000000 --- a/src/__tests__/typescript.test.ts +++ /dev/null @@ -1,80 +0,0 @@ -import * as npm from '../npm'; -import { getTypeScriptSupport } from '../typescriptSupport'; -import { fileExistsInUnpkg } from '../unpkg'; - -jest.mock('../npm'); -jest.mock('../unpkg'); - -describe('getTypeScriptSupport()', () => { - it('If types are already calculated - return early', async () => { - const typesSupport = await getTypeScriptSupport({ - name: 'Has Types', - types: { ts: 'included' }, - version: '1.0.0', - }); - - expect(typesSupport).toEqual({ types: { ts: 'included' } }); - }); - - describe('without types/typings', () => { - it('Checks for @types/[name]', async () => { - // @ts-expect-error - npm.validatePackageExists.mockResolvedValue(true); - const atTypesSupport = await getTypeScriptSupport({ - name: 'my-lib', - types: { ts: false }, - version: '1.0.0', - }); - expect(atTypesSupport).toEqual({ - types: { - ts: 'definitely-typed', - definitelyTyped: '@types/my-lib', - }, - }); - }); - - it('Checks for @types/[scope__name]', async () => { - // @ts-expect-error - npm.validatePackageExists.mockResolvedValue(true); - const atTypesSupport = await getTypeScriptSupport({ - name: '@my-scope/my-lib', - types: { ts: false }, - version: '1.0.0', - }); - expect(atTypesSupport).toEqual({ - types: { - ts: 'definitely-typed', - definitelyTyped: '@types/my-scope__my-lib', - }, - }); - }); - - it('Checks for a d.ts resolved version of main', async () => { - // @ts-expect-error - npm.validatePackageExists.mockResolvedValue(false); - // @ts-expect-error - fileExistsInUnpkg.mockResolvedValue(true); - - const typesSupport = await getTypeScriptSupport({ - name: 'my-lib', - types: { ts: { possible: true, dtsMain: 'main.d.ts' } }, - version: '1.0.0', - }); - expect(typesSupport).toEqual({ types: { ts: 'included' } }); - }); - - it('Handles not having any possible TS types', async () => { - // @ts-expect-error - npm.validatePackageExists.mockResolvedValue(false); - // @ts-expect-error - fileExistsInUnpkg.mockResolvedValue(false); - - const typesSupport = await getTypeScriptSupport({ - name: 'my-lib', - types: { ts: false }, - version: '1.0.0', - }); - expect(typesSupport).toEqual({ types: { ts: false } }); - }); - }); -}); diff --git a/src/config.ts b/src/config.ts index 23581f480..1316b99fe 100644 --- a/src/config.ts +++ b/src/config.ts @@ -152,6 +152,8 @@ export const config = { jsDelivrHitsEndpoint: 'https://data.jsdelivr.com/v1/stats/packages/npm/month/all', jsDelivrPackageEndpoint: 'https://data.jsdelivr.com/v1/package/npm', + typescriptTypesIndex: + 'https://typespublisher.blob.core.windows.net/typespublisher/data/search-index-min.json', unpkgRoot: 'https://unpkg.com', maxObjSize: 450000, popularDownloadsRatio: 0.005, diff --git a/src/index.ts b/src/index.ts index 2525bc262..393a7c219 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import * as algolia from './algolia/index'; import * as bootstrap from './bootstrap'; import { config } from './config'; import * as jsDelivr from './jsDelivr/index'; +import * as typescript from './typescript/index'; import { datadog } from './utils/datadog'; import { log } from './utils/log'; import * as sentry from './utils/sentry'; @@ -43,6 +44,7 @@ async function main(): Promise { // Preload some useful data await jsDelivr.loadHits(); + await typescript.loadTypesIndex(); // then we run the bootstrap // after a bootstrap is done, it's moved to main (with settings) diff --git a/src/saveDocs.ts b/src/saveDocs.ts index 7979d8e2a..4741b7fca 100644 --- a/src/saveDocs.ts +++ b/src/saveDocs.ts @@ -7,7 +7,7 @@ import formatPkg from './formatPkg'; import * as jsDelivr from './jsDelivr'; import * as npm from './npm'; import type { GetPackage } from './npm/types'; -import { getTSSupport } from './typescriptSupport'; +import { getTSSupport } from './typescript/index'; import { datadog } from './utils/datadog'; import { log } from './utils/log'; diff --git a/src/typescript/index.test.ts b/src/typescript/index.test.ts new file mode 100644 index 000000000..ce55a9c39 --- /dev/null +++ b/src/typescript/index.test.ts @@ -0,0 +1,143 @@ +import * as npm from '../npm'; +import { fileExistsInUnpkg } from '../unpkg'; + +import * as api from './index'; + +jest.mock('../npm'); +jest.mock('../unpkg'); + +describe('loadTypesIndex()', () => { + it('should download and cache all @types', async () => { + expect(api.typesCache).not.toHaveProperty('algoliasearch'); + expect(api.isDefinitelyTyped({ name: 'algoliasearch' })).toBe(undefined); + + await api.loadTypesIndex(); + expect(api.typesCache).toHaveProperty('algoliasearch'); + expect(api.typesCache).not.toHaveProperty('algoliasearch/lite'); + + expect(api.typesCache.algoliasearch).toBe('algoliasearch'); + expect(api.typesCache['algoliasearch/lite']).toBe(undefined); + expect(api.typesCache.doesnotexist).toBe(undefined); + + expect(api.isDefinitelyTyped({ name: 'algoliasearch' })).toBe( + 'algoliasearch' + ); + }); +}); + +describe('getTypeScriptSupport()', () => { + it('If types are already calculated - return early', async () => { + const typesSupport = await api.getTypeScriptSupport({ + name: 'Has Types', + types: { ts: 'included' }, + version: '1.0', + }); + + expect(typesSupport).toEqual({ types: { ts: 'included' } }); + }); + + it('Handles not having any possible TS types', async () => { + const typesSupport = await api.getTypeScriptSupport({ + name: 'my-lib', + types: { ts: false }, + version: '1.0', + }); + expect(typesSupport).toEqual({ types: { ts: false } }); + }); + + describe('Definitely Typed', () => { + it('Checks for @types/[name]', async () => { + const atTypesSupport = await api.getTypeScriptSupport({ + name: 'lodash.valuesin', + types: { ts: false }, + version: '1.0', + }); + expect(atTypesSupport).toEqual({ + types: { + ts: 'definitely-typed', + definitelyTyped: '@types/lodash.valuesin', + }, + }); + }); + + it('Checks for @types/[scope__name]', async () => { + const atTypesSupport = await api.getTypeScriptSupport({ + name: '@mapbox/geojson-area', + types: { ts: false }, + version: '1.0', + }); + expect(atTypesSupport).toEqual({ + types: { + ts: 'definitely-typed', + definitelyTyped: '@types/mapbox__geojson-area', + }, + }); + + const atTypesSupport2 = await api.getTypeScriptSupport({ + name: '@reach/router', + types: { ts: false }, + version: '1.0', + }); + expect(atTypesSupport2).toEqual({ + types: { + ts: 'definitely-typed', + definitelyTyped: '@types/reach__router', + }, + }); + }); + }); + + describe('unpkg', () => { + it('Checks for a d.ts resolved version of main', async () => { + // @ts-expect-error + npm.validatePackageExists.mockResolvedValue(false); + // @ts-expect-error + fileExistsInUnpkg.mockResolvedValue(true); + + const typesSupport = await api.getTypeScriptSupport({ + name: 'my-lib', + types: { ts: { possible: true, dtsMain: 'main.d.ts' } }, + version: '1.0.0', + }); + expect(typesSupport).toEqual({ types: { ts: 'included' } }); + }); + }); + + // TO DO : reup this + // adescribe('FilesList', () => { + // ait('should match a correct filesList', async () => { + // const atTypesSupport = await api.getTypeScriptSupport( + // { + // name: 'doesnotexist', + // types: { ts: false }, + // version: '1.0', + + // }, + // [{ name: 'index.js' }, { name: 'index.d.ts' }] + // ); + // expect(atTypesSupport).toEqual({ + // types: { + // _where: 'filesList', + // ts: 'included', + // }, + // }); + // }); + + // ait('should not match an incorrect filesList', async () => { + // const atTypesSupport = await api.getTypeScriptSupport( + // { + // name: 'doesnotexist', + // types: { ts: false }, + // version: '1.0', + + // }, + // [{ name: 'index.js' }, { name: 'index.ts' }, { name: 'index.md' }] + // ); + // expect(atTypesSupport).toEqual({ + // types: { + // ts: false, + // }, + // }); + // }); + // }); +}); diff --git a/src/typescript/index.ts b/src/typescript/index.ts new file mode 100644 index 000000000..c660851ea --- /dev/null +++ b/src/typescript/index.ts @@ -0,0 +1,108 @@ +import type { RawPkg } from '../@types/pkg'; +import { config } from '../config'; +import { fileExistsInUnpkg } from '../unpkg'; +import { datadog } from '../utils/datadog'; +import { log } from '../utils/log'; +import { request } from '../utils/request'; + +interface TypeList { + p: string; // url + l: string; // display name + t: string; // package name + // don't known + d: number; + g: string[]; + m: string[]; +} + +export const typesCache: Record = {}; + +/** + * Microsoft build a index.json with all @types/* on each publication. + * - https://github.com/microsoft/types-publisher/blob/master/src/create-search-index.ts. + */ +export async function loadTypesIndex(): Promise { + const start = Date.now(); + const { body } = await request(config.typescriptTypesIndex, { + decompress: true, + responseType: 'json', + }); + + log.info(`📦 Typescript preload, found ${body.length} @types`); + + // m = modules associated + // t = @types/ + body.forEach((type) => { + typesCache[unmangle(type.t)] = type.t; + }); + + datadog.timing('typescript.loadTypesIndex', Date.now() - start); +} + +export function isDefinitelyTyped({ name }): string | undefined { + return typesCache[unmangle(name)]; +} + +export function unmangle(name: string): string { + // https://github.com/algolia/npm-search/pull/407/files#r316562095 + return name.replace('__', '/').replace('@', ''); +} + +/** + * Basically either + * - { types: { ts: false }} for no existing TypeScript support + * - { types: { ts: "@types/module" }} - for definitely typed support + * - { types: { ts: "included" }} - for types shipped with the module. + */ +export async function getTypeScriptSupport( + pkg: Pick +): Promise> { + // Already calculated in `formatPkg` + if (pkg.types.ts === 'included') { + return { types: pkg.types }; + } + + // The 2nd most likely is definitely typed + const defTyped = isDefinitelyTyped({ name: pkg.name }); + if (defTyped) { + return { + types: { + ts: 'definitely-typed', + definitelyTyped: `@types/${defTyped}`, + }, + }; + } + + if (pkg.types.ts === false) { + return { types: { ts: false } }; + } + + // Do we have a main .d.ts file? + // TO DO: replace this with a list of files check + if (pkg.types.ts !== 'definitely-typed' && pkg.types.ts.possible === true) { + const resolved = await fileExistsInUnpkg( + pkg.name, + pkg.version, + pkg.types.ts.dtsMain + ); + if (resolved) { + return { types: { ts: 'included' } }; + } + } + + return { types: { ts: false } }; +} + +/** + * Check if packages have Typescript definitions. + */ +export async function getTSSupport( + pkgs: Array> +): Promise>> { + const start = Date.now(); + + const all = await Promise.all(pkgs.map(getTypeScriptSupport)); + + datadog.timing('getTSSupport', Date.now() - start); + return all; +} diff --git a/src/typescriptSupport.ts b/src/typescriptSupport.ts deleted file mode 100644 index 3bec561b9..000000000 --- a/src/typescriptSupport.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { RawPkg } from './@types/pkg'; -import * as npm from './npm'; -import { fileExistsInUnpkg } from './unpkg'; -import { datadog } from './utils/datadog'; - -/** - * Basically either - * - { types: { ts: false }} for no existing TypeScript support - * - { types: { ts: "@types/module" }} - for definitely typed support - * - { types: { ts: "included" }} - for types shipped with the module. - * - */ -export async function getTypeScriptSupport( - pkg: Pick -): Promise> { - // Already calculated in `formatPkg` - if (typeof pkg.types.ts === 'string') { - return { types: pkg.types }; - } - - // The 2nd most likely is definitely typed - const defTypeName = `@types/${pkg.name.replace('@', '').replace('/', '__')}`; - const defTyped = await npm.validatePackageExists(defTypeName); - if (defTyped) { - return { - types: { - ts: 'definitely-typed', - definitelyTyped: defTypeName, - }, - }; - } - - if (pkg.types.ts === false) { - return { types: { ts: false } }; - } - - // Do we have a main .d.ts file? - if (pkg.types.ts.possible === true) { - const resolved = await fileExistsInUnpkg( - pkg.name, - pkg.version, - pkg.types.ts.dtsMain - ); - if (resolved) { - return { types: { ts: 'included' } }; - } - } - - return { types: { ts: false } }; -} - -export async function getTSSupport( - pkgs: Array> -): Promise>> { - const start = Date.now(); - - const all = await Promise.all(pkgs.map(getTypeScriptSupport)); - - datadog.timing('getTSSupport', Date.now() - start); - return all; -}