diff --git a/developer/docs/help/reference/kmc/cli/reference.md b/developer/docs/help/reference/kmc/cli/reference.md index 1de0b64c76d..763d5ad71f8 100644 --- a/developer/docs/help/reference/kmc/cli/reference.md +++ b/developer/docs/help/reference/kmc/cli/reference.md @@ -493,9 +493,8 @@ following sources: `author.bcp47.uniq` id pattern, or a keyboard id pattern (where period `.` is not permitted) * A GitHub repository or subfolder within a repository that matches the Keyman - keyboard/model repository layout. The branch name is optional, and will use - the default branch from the repository if omitted. For example, - `github:keyman-keyboards/khmer_angkor:main:/khmer_angkor.kpj` + keyboard/model repository layout. For example, + `https://github.com/keyman-keyboards/khmer_angkor/tree/main/khmer_angkor.kpj` `-o, --out-path ` diff --git a/developer/src/common/web/utils/src/cloud-urls.ts b/developer/src/common/web/utils/src/cloud-urls.ts new file mode 100644 index 00000000000..33f9c669b9b --- /dev/null +++ b/developer/src/common/web/utils/src/cloud-urls.ts @@ -0,0 +1,18 @@ +/** + * Matches a Keyman keyboard resource, based on the permanent home page for the + * keyboard on keyman.com, `https://keyman.com/keyboards/` + */ +export const KEYMANCOM_CLOUD_URI = /^(?:http(?:s)?:\/\/)?keyman\.com\/keyboards\/(?[a-z0-9_.-]+)/i; + +/** + * Matches a `cloud:` URI for a Keyman resource (e.g. keyboard or lexical + * model) + */ +export const CLOUD_URI = /^cloud:(?.+)$/i; + + +export interface CloudUriRegexMatchArray extends RegExpMatchArray { + groups?: { + id?: string; + } +} \ No newline at end of file diff --git a/developer/src/common/web/utils/src/github-urls.ts b/developer/src/common/web/utils/src/github-urls.ts new file mode 100644 index 00000000000..1a65106bf1f --- /dev/null +++ b/developer/src/common/web/utils/src/github-urls.ts @@ -0,0 +1,35 @@ +/** + * Matches only a GitHub permanent raw URI with a commit hash, without any other + * components; note hash is called branch to match other URI formats + */ +export const GITHUB_STABLE_SOURCE = /^https:\/\/github\.com\/(?[a-zA-Z0-9-]+)\/(?[\w\.-]+)\/raw\/(?[a-f0-9]{40})\/(?.+)$/; + +/** + * Matches any GitHub git resource raw 'user content' URI which can be + * translated to a permanent URI with a commit hash + */ +export const GITHUB_RAW_URI = /^https:\/\/raw\.githubusercontent\.com\/(?[a-zA-Z0-9-]+)\/(?[\w\.-]+)\/(?:refs\/(?:heads|tags)\/)?(?[^/]+)\/(?.+)$/; + +/** + * Matches any GitHub git resource raw URI which can be translated to a + * permanent URI with a commit hash + */ +export const GITHUB_URI = /^https:\/\/github\.com\/(?[a-zA-Z0-9-]+)\/(?[\w\.-]+)\/(?:raw|blob|tree)\/(?:refs\/(?:heads|tags)\/)?(?[^/]+)\/(?.+)$/; + +/** + * Matches any GitHub git resource raw URI which can be translated to a + * permanent URI with a commit hash, with the http[s] protocol optional, for + * matching user-supplied URLs. groups are: `owner`, `repo`, `branch`, and + * `path`. + */ +export const GITHUB_URI_OPTIONAL_PROTOCOL = /^(?:http(?:s)?:\/\/)?github\.com\/(?[a-zA-Z0-9-]+)\/(?[\w\.-]+)(?:\/(?:(?:raw|blob|tree)\/(?:refs\/(?:heads|tags)\/)?(?[^/]+)\/(?.*))?)?$/; + + +export interface GitHubRegexMatchArray extends RegExpMatchArray { + groups?: { + owner?: string; + repo?: string; + branch?: string; + path?: string; + } +} \ No newline at end of file diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 5b4c9c0e91a..290efd5fc70 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -66,3 +66,6 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js'; + +export * as GitHubUrls from './github-urls.js'; +export * as CloudUrls from './cloud-urls.js'; \ No newline at end of file diff --git a/developer/src/kmc-copy/src/KeymanProjectCopier.ts b/developer/src/kmc-copy/src/KeymanProjectCopier.ts index e463ec1b4c0..40f68fc1fc4 100644 --- a/developer/src/kmc-copy/src/KeymanProjectCopier.ts +++ b/developer/src/kmc-copy/src/KeymanProjectCopier.ts @@ -4,7 +4,7 @@ * Copy a keyboard or lexical model project */ -import { CompilerCallbacks, CompilerLogLevel, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KeymanDeveloperProject, KeymanDeveloperProjectOptions, KPJFileReader, KPJFileWriter, KpsFileReader, KpsFileWriter } from "@keymanapp/developer-utils"; +import { CloudUrls, GitHubUrls, CompilerCallbacks, CompilerLogLevel, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KeymanDeveloperProject, KeymanDeveloperProjectOptions, KPJFileReader, KPJFileWriter, KpsFileReader, KpsFileWriter } from "@keymanapp/developer-utils"; import { KeymanFileTypes } from "@keymanapp/common-types"; import { CopierMessages } from "./copier-messages.js"; @@ -86,6 +86,9 @@ export class KeymanProjectCopier implements KeymanCompiler { relocateExternalFiles: boolean = false; // TODO-COPY: support public async init(callbacks: CompilerCallbacks, options: CopierOptions): Promise { + if(!callbacks || !options) { + return false; + } this.callbacks = callbacks; this.options = options; this.cloudSource = new KeymanCloudSource(this.callbacks); @@ -97,7 +100,7 @@ export class KeymanProjectCopier implements KeymanCompiler { * artifacts on success. The files are passed in by name, and the compiler * will use callbacks as passed to the {@link KeymanProjectCopier.init} * function to read any input files by disk. - * @param source Source file or folder to copy. Can be a local file or folder, github:repo[:path], or cloud:id + * @param source Source file or folder to copy. Can be a local file or folder, https://github.com/.../repo[/path], or cloud:id * @returns Binary artifacts on success, null on failure. */ public async run(source: string): Promise { @@ -151,10 +154,10 @@ export class KeymanProjectCopier implements KeymanCompiler { * @returns path to .kpj (either local or remote) */ private async getSourceProject(source: string): Promise { - if(source.startsWith('github:')) { - // `github:owner/repo:path/to/kpj`, referencing a .kpj file + if(source.match(GitHubUrls.GITHUB_URI_OPTIONAL_PROTOCOL) || source.match(GitHubUrls.GITHUB_RAW_URI)) { + // `[https://]github.com/owner/repo/[tree|blob|raw]/[refs/...]/branch/path/to/kpj`, referencing a .kpj file return await this.getGitHubSourceProject(source); - } else if(source.startsWith('cloud:')) { + } else if(source.match(CloudUrls.CLOUD_URI) || source.match(CloudUrls.KEYMANCOM_CLOUD_URI)) { // `cloud:id`, referencing a Keyman Cloud keyboard return await this.getCloudSourceProject(source); } else if(this.callbacks.fs.existsSync(source) && source.endsWith(KeymanFileTypes.Source.Project) && !this.callbacks.isDirectory(source)) { @@ -194,64 +197,69 @@ export class KeymanProjectCopier implements KeymanCompiler { /** * Resolve path to GitHub source, which must be in the following format: - * `github:owner/repo[:branch]:path/to/kpj` + * `[https://]github.com/owner/repo/branch/path/to/kpj` * The path must be fully qualified, referencing the .kpj file; it * cannot just be the folder where the .kpj is found * @param source * @returns a promise: GitHub reference to the source for the keyboard, or null on failure */ private async getGitHubSourceProject(source: string): Promise { - const parts = source.split(':'); - if(parts.length < 3 || parts.length > 4 || !parts[1].match(/^[a-z0-9-]+\/[a-z0-9._-]+$/i)) { - // https://stackoverflow.com/questions/59081778/rules-for-special-characters-in-github-repository-name - this.callbacks.reportMessage(CopierMessages.Error_InvalidGitHubSource({source})); - return null; + const parts: GitHubUrls.GitHubRegexMatchArray = + GitHubUrls.GITHUB_URI_OPTIONAL_PROTOCOL.exec(source) ?? + GitHubUrls.GITHUB_RAW_URI.exec(source); + if(!parts) { + throw new Error('Expected GITHUB_URI_OPTIONAL_PROTOCOL or GITHUB_RAW_URI to match'); } - const origin = parts[1].split('/'); - - const ref: GitHubRef = new GitHubRef({ - owner: origin[0], - repo: origin[1], - branch: null, - path: null - }); + const ref: GitHubRef = new GitHubRef(parts); - if(parts.length == 4) { - ref.branch = parts[2]; - ref.path = parts[3]; - } else { + if(!ref.branch) { ref.branch = await this.cloudSource.getDefaultBranchFromGitHub(ref); if(!ref.branch) { this.callbacks.reportMessage(CopierMessages.Error_CouldNotFindDefaultBranchOnGitHub({ref: ref.toString()})); return null; } - ref.path = parts[2]; - + } + if(!ref.path) { + ref.path = '/'; } if(!ref.path.startsWith('/')) { ref.path = '/' + ref.path; } + if(ref.path != '/') { + if(!ref.path.endsWith('.kpj')) { + // Assumption, project filename matches folder name + if(ref.path.endsWith('/')) { + ref.path = ref.path.substring(0, ref.path.length-1); + } + ref.path = ref.path + '/' + this.callbacks.path.basename(ref.path) + '.kpj'; + } + } + return ref; } /** * Resolve path to Keyman Cloud source (which is on GitHub), which must be in * the following format: - * `cloud:keyboard_id|model_id` + * `cloud:keyboard_id`, or + * `cloud:model_id`, or + * `https://keyman.com/keyboards/keyboard_id` * The `keyboard_id` parameter should be a valid id (a-z0-9_), as found at - * https://keyman.com/keyboards; alternativel if it is a model_id, it should + * https://keyman.com/keyboards; alternatively if it is a model_id, it should * have the format author.bcp47.uniq * @param source * @returns a promise: GitHub reference to the source for the keyboard, or null on failure */ private async getCloudSourceProject(source: string): Promise { - const parts = source.split(':'); - const id = parts[1]; + const parts = CloudUrls.CLOUD_URI.exec(source) ?? CloudUrls.KEYMANCOM_CLOUD_URI.exec(source); + if(!parts) { + throw new Error('Expected CLOUD_URI or KEYMANCOM_CLOUD_URI to match'); + } + const id: string = parts.groups.id; const isModel = /^[^.]+\.[^.]+\.[^.]+$/.test(id); - const remote = await this.cloudSource.getSourceFromKeymanCloud(id, isModel); if(!remote) { return null; @@ -687,4 +695,11 @@ export class KeymanProjectCopier implements KeymanCompiler { return true; } /* c8 ignore stop */ + + /** @internal */ + public unitTestEndPoints = { + getGithubSourceProject: this.getGitHubSourceProject.bind(this), + getCloudSourceProject: this.getCloudSourceProject.bind(this) + }; + } diff --git a/developer/src/kmc-copy/src/cloud.ts b/developer/src/kmc-copy/src/cloud.ts index aeb5171895f..7b3ef1ff119 100644 --- a/developer/src/kmc-copy/src/cloud.ts +++ b/developer/src/kmc-copy/src/cloud.ts @@ -3,7 +3,7 @@ * * GitHub and Keyman Cloud interface wrappers */ -import { CompilerCallbacks } from "@keymanapp/developer-utils"; +import { CompilerCallbacks, GitHubUrls } from "@keymanapp/developer-utils"; import { CopierMessages } from "./copier-messages.js"; import { KeymanFileTypes } from "@keymanapp/common-types"; @@ -12,17 +12,24 @@ export class GitHubRef { public repo: string; public branch: string; public path: string; - constructor(owner: string | GitHubRef, repo?: string, branch?: string, path?: string) { + constructor(owner: string | GitHubRef | GitHubUrls.GitHubRegexMatchArray, repo?: string, branch?: string, path?: string) { if(typeof owner == 'string') { this.owner = owner; this.repo = repo; this.branch = branch; this.path = path; - } else { + } else if("groups" in owner) { + this.owner = owner.groups.owner; + this.repo = owner.groups.repo; + this.branch = owner.groups.branch; + this.path = owner.groups.path; + } else if("owner" in owner) { this.owner = owner.owner; this.repo = owner.repo; this.branch = owner.branch; this.path = owner.path; + } else { + throw new Error(`Unrecognized GitHubRef '${owner}'`) } } toString() { diff --git a/developer/src/kmc-copy/src/copier-messages.ts b/developer/src/kmc-copy/src/copier-messages.ts index 02294a90815..78a948ec79b 100644 --- a/developer/src/kmc-copy/src/copier-messages.ts +++ b/developer/src/kmc-copy/src/copier-messages.ts @@ -117,18 +117,7 @@ export class CopierMessages { `Dry run requested. No changes have been saved` ); - static ERROR_InvalidGitHubSource = SevError | 0x0011; - static Error_InvalidGitHubSource = (o:{source: string}) => m( - this.ERROR_InvalidGitHubSource, - `Source project specification '${def(o.source)}' is not a valid GitHub reference`, - `The source project specification for GitHub sources must match the pattern: - github:\\[:\\]:\\ - The path must include the .kpj filename and may optionally begin with a forward slash. - The following are valid examples: - github:keymanapp/keyboards:master:release/k/khmer_angkor/khmer_angkor.kpj - github:keymanapp/keyboards:release/k/khmer_angkor/khmer_angkor.kpj - github:keymanapp/keyboards:/release/k/khmer_angkor/khmer_angkor.kpj` - ); + // 0x0011 unused static ERROR_CannotDownloadFolderFromGitHub = SevError | 0x0012; static Error_CannotDownloadFolderFromGitHub = (o:{ref: string, message?: string, cause?: string}) => m( diff --git a/developer/src/kmc-copy/test/copier.tests.ts b/developer/src/kmc-copy/test/copier.tests.ts index 3cf9e4c0ee6..6d583862aea 100644 --- a/developer/src/kmc-copy/test/copier.tests.ts +++ b/developer/src/kmc-copy/test/copier.tests.ts @@ -12,6 +12,7 @@ import { assert } from 'chai'; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KeymanProjectCopier } from '../src/KeymanProjectCopier.js'; import { makePathToFixture } from './helpers/index.js'; +import { GitHubRef } from './cloud.js'; const { TEST_SAVE_ARTIFACTS, TEST_SAVE_FIXTURES } = env; let outputRoot: string = '/an/imaginary/root/'; @@ -374,7 +375,7 @@ describe('KeymanProjectCopier', function() { // armenian_mnemonic selected because (a) small, and (b) has v2.0 project, so // that exercises the folder retrieval as well - const result = await copier.run('github:keymanapp/keyboards:release/a/armenian_mnemonic/armenian_mnemonic.kpj'); + const result = await copier.run('github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj'); // We should have no messages and a successful result assert.isOk(result); @@ -416,6 +417,81 @@ describe('KeymanProjectCopier', function() { }); }); + // Keyman Cloud patterns + + const cloud_khmer_angkor: GitHubRef = { branch: 'master', owner: 'keymanapp', repo: 'keyboards', path: '/release/k/khmer_angkor/khmer_angkor.kpj' }; + const cloud_nrc_en_mtnt: GitHubRef = { branch: 'master', owner: 'keymanapp', repo: 'lexical-models', path: '/release/nrc/nrc.en.mtnt/nrc.en.mtnt.kpj' }; + const cloud_urls: [string,GitHubRef][] = [ + ['cloud:khmer_angkor', cloud_khmer_angkor], + ['https://keyman.com/keyboards/khmer_angkor', cloud_khmer_angkor], + ['https://keyman.com/keyboards/khmer_angkor/', cloud_khmer_angkor], + ['keyman.com/keyboards/khmer_angkor/', cloud_khmer_angkor], + ['http://keyman.com/keyboards/khmer_angkor#abc', cloud_khmer_angkor], + ['cloud:nrc.en.mtnt', cloud_nrc_en_mtnt], + ]; + + cloud_urls.forEach(url => { + it(`should parse URL '${url[0]}' and figure out the .kpj`, async function() { + // url --> + const copier = new KeymanProjectCopier(); + assert.isTrue(await copier.init(callbacks, { + dryRun: false, + outPath: '' + })); + + const ref = await copier.unitTestEndPoints.getCloudSourceProject(url[0]); + assert.isNotNull(ref); + assert.deepEqual(ref, url[1]); + }); + }) + + + // GitHub patterns that should match as inputs for kmc-copy source + + const armenian_mnemonic_urls = [ { + branch: 'master', urls: [ + 'github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic', + 'http://github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic', + 'https://github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic', + 'https://github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic/', + 'https://github.com/keymanapp/keyboards/tree/refs/heads/master/release/a/armenian_mnemonic', + 'https://github.com/keymanapp/keyboards/tree/refs/heads/master/release/a/armenian_mnemonic/', + 'https://github.com/keymanapp/keyboards/raw/refs/heads/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj', + 'https://github.com/keymanapp/keyboards/raw/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj', + 'https://github.com/keymanapp/keyboards/blob/refs/heads/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj', + 'https://github.com/keymanapp/keyboards/blob/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj', + + // And similar patterns for raw.githubusercontent.com + + 'https://raw.githubusercontent.com/keymanapp/keyboards/refs/heads/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj', + 'https://raw.githubusercontent.com/keymanapp/keyboards/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj', + ]}, { + branch: '78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2', urls: [ + 'https://github.com/keymanapp/keyboards/blob/78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2/release/a/armenian_mnemonic/armenian_mnemonic.kpj', + 'https://github.com/keymanapp/keyboards/tree/78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2/release/a/armenian_mnemonic', + 'https://github.com/keymanapp/keyboards/tree/78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2/release/a/armenian_mnemonic/', + 'https://raw.githubusercontent.com/keymanapp/keyboards/78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2/release/a/armenian_mnemonic/armenian_mnemonic.kpj', + ]}]; + + armenian_mnemonic_urls.forEach(({branch,urls}) => urls.forEach(url => { + it(`should parse URL '${url}' and figure out the .kpj`, async function() { + // url --> + const copier = new KeymanProjectCopier(); + assert.isTrue(await copier.init(callbacks, { + dryRun: false, + outPath: '' + })); + + const ref = await copier.unitTestEndPoints.getGithubSourceProject(url); + assert.deepEqual(ref, { + branch, + owner: 'keymanapp', + repo: 'keyboards', + path: '/release/a/armenian_mnemonic/armenian_mnemonic.kpj' + }); + }); + })); + // TODO-COPY: additional tests it.skip('should copy a disorganized project into current structure', async function() {}); it.skip('should copy a standalone .kmn into a new project', async function() {}); diff --git a/developer/src/kmc-copy/test/fixtures/online/api.keyman.com/#keyboard#khmer_angkor b/developer/src/kmc-copy/test/fixtures/online/api.keyman.com/#keyboard#khmer_angkor new file mode 100644 index 00000000000..17b495a24bb --- /dev/null +++ b/developer/src/kmc-copy/test/fixtures/online/api.keyman.com/#keyboard#khmer_angkor @@ -0,0 +1,65 @@ +{ + "id": "khmer_angkor", + "name": "Khmer Angkor", + "license": "mit", + "authorName": "Makara Sok", + "authorEmail": "makara_sok@sil.org", + "description": "

Khmer Unicode keyboard layout based on the NiDA keyboard layout.\nAutomatically corrects many common keying errors.

", + "languages": { + "km": { + "examples": [ + { + "keys": "x j m E r", + "note": "Name of language", + "text": "\u1781\u17d2\u1798\u17c2\u179a" + } + ], + "font": { + "family": "Khmer Mondulkiri", + "source": [ + "Mondulkiri-R.ttf" + ] + }, + "oskFont": { + "family": "KbdKhmr", + "source": [ + "KbdKhmr.ttf" + ] + }, + "languageName": "Khmer", + "displayName": "Khmer" + } + }, + "lastModifiedDate": "2024-07-03T15:47:38.000Z", + "packageFilename": "khmer_angkor.kmp", + "packageFileSize": 4259005, + "jsFilename": "khmer_angkor.js", + "jsFileSize": 70494, + "packageIncludes": [ + "visualKeyboard", + "welcome", + "fonts", + "documentation" + ], + "version": "1.5", + "encodings": [ + "unicode" + ], + "platformSupport": { + "windows": "full", + "macos": "full", + "linux": "full", + "desktopWeb": "full", + "ios": "full", + "android": "full", + "mobileWeb": "full" + }, + "minKeymanVersion": "10.0", + "sourcePath": "release/k/khmer_angkor", + "helpLink": "https://help.keyman.com/keyboard/khmer_angkor", + "related": { + "khmer10": { + "deprecates": true + } + } +} \ No newline at end of file diff --git a/developer/src/kmc-copy/test/fixtures/online/api.keyman.com/#model#nrc.en.mtnt b/developer/src/kmc-copy/test/fixtures/online/api.keyman.com/#model#nrc.en.mtnt new file mode 100644 index 00000000000..bee31a4fe4a --- /dev/null +++ b/developer/src/kmc-copy/test/fixtures/online/api.keyman.com/#model#nrc.en.mtnt @@ -0,0 +1,23 @@ +{ + "languages": [ + "en", + "en-us", + "en-ca" + ], + "id": "nrc.en.mtnt", + "name": "English language model mined from MTNT", + "license": "mit", + "authorName": "Eddie Antonio Santos", + "authorEmail": "easantos@ualberta.ca", + "description": "

A unigram language model for English derived from the MTNT corpus http://www.cs.cmu.edu/~pmichel1/mtnt/. This corpus itself is gathered from Reddit, so it is unfiltered internet discussion. This is not humanity at its prettiest!

", + "lastModifiedDate": "2024-09-16T01:05:45.000Z", + "packageFilename": "https://keyman.com/go/package/download/model/nrc.en.mtnt?version=0.3.3&update=1", + "packageFileSize": 332955, + "jsFilename": "https://downloads.keyman.com/models/nrc.en.mtnt/0.3.3/nrc.en.mtnt.model.js", + "jsFileSize": 2713050, + "packageIncludes": [], + "version": "0.3.3", + "minKeymanVersion": "12.0", + "helpLink": "https://help.keyman.com/model/nrc.en.mtnt", + "sourcePath": "release/nrc/nrc.en.mtnt" +} \ No newline at end of file diff --git a/developer/src/kmc-package/src/compiler/get-file-data.ts b/developer/src/kmc-package/src/compiler/get-file-data.ts index 198ab03f73a..c17de052f99 100644 --- a/developer/src/kmc-package/src/compiler/get-file-data.ts +++ b/developer/src/kmc-package/src/compiler/get-file-data.ts @@ -3,7 +3,7 @@ * * Created by mcdurdin on 2024-11-11 */ -import { CompilerCallbacks } from '@keymanapp/developer-utils'; +import { CompilerCallbacks, GitHubUrls } from '@keymanapp/developer-utils'; import { PackageCompilerMessages } from './package-compiler-messages.js'; /** @@ -16,24 +16,6 @@ const URL_SOURCE = /^http(?:s)?:\/\/(.+)$/; */ const FLO_SOURCE = /^flo:(?[a-z0-9_-]+)$/; -/** - * Matches only a GitHub permanent raw URI with a commit hash, without any other - * components - */ -const GITHUB_STABLE_SOURCE = /^https:\/\/github\.com\/(?[a-zA-Z0-9-]+)\/(?[\w\.-]+)\/raw\/(?[a-f0-9]{40})\/(?.+)$/; - -/** - * Matches any GitHub git resource raw 'user content' URI which can be - * translated to a permanent URI with a commit hash - */ -const GITHUB_RAW_URI = /^https:\/\/raw\.githubusercontent\.com\/(?[a-zA-Z0-9-]+)\/(?[\w\.-]+)\/(?:refs\/(?:heads|tags)\/)?(?[^/]+)\/(?.+)$/; - -/** - * Matches any GitHub git resource raw URI which can be translated to a - * permanent URI with a commit hash - */ -const GITHUB_URI = /^https:\/\/github\.com\/(?[a-zA-Z0-9-].+)\/(?[\w\.-]+)\/(?:raw|blob|tree)\/(?:refs\/(?:heads|tags)\/)?(?[^/]+)\/(?.+)$/; - export interface KmpCompilerFileDataResult { data: Uint8Array; basename: string; @@ -50,7 +32,7 @@ export function isLocalFile(inputFilename: string) { } export async function getFileDataFromRemote(callbacks: CompilerCallbacks, inputFilename: string, sourceFilename: string): Promise { - const matches = GITHUB_STABLE_SOURCE.exec(inputFilename); + const matches: GitHubUrls.GitHubRegexMatchArray = GitHubUrls.GITHUB_STABLE_SOURCE.exec(inputFilename); if(!matches) { callbacks.reportMessage(PackageCompilerMessages.Error_UriIsNotARecognizedStableGitHubUri({url: inputFilename})); return null; @@ -73,9 +55,9 @@ export async function getFileDataFromRemote(callbacks: CompilerCallbacks, inputF return result; } -async function getFileDataFromGitHub(callbacks: CompilerCallbacks, inputFilename: string, matches: RegExpExecArray): Promise { - // /^github:(?[a-zA-Z0-9-].+)\/(?[\w\.-]+)\/raw\/(?[a-f0-9]{40})\/(?.+)$/ - const githubUrl = `https://github.com/${matches.groups.name}/${matches.groups.repo}/raw/${matches.groups.hash}/${matches.groups.path}`; +async function getFileDataFromGitHub(callbacks: CompilerCallbacks, inputFilename: string, matches: GitHubUrls.GitHubRegexMatchArray): Promise { + // /^github:(?[a-zA-Z0-9-].+)\/(?[\w\.-]+)\/raw\/(?[a-f0-9]{40})\/(?.+)$/ + const githubUrl = `https://github.com/${matches.groups.owner}/${matches.groups.repo}/raw/${matches.groups.branch}/${matches.groups.path}`; try { const res = await callbacks.net.fetchBlob(githubUrl); if(!res) { @@ -94,9 +76,9 @@ async function checkSourceFile(callbacks: CompilerCallbacks, inputFilename: stri let stableUrl: string; if(FLO_SOURCE.test(sourceFilename)) { stableUrl = await getFileStableRefFromFlo(callbacks, sourceFilename); - } else if(GITHUB_URI.test(sourceFilename)) { + } else if(GitHubUrls.GITHUB_URI.test(sourceFilename)) { stableUrl = await getFileStableRefFromGitHub(callbacks, sourceFilename); - } else if(GITHUB_RAW_URI.test(sourceFilename)) { + } else if(GitHubUrls.GITHUB_RAW_URI.test(sourceFilename)) { stableUrl = await getFileStableRefFromGitHub(callbacks, sourceFilename); } else { callbacks.reportMessage(PackageCompilerMessages.Error_InvalidSourceFileReference({source: sourceFilename, name: inputFilename})); @@ -182,7 +164,7 @@ async function getFileStableRefFromFlo(callbacks: CompilerCallbacks, floSource: // we don't use flourl at this time, becase we want a GitHub reference that // can be resolved to a stable URI - const ghmatches = GITHUB_URI.exec(file.url) ?? GITHUB_RAW_URI.exec(file.url); + const ghmatches: GitHubUrls.GitHubRegexMatchArray = GitHubUrls.GITHUB_URI.exec(file.url) ?? GitHubUrls.GITHUB_RAW_URI.exec(file.url); /* c8 ignore next 4 */ if(!ghmatches) { callbacks.reportMessage(PackageCompilerMessages.Error_FontInFloDoesNotHaveARecognizedGitHubUri({filename: floSource, url: file.url})); @@ -193,7 +175,7 @@ async function getFileStableRefFromFlo(callbacks: CompilerCallbacks, floSource: } async function getFileStableRefFromGitHub(callbacks: CompilerCallbacks, source: string): Promise { - const matches = GITHUB_URI.exec(source) ?? GITHUB_RAW_URI.exec(source); + const matches: GitHubUrls.GitHubRegexMatchArray = GitHubUrls.GITHUB_URI.exec(source) ?? GitHubUrls.GITHUB_RAW_URI.exec(source); if(!matches) { callbacks.reportMessage(PackageCompilerMessages.Error_UriIsNotARecognizedGitHubUri({url: source})); return null; @@ -201,10 +183,10 @@ async function getFileStableRefFromGitHub(callbacks: CompilerCallbacks, source: return await resolveGitHubToStableRef(callbacks, matches); } -async function resolveGitHubToStableRef(callbacks: CompilerCallbacks, matches: RegExpExecArray): Promise { +async function resolveGitHubToStableRef(callbacks: CompilerCallbacks, matches: GitHubUrls.GitHubRegexMatchArray): Promise { let commit: any = null; - const url = `https://api.github.com/repos/${matches.groups.name}/${matches.groups.repo}/commits/${matches.groups.branch}?path=${matches.groups.path}`; + const url = `https://api.github.com/repos/${matches.groups.owner}/${matches.groups.repo}/commits/${matches.groups.branch}?path=${matches.groups.path}`; try { commit = await callbacks.net.fetchJSON(url); /* c8 ignore next 8 */ @@ -217,9 +199,11 @@ async function resolveGitHubToStableRef(callbacks: CompilerCallbacks, matches: R throw e; } - return `https://github.com/${matches.groups.name}/${matches.groups.repo}/raw/${commit.sha}/${matches.groups.path}`; + return `https://github.com/${matches.groups.owner}/${matches.groups.repo}/raw/${commit.sha}/${matches.groups.path}`; } +const GITHUB_STABLE_SOURCE = GitHubUrls.GITHUB_STABLE_SOURCE; + /** @internal */ export const unitTestEndpoints = { GITHUB_STABLE_SOURCE, diff --git a/developer/src/kmc-package/test/package-compiler.tests.ts b/developer/src/kmc-package/test/package-compiler.tests.ts index 6be9445549b..96a1c02672c 100644 --- a/developer/src/kmc-package/test/package-compiler.tests.ts +++ b/developer/src/kmc-package/test/package-compiler.tests.ts @@ -277,9 +277,9 @@ describe('KmpCompiler', function () { const source = 'https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf'; const matches = getFileDataEndpoints.GITHUB_STABLE_SOURCE.exec(source); assert.isNotNull(matches); - assert.equal(matches.groups.name, 'silnrsi'); + assert.equal(matches.groups.owner, 'silnrsi'); assert.equal(matches.groups.repo, 'fonts'); - assert.equal(matches.groups.hash, 'b88c7af5d16681bd137156929ff8baec82526560'); + assert.equal(matches.groups.branch, 'b88c7af5d16681bd137156929ff8baec82526560'); assert.equal(matches.groups.path, 'fonts/sil/alkalami/Alkalami-Regular.ttf'); const res = await getFileDataEndpoints.getFileDataFromGitHub(callbacks, '', matches); assert.isNotNull(res); diff --git a/developer/src/kmc/src/commands/copy.ts b/developer/src/kmc/src/commands/copy.ts index 9231ba3a5ec..2f22b24a0a4 100644 --- a/developer/src/kmc/src/commands/copy.ts +++ b/developer/src/kmc/src/commands/copy.ts @@ -27,8 +27,9 @@ export function declareCopy(program: Command) { * a .kpj file, e.g. ./keyboards/khmer_angkor/khmer_angkor.kpj * a local folder (with a .kpj file in it), e.g. ./keyboards/khmer_angkor * a cloud keyboard or lexical model, cloud:id, e.g. cloud:khmer_angkor - * a GitHub repository, optional branch, and path, github:owner/repo[:branch]:path - e.g. github:keyman-keyboards/khmer_angkor:main:/khmer_angkor.kpj + * a GitHub repository, branch, and path, [https://]github.com/owner/repo/tree/branch/path + e.g. https://github.com/keyman-keyboards/khmer_angkor/tree/main/khmer_angkor.kpj or + github.com/keymanapp/keyboards/tree/master/release/k/khmer_angkor `); }