diff --git a/package-lock.json b/package-lock.json index 9e1cb4bde..6d63047d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,12 +21,14 @@ "@mapeo/sqlite-indexer": "1.0.0-alpha.8", "@sinclair/typebox": "^0.29.6", "b4a": "^1.6.3", + "bcp-47": "^2.1.0", "better-sqlite3": "^8.7.0", "big-sparse-array": "^1.0.3", "bogon": "^1.1.0", "compact-encoding": "^2.12.0", "corestore": "^6.8.4", "debug": "^4.3.4", + "dot-prop": "^8.0.2", "drizzle-orm": "^0.30.8", "fastify": ">= 4", "fastify-plugin": "^4.5.1", @@ -2256,6 +2258,20 @@ ], "license": "MIT" }, + "node_modules/bcp-47": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/bcp-47/-/bcp-47-2.1.0.tgz", + "integrity": "sha512-9IIS3UPrvIa1Ej+lVDdDwO7zLehjqsaByECw0bu2RRGP73jALm6FYbzI5gWbgHLvNdkvfXB5YrSbocZdOS0c0w==", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/better-sqlite3": { "version": "8.7.0", "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.7.0.tgz", @@ -3172,6 +3188,31 @@ "node": ">=6.0.0" } }, + "node_modules/dot-prop": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-8.0.2.tgz", + "integrity": "sha512-xaBe6ZT4DHPkg0k4Ytbvn5xoxgpG0jOS1dYxSOwAHPuNLjP3/OzN0gH55SrLqpx8cBfSaVt91lXYkApjb+nYdQ==", + "dependencies": { + "type-fest": "^3.8.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dot-prop/node_modules/type-fest": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz", + "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dprint-node": { "version": "1.0.7", "dev": true, @@ -3286,9 +3327,9 @@ } }, "node_modules/drizzle-orm": { - "version": "0.30.8", - "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-0.30.8.tgz", - "integrity": "sha512-9pBJA0IjnpPpzZ6s9jlS1CQAbKoBmbn2GJesPhXaVblAA/joOJ4AWWevYcqvLGj9SvThBAl7WscN8Zwgg5mnTw==", + "version": "0.30.9", + "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-0.30.9.tgz", + "integrity": "sha512-VOiCFsexErmgqvNCOmbzmqDCZzZsHoz6SkWAjTFxsTr1AllKDbDJ2+GgedLXsXMDgpg/ljDG1zItIFeZtiO2LA==", "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=3", @@ -4863,6 +4904,28 @@ "node": ">= 0.4" } }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-array-buffer": { "version": "3.0.2", "dev": true, @@ -4943,6 +5006,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-docker": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", diff --git a/package.json b/package.json index 129fdf94e..4febdf73b 100644 --- a/package.json +++ b/package.json @@ -149,12 +149,14 @@ "@mapeo/sqlite-indexer": "1.0.0-alpha.8", "@sinclair/typebox": "^0.29.6", "b4a": "^1.6.3", + "bcp-47": "^2.1.0", "better-sqlite3": "^8.7.0", "big-sparse-array": "^1.0.3", "bogon": "^1.1.0", "compact-encoding": "^2.12.0", "corestore": "^6.8.4", "debug": "^4.3.4", + "dot-prop": "^8.0.2", "drizzle-orm": "^0.30.8", "fastify": ">= 4", "fastify-plugin": "^4.5.1", diff --git a/src/datatype/index.d.ts b/src/datatype/index.d.ts index 7bb0ac39b..ee6b72cac 100644 --- a/src/datatype/index.d.ts +++ b/src/datatype/index.d.ts @@ -13,6 +13,7 @@ import { SQLiteSelectBase } from 'drizzle-orm/sqlite-core' import { RunResult } from 'better-sqlite3' import type Hypercore from 'hypercore' import { TypedEmitter } from 'tiny-typed-emitter' +import TranslationApi from '../translation-api.js' type MapeoDocTableName = `${MapeoDoc['schemaName']}Table` type GetMapeoDocTables = T[keyof T & MapeoDocTableName] @@ -52,11 +53,13 @@ export class DataType< table, getPermissions, db, + getTranslations, }: { table: TTable dataStore: TDataStore db: import('drizzle-orm/better-sqlite3').BetterSQLite3Database getPermissions?: () => any + getTranslations: TranslationApi['get'] }) get [kTable](): TTable @@ -79,12 +82,16 @@ export class DataType< > >(value: T): Promise - getByDocId(docId: string): Promise + getByDocId( + docId: string, + opts?: { lang?: string } + ): Promise - getByVersionId(versionId: string): Promise + getByVersionId(versionId: string, opts?: { lang?: string }): Promise getMany(opts?: { includeDeleted?: boolean + lang?: string }): Promise> update< diff --git a/src/datatype/index.js b/src/datatype/index.js index b4e876b10..85b52f012 100644 --- a/src/datatype/index.js +++ b/src/datatype/index.js @@ -7,6 +7,8 @@ import { noop, deNullify } from '../utils.js' import { NotFoundError } from '../errors.js' import crypto from 'hypercore-crypto' import { TypedEmitter } from 'tiny-typed-emitter' +import { parse as parseBCP47 } from 'bcp-47' +import { setProperty, getProperty } from 'dot-prop' /** * @typedef {import('@mapeo/schema').MapeoDoc} MapeoDoc @@ -70,6 +72,7 @@ export class DataType extends TypedEmitter { #schemaName #sql #db + #getTranslations /** * @@ -77,15 +80,17 @@ export class DataType extends TypedEmitter { * @param {TTable} opts.table * @param {TDataStore} opts.dataStore * @param {import('drizzle-orm/better-sqlite3').BetterSQLite3Database} opts.db + * @param {import('../translation-api.js').default['get']} opts.getTranslations * @param {() => any} [opts.getPermissions] */ - constructor({ dataStore, table, getPermissions, db }) { + constructor({ dataStore, table, getPermissions, db, getTranslations }) { super() this.#dataStore = dataStore this.#table = table this.#schemaName = /** @type {TSchemaName} */ (getTableConfig(table).name) this.#getPermissions = getPermissions this.#db = db + this.#getTranslations = getTranslations this.#sql = { getByDocId: db .select() @@ -172,26 +177,73 @@ export class DataType extends TypedEmitter { /** * @param {string} docId + * @param {{ lang?: string }} [opts] */ - async getByDocId(docId) { + async getByDocId(docId, { lang } = {}) { await this.#dataStore.indexer.idle() - const result = this.#sql.getByDocId.get({ docId }) + const result = /** @type {undefined | MapeoDoc} */ ( + this.#sql.getByDocId.get({ docId }) + ) if (!result) throw new NotFoundError() - return deNullify(result) + return this.#translate(deNullify(result), { lang }) } - /** @param {string} versionId */ - async getByVersionId(versionId) { - return this.#dataStore.read(versionId) + /** + * @param {string} versionId + * @param {{ lang?: string }} [opts] + */ + async getByVersionId(versionId, { lang } = {}) { + const result = await this.#dataStore.read(versionId) + return this.#translate(result, { lang }) } - /** @param {{ includeDeleted?: boolean }} [opts] */ - async getMany({ includeDeleted = false } = {}) { + /** + * @param {MapeoDoc} doc + * @param {{ lang?: string }} [opts] + */ + async #translate(doc, { lang } = {}) { + if (!lang) return doc + + const { language, region } = parseBCP47(lang) + if (!language) return doc + const translatedDoc = JSON.parse(JSON.stringify(doc)) + + let value = { + languageCode: language, + schemaNameRef: translatedDoc.schemaName, + docIdRef: translatedDoc.docId, + regionCode: region !== null ? region : undefined, + } + let translations = await this.#getTranslations(value) + // if passing a region code returns no matches, + // fallback to matching only languageCode + if (translations.length === 0 && value.regionCode) { + value.regionCode = undefined + translations = await this.#getTranslations(value) + } + + for (let translation of translations) { + if (typeof getProperty(doc, translation.fieldRef) === 'string') { + setProperty(doc, translation.fieldRef, translation.message) + } + } + return doc + } + + /** @param {{ includeDeleted?: boolean, lang?: string }} [opts] */ + async getMany({ includeDeleted = false, lang } = {}) { await this.#dataStore.indexer.idle() const rows = includeDeleted ? this.#sql.getManyWithDeleted.all() : this.#sql.getMany.all() - return rows.map((doc) => deNullify(doc)) + return await Promise.all( + rows.map( + async (doc) => + await this.#translate(deNullify(/** @type {MapeoDoc} */ (doc)), { + lang, + }) + ) + ) } /** diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 197b8053e..31712e12f 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -83,6 +83,7 @@ export class MapeoProject extends TypedEmitter { #memberApi #iconApi #syncApi + /** @type {TranslationApi} */ #translationApi #l @@ -177,6 +178,7 @@ export class MapeoProject extends TypedEmitter { }, logger: this.#l, }) + this.#dataStores = { auth: new DataStore({ coreManager: this.#coreManager, @@ -201,56 +203,71 @@ export class MapeoProject extends TypedEmitter { storage: indexerStorage, }), } + + /** @type {typeof TranslationApi.prototype.get} */ + const getTranslations = (...args) => this.$translation.get(...args) this.#dataTypes = { observation: new DataType({ dataStore: this.#dataStores.data, table: observationTable, db, + getTranslations, }), track: new DataType({ dataStore: this.#dataStores.data, table: trackTable, db, + getTranslations, }), preset: new DataType({ dataStore: this.#dataStores.config, table: presetTable, db, + getTranslations, }), field: new DataType({ dataStore: this.#dataStores.config, table: fieldTable, db, + getTranslations, }), projectSettings: new DataType({ dataStore: this.#dataStores.config, table: projectSettingsTable, db: sharedDb, + getTranslations, }), coreOwnership: new DataType({ dataStore: this.#dataStores.auth, table: coreOwnershipTable, db, + getTranslations, }), role: new DataType({ dataStore: this.#dataStores.auth, table: roleTable, db, + getTranslations, }), deviceInfo: new DataType({ dataStore: this.#dataStores.config, table: deviceInfoTable, db, + getTranslations, }), icon: new DataType({ dataStore: this.#dataStores.config, table: iconTable, db, + getTranslations, }), translation: new DataType({ dataStore: this.#dataStores.config, table: translationTable, db, + getTranslations: () => { + throw new Error('Cannot get translation for translations') + }, }), } const identityKeypair = keyManager.getIdentityKeypair() diff --git a/test-e2e/translation-api.js b/test-e2e/translation-api.js index 95cfd0aa0..7414bafaf 100644 --- a/test-e2e/translation-api.js +++ b/test-e2e/translation-api.js @@ -142,6 +142,107 @@ test('translation api - put() and get() fields', async (t) => { } }) +test('translation api - passing `lang` to dataType', async (t) => { + const [manager] = await createManagers(1, t, 'mobile') + const project = await manager.getProject( + await manager.createProject({ + configPath: defaultConfigPath, + }) + ) + + // fields + const fields = await project.field.getMany() + const fieldTranslationsDoc = fields + .map((field) => { + const matchingTranslation = fieldTranslations.find((translation) => { + return translation.message === fieldsTranslationMap[field.label] + }) + if (matchingTranslation) + return { docIdRef: field.docId, ...matchingTranslation } + }) + .filter(isDefined) + for (const translationDoc of fieldTranslationsDoc) { + const { docIdRef, message, fieldRef } = await project.$translation.put( + translationDoc + ) + + /** @type {Record} */ + const translatedField = await project.field.getByDocId(docIdRef, { + lang: 'es', + }) + t.is( + translatedField[fieldRef], + message, + `passing 'lang' returns the correct translated field` + ) + + /** @type {Record} */ + const untranslatedField = await project.field.getByDocId(docIdRef) + t.not( + untranslatedField[fieldRef], + message, + `not passing 'lang' won't give a translated field` + ) + + /** @type {Record} */ + const fallbackRegionCodeTranslatedField = await project.field.getByDocId( + docIdRef, + { lang: 'es-CO' } + ) + t.is( + fallbackRegionCodeTranslatedField[fieldRef], + message, + `passing 'lang' with untranslated 'regionCode' returns a fallback translated field matching 'languageCode'` + ) + } + + // presets + const presets = await project.preset.getMany() + const presetTranslationsDoc = presets + .map((preset) => { + const matchingTranslation = presetTranslations.find((translation) => { + return translation.message === presetsTranslationMap[preset.name] + }) + if (matchingTranslation) + return { docIdRef: preset.docId, ...matchingTranslation } + }) + .filter(isDefined) + for (let translationDoc of presetTranslationsDoc) { + const { docIdRef, message, fieldRef } = await project.$translation.put( + translationDoc + ) + + /** @type {Record} */ + const translatedPreset = await project.preset.getByDocId(docIdRef, { + lang: 'es', + }) + t.is( + translatedPreset[fieldRef], + message, + `passing 'lang' returns the correct translated preset` + ) + + /** @type {Record} */ + const untranslatedPreset = await project.preset.getByDocId(docIdRef) + t.not( + untranslatedPreset[fieldRef], + message, + `not passing 'lang' won't give a translated preset` + ) + + /** @type {Record} */ + const fallbackRegionCodeTranslatedPreset = await project.preset.getByDocId( + docIdRef, + { lang: 'es-CO' } + ) + t.is( + fallbackRegionCodeTranslatedPreset[fieldRef], + message, + `passing 'lang' with untranslated 'regionCode' returns a fallback translated preset matching 'languageCode'` + ) + } +}) + test('translation api - re-loading from disk', async (t) => { const custodian = new ManagerCustodian(t) const deps = { diff --git a/tests/data-type.js b/tests/data-type.js index cc11550cf..27d347260 100644 --- a/tests/data-type.js +++ b/tests/data-type.js @@ -8,7 +8,7 @@ import { } from './helpers/core-manager.js' import RAM from 'random-access-memory' import crypto from 'hypercore-crypto' -import { observationTable } from '../src/schema/project.js' +import { observationTable, translationTable } from '../src/schema/project.js' import { DataType, kCreateWithDocId } from '../src/datatype/index.js' import { IndexWriter } from '../src/index-writer/index.js' import { NotFoundError } from '../src/errors.js' @@ -17,6 +17,10 @@ import Database from 'better-sqlite3' import { drizzle } from 'drizzle-orm/better-sqlite3' import { migrate } from 'drizzle-orm/better-sqlite3/migrator' import { randomBytes } from 'crypto' +import TranslationApi from '../src/translation-api.js' +import { getProperty } from 'dot-prop' +import { decode, decodeBlockPrefix } from '@mapeo/schema' +import { assert } from '../src/utils.js' /** @type {import('@mapeo/schema').ObservationValue} */ const obsFixture = { @@ -51,6 +55,9 @@ test('private createWithDocId() method', async (t) => { dataStore, table: observationTable, db, + getTranslations() { + throw new Error('Translations should not be fetched in this test') + }, }) const customId = randomBytes(8).toString('hex') const obs = await dataType[kCreateWithDocId](customId, obsFixture) @@ -83,6 +90,9 @@ test('private createWithDocId() method throws when doc exists', async (t) => { dataStore, table: observationTable, db, + getTranslations() { + throw new Error('Translations should not be fetched in this test') + }, }) const customId = randomBytes(8).toString('hex') await dataType[kCreateWithDocId](customId, obsFixture) @@ -167,6 +177,75 @@ test('delete()', async (t) => { ) }) +test('translation', async (t) => { + const projectKey = randomBytes(32) + const { dataType, translationApi } = await testenv({ projectKey }) + /** @type {import('@mapeo/schema').ObservationValue} */ + const observation = { + schemaName: 'observation', + refs: [], + tags: { + type: 'point', + }, + attachments: [], + metadata: {}, + } + + const doc = await dataType.create(observation) + const translation = { + /** @type {'translation'} */ + schemaName: 'translation', + schemaNameRef: 'observation', + docIdRef: doc.docId, + fieldRef: 'tags.type', + languageCode: 'es', + regionCode: 'AR', + message: 'punto', + } + await translationApi.put(translation) + translationApi.index(translation) + + t.is( + translation.message, + getProperty( + await dataType.getByDocId(doc.docId, { lang: 'es' }), + translation.fieldRef + ), + `we get a valid translated field` + ) + t.is( + translation.message, + getProperty( + await dataType.getByDocId(doc.docId, { lang: 'es-AR' }), + translation.fieldRef + ), + `we get a valid translated field` + ) + t.is( + translation.message, + getProperty( + await dataType.getByDocId(doc.docId, { lang: 'es-ES' }), + translation.fieldRef + ), + `passing an untranslated regionCode, still returns a translated field, since we fallback to only matching languageCode` + ) + + t.is( + getProperty(observation, 'tags.type'), + getProperty( + await dataType.getByDocId(doc.docId, { lang: 'de' }), + 'tags.type' + ), + `passing an untranslated language code returns the untranslated message` + ) + + t.is( + getProperty(observation, 'tags.type'), + getProperty(await dataType.getByDocId(doc.docId), 'tags.type'), + `not passing a a language code returns the untranslated message` + ) +}) + /** * @param {object} opts * @param {Buffer} [opts.projectKey] @@ -181,7 +260,7 @@ async function testenv(opts) { const coreManager = createCoreManager({ ...opts, db }) const indexWriter = new IndexWriter({ - tables: [observationTable], + tables: [observationTable, translationTable], sqlite, }) @@ -191,11 +270,58 @@ async function testenv(opts) { batch: async (entries) => indexWriter.batch(entries), storage: () => new RAM(), }) + + const configDataStore = new DataStore({ + coreManager, + namespace: 'config', + batch: async (entries) => { + /** @type {import('multi-core-indexer').Entry[]} */ + const entriesToIndex = [] + for (const entry of entries) { + const { schemaName } = decodeBlockPrefix(entry.block) + try { + if (schemaName === 'translation') { + const doc = decode(entry.block, { + coreDiscoveryKey: entry.key, + index: entry.index, + }) + assert( + doc.schemaName === 'translation', + 'expected a translation doc' + ) + translationApi.index(doc) + } + entriesToIndex.push(entry) + } catch { + // Ignore errors thrown by values that can't be decoded for now + } + } + const indexed = await indexWriter.batch(entriesToIndex) + return indexed + }, + storage: () => new RAM(), + }) + + const translationDataType = new DataType({ + dataStore: configDataStore, + table: translationTable, + db, + getTranslations: () => { + throw new Error('Cannot get translations for translations') + }, + }) + + const translationApi = new TranslationApi({ + dataType: translationDataType, + table: translationTable, + }) + const dataType = new DataType({ dataStore, table: observationTable, db, + getTranslations: translationApi.get.bind(translationApi), }) - return { coreManager, dataType, dataStore } + return { coreManager, dataType, dataStore, translationApi } } diff --git a/tests/icon-api.js b/tests/icon-api.js index 40febbb65..ae397bfae 100644 --- a/tests/icon-api.js +++ b/tests/icon-api.js @@ -685,6 +685,9 @@ function setup({ dataStore: iconDataStore, table: iconTable, db, + getTranslations() { + throw new Error('Translations should not be fetched in this test') + }, }) const iconApi = new IconApi({ diff --git a/tests/translation-api.js b/tests/translation-api.js index 27a0f337f..3748e8481 100644 --- a/tests/translation-api.js +++ b/tests/translation-api.js @@ -123,8 +123,11 @@ function setup() { const dataType = new DataType({ dataStore, - table: table, + table, db, + getTranslations() { + throw new Error('Cannot get translations from translations') + }, }) return new TranslationApi({