From 0afaec262170408e2448b037c14efd96b05b8f08 Mon Sep 17 00:00:00 2001 From: Ian Philips Date: Fri, 22 Apr 2022 14:54:11 -0600 Subject: [PATCH 1/3] Support local firestore emulation --- functions/.gitignore | 2 ++ functions/README.md | 16 ++++++++++++++-- functions/package.json | 11 +++++++++-- web/README.md | 5 +++++ web/lib/firebase/api-call.ts | 5 ++--- web/lib/firebase/init.ts | 20 +++++++++++++++++++- web/package.json | 1 + 7 files changed, 52 insertions(+), 8 deletions(-) diff --git a/functions/.gitignore b/functions/.gitignore index f6db6f5f16..b4eba65097 100644 --- a/functions/.gitignore +++ b/functions/.gitignore @@ -15,3 +15,5 @@ node_modules/ package-lock.json ui-debug.log firebase-debug.log +firestore-debug.log +firestore_export/ \ No newline at end of file diff --git a/functions/README.md b/functions/README.md index 2bce0c9271..b348ba73f8 100644 --- a/functions/README.md +++ b/functions/README.md @@ -21,12 +21,24 @@ Adapted from https://firebase.google.com/docs/functions/get-started 4. `$ firebase use dev` to choose the dev project 5. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev (TODO: maybe not for Manifold) +### Preparing local Firestore database: + +0. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI +1. `$ brew install java` to install java if you don't already have it + 1. `$ echo 'export PATH="/usr/local/opt/openjdk/bin:$PATH"' >> ~/.zshrc` to add java to your path +2. `$ gcloud auth login` to authenticate the CLI tools to Firebase +3. `$ gcloud config set project ` to choose the project (`$ gcloud projects list` to see options) +4. `$ mkdir firestore_export` to create a folder to store the exported database +5. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local + 1. TODO: this won't work when open source, we'll have to point to the public db + ## Developing locally -0. `$ yarn dev` to spin up the emulators +1. `$ yarn serve` to spin up the emulators The Emulator UI is at http://localhost:4000; the functions are hosted on :5001. Note: You have to kill and restart emulators when you change code; no hot reload =( -1. Connect to emulators by enabling `functions.useEmulator('localhost', 5001)` +2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend + 1. Note: emulated database is cleared after every shutdown ## Debugging diff --git a/functions/package.json b/functions/package.json index 144d3d54ff..8c137f67ca 100644 --- a/functions/package.json +++ b/functions/package.json @@ -1,14 +1,21 @@ { "name": "functions", "version": "1.0.0", + "config": { + "firestore": "dev-mantic-markets.appspot.com" + }, "scripts": { "build": "tsc", "watch": "tsc -w", - "serve": "yarn build && firebase emulators:start --only functions", "shell": "yarn build && firebase functions:shell", "start": "yarn shell", "deploy": "firebase deploy --only functions", - "logs": "firebase functions:log" + "logs": "firebase functions:log", + "serve": "yarn build && firebase emulators:start --only functions,firestore --import=./firestore_export", + "db:update-local-from-remote": "yarn db:backup-remote && gsutil rsync -r gs://$npm_package_config_firestore/firestore_export ./firestore_export", + "db:backup-local": "firebase emulators:export --force ./firestore_export", + "db:rename-remote-backup-folder": "gsutil mv gs://$npm_package_config_firestore/firestore_export gs://$npm_package_config_firestore/firestore_export_$(date +%d-%m-%Y-%H-%M)", + "db:backup-remote": "yarn db:rename-remote-backup-folder && gcloud firestore export gs://$npm_package_config_firestore/firestore_export/" }, "main": "lib/functions/src/index.js", "dependencies": { diff --git a/web/README.md b/web/README.md index f9660eec76..0eeb1a7af7 100644 --- a/web/README.md +++ b/web/README.md @@ -8,6 +8,11 @@ (`yarn dev` will point you to prod database) +### Running with local emulated database and functions + +1. `yarn serve` first in `/functions` and wait for it to start +2. `yarn dev:emulate` will point you to the emulated database + ## Formatting Before committing, run `yarn format` to format your code. diff --git a/web/lib/firebase/api-call.ts b/web/lib/firebase/api-call.ts index cca726bc40..1c5522e705 100644 --- a/web/lib/firebase/api-call.ts +++ b/web/lib/firebase/api-call.ts @@ -1,10 +1,9 @@ -import { getFunctions, httpsCallable } from 'firebase/functions' +import { httpsCallable } from 'firebase/functions' import { Fold } from '../../../common/fold' import { User } from '../../../common/user' import { randomString } from '../../../common/util/random' import './init' - -const functions = getFunctions() +import { functions } from './init' export const cloudFunction = (name: string) => httpsCallable(functions, name) diff --git a/web/lib/firebase/init.ts b/web/lib/firebase/init.ts index b11ad3559a..46f143309e 100644 --- a/web/lib/firebase/init.ts +++ b/web/lib/firebase/init.ts @@ -1,8 +1,26 @@ import { getFirestore } from '@firebase/firestore' import { initializeApp, getApps, getApp } from 'firebase/app' import { FIREBASE_CONFIG } from '../../../common/envs/constants' +import { connectFirestoreEmulator } from 'firebase/firestore' +import { connectFunctionsEmulator, getFunctions } from 'firebase/functions' // Initialize Firebase export const app = getApps().length ? getApp() : initializeApp(FIREBASE_CONFIG) +export const db = getFirestore() +export const functions = getFunctions() -export const db = getFirestore(app) +const EMULATORS_STARTED = 'EMULATORS_STARTED' +function startEmulators() { + // I don't like this but this is the only way to reconnect to the emulators without error, see: https://stackoverflow.com/questions/65066963/firebase-firestore-emulator-error-host-has-been-set-in-both-settings-and-usee + // @ts-ignore + if (!global[EMULATORS_STARTED]) { + // @ts-ignore + global[EMULATORS_STARTED] = true + connectFirestoreEmulator(db, 'localhost', 8080) + connectFunctionsEmulator(functions, 'localhost', 5001) + } +} + +if (process.env.NEXT_PUBLIC_FIREBASE_EMULATE) { + startEmulators() +} diff --git a/web/package.json b/web/package.json index d398821f6b..b1c5443fe2 100644 --- a/web/package.json +++ b/web/package.json @@ -7,6 +7,7 @@ "devdev": "NEXT_PUBLIC_FIREBASE_ENV=DEV concurrently -n NEXT,TS -c magenta,cyan \"FIREBASE_ENV=DEV next dev -p 3000\" \"FIREBASE_ENV=DEV yarn ts --watch\" # see https://github.com/vercel/next.js/discussions/33634", "dev:dev": "yarn devdev", "dev:the": "NEXT_PUBLIC_FIREBASE_ENV=THEOREMONE concurrently -n NEXT,TS -c magenta,cyan \"FIREBASE_ENV=THEOREMONE next dev -p 3000\" \"FIREBASE_ENV=THEOREMONE yarn ts --watch\"", + "dev:emulate": "NEXT_PUBLIC_FIREBASE_EMULATE=TRUE yarn devdev", "ts": "tsc --noEmit --incremental --preserveWatchOutput --pretty", "build": "next build", "start": "next start", From 8efd0b6f9c16ab6cdefa2e8d6785bc264fb38422 Mon Sep 17 00:00:00 2001 From: Ian Philips Date: Mon, 25 Apr 2022 09:29:28 -0600 Subject: [PATCH 2/3] Update readme --- functions/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/functions/README.md b/functions/README.md index b348ba73f8..3e2703fe9a 100644 --- a/functions/README.md +++ b/functions/README.md @@ -19,7 +19,7 @@ Adapted from https://firebase.google.com/docs/functions/get-started 2. `$ yarn` to install JS dependencies 3. `$ firebase login` to authenticate the CLI tools to Firebase 4. `$ firebase use dev` to choose the dev project -5. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev (TODO: maybe not for Manifold) +5. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev ### Preparing local Firestore database: @@ -40,6 +40,13 @@ Adapted from https://firebase.google.com/docs/functions/get-started 2. `$ yarn dev:emulate` in `/web` to connect to emulators with the frontend 1. Note: emulated database is cleared after every shutdown +## Firestore Commands + +- `db:update-local-from-remote` - Pull the remote db from Firestore to local, also calls: + - `db:backup-remote` - Exports the remote dev db to the backup folder on Google Cloud Storage (called on every `db:update-local-from-remote`) + - `db:rename-remote-backup-folder` - Renames the remote backup folder (called on every `db:backup-remote` to preserve the previous db backup) +- `db:backup-local` - Save the local db changes to the disk (overwrites existing) + ## Debugging - Find local logs directly in the shell that ran `$ yarn dev` From c17a1ab9378e84a143506901372280f7c900feaa Mon Sep 17 00:00:00 2001 From: Ian Philips Date: Mon, 25 Apr 2022 09:44:07 -0600 Subject: [PATCH 3/3] Update readme --- functions/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/functions/README.md b/functions/README.md index 3e2703fe9a..5f24a21792 100644 --- a/functions/README.md +++ b/functions/README.md @@ -19,17 +19,17 @@ Adapted from https://firebase.google.com/docs/functions/get-started 2. `$ yarn` to install JS dependencies 3. `$ firebase login` to authenticate the CLI tools to Firebase 4. `$ firebase use dev` to choose the dev project -5. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev -### Preparing local Firestore database: +### For local development -0. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI -1. `$ brew install java` to install java if you don't already have it +0. `$ firebase functions:config:get > .runtimeconfig.json` to cache secrets for local dev +1. [Install](https://cloud.google.com/sdk/docs/install) gcloud CLI +2. `$ brew install java` to install java if you don't already have it 1. `$ echo 'export PATH="/usr/local/opt/openjdk/bin:$PATH"' >> ~/.zshrc` to add java to your path -2. `$ gcloud auth login` to authenticate the CLI tools to Firebase -3. `$ gcloud config set project ` to choose the project (`$ gcloud projects list` to see options) -4. `$ mkdir firestore_export` to create a folder to store the exported database -5. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local +3. `$ gcloud auth login` to authenticate the CLI tools to Google Cloud +4. `$ gcloud config set project ` to choose the project (`$ gcloud projects list` to see options) +5. `$ mkdir firestore_export` to create a folder to store the exported database +6. `$ yarn db:update-local-from-remote` to pull the remote db from Firestore to local 1. TODO: this won't work when open source, we'll have to point to the public db ## Developing locally