Add ability to add custom code in documents

.devcontainer/Dockerfile

@@ -0,0 +1,12 @@
+
+FROM node:22-bookworm
+RUN apt-get update && apt-get install -y python3.11 python3-pip pipx gdb
+COPY sandbox/requirements3.txt package.json yarn.lock /grist/
+RUN pip3 install --break-system-packages -r /grist/requirements3.txt
+RUN yarn install --frozen-lockfile --verbose --network-timeout 600000
+
+# absolutely bad idea normally, but I could not get python to attach to a running process otherwise
+# it always failed with "ptrace: Operation not permitted."
+RUN chmod u+s /usr/bin/gdb
+
+ENV GRIST_HOST="0.0.0.0"

.devcontainer/devcontainer.json

@@ -0,0 +1,44 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+	"name": "Existing Dockerfile",
+	"build": {
+		// Sets the run context to one level up instead of the .devcontainer folder.
+		"context": "..",
+		// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+		"dockerfile": "./Dockerfile"
+	},
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {
+			"nodeGypDependencies": true,
+			"installYarnUsingApt": true,
+			"version": "lts",
+			"pnpmVersion": "latest",
+			"nvmVersion": "latest"
+		},
+		"ghcr.io/devcontainers/features/python:1": {
+			"installTools": true,
+			"version": "os-provided"
+		}
+	},
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [
+		8484
+	],
+	// Uncomment the next line to run commands after the container is created.
+	// "postCreateCommand": "cat /etc/os-release",
+	"postCreateCommand": "yarn install && yarn install:python",
+	// Configure tool-specific properties.
+	// "customizations": {},
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	"remoteUser": "node",
+	"capAdd": [
+		"SYS_PTRACE"
+	],
+	"securityOpt": [
+		"seccomp=unconfined",
+		"apparmor=unconfined",
+	],
+}

app/client/ui/DocumentSettings.ts

@@ -2,6 +2,7 @@
  * This module export a component for editing some document settings consisting of the timezone,
  * (new settings to be added here ...).
  */
+import * as AceEditor from 'app/client/components/AceEditor';
 import {cssPrimarySmallLink, cssSmallButton, cssSmallLinkButton} from 'app/client/components/Forms/styles';
 import {GristDoc} from 'app/client/components/GristDoc';
 import {ACIndexImpl} from 'app/client/lib/ACIndex';
@@ -41,6 +42,7 @@ export class DocSettingsPage extends Disposable {
   private _timezone = this._docInfo.timezone;
   private _locale: KoSaveableObservable<string> = this._docInfo.documentSettingsJson.prop('locale');
   private _currency: KoSaveableObservable<string|undefined> = this._docInfo.documentSettingsJson.prop('currency');
+  private _customCode: KoSaveableObservable<string | undefined> = this._docInfo.documentSettingsJson.prop('customCode');
   private _engine: Computed<EngineCode|undefined> = Computed.create(this, (
     use => use(this._docInfo.documentSettingsJson.prop('engine'))
   ))
@@ -85,6 +87,16 @@ export class DocSettingsPage extends Disposable {
               {defaultCurrencyLabel: t("Local currency ({{currency}})", {currency: getCurrency(l)})})
           )
         }),
+        dom.create(AdminSectionItem, {
+          id: 'custom_code',
+          name: t('CustomCode'),
+          expandedContent: dom('div',
+            t('Custom python code to include when generating the model. Useful for defining custom functions.'),
+            AceEditor.create({ observable: this._customCode }).buildDom((aceObj: any) => {
+              aceObj.renderer.setShowGutter(true);
+            }),
+          ),
+        }),
       ]),
 
       dom.create(AdminSection, t('Data Engine'), [

app/common/DocumentSettings.ts

@@ -2,6 +2,7 @@ export interface DocumentSettings {
   locale: string;
   currency?: string;
   engine?: EngineCode;
+  customCode?: string;
 }
 
 /**

sandbox/grist/docactions.py

@@ -99,6 +99,13 @@ def BulkUpdateRecord(self, table_id, row_ids, columns):
        ("recalcWhen" in columns or "recalcDeps" in columns)):
       self._engine.trigger_columns_changed()
 
+    # If we're updating documentSettings, rebuild usercode as the code might have changed.
+    if table_id == "_grist_DocInfo" and "documentSettings" in columns:
+      self._engine.rebuild_usercode()
+      for table in self._engine.tables.keys():
+        # I guess we invalidate everything, as we do not analyze in which regards the code may have changed :S
+        self._engine.invalidate_records(table)
+
   def ReplaceTableData(self, table_id, row_ids, column_values):
     old_data = self._engine.fetch_table(table_id, formulas=False)
     self._engine.out_actions.undo.append(actions.ReplaceTableData(*old_data))

sandbox/grist/engine.py

@@ -4,6 +4,7 @@
 dependency tracking.
 """
 import itertools
+import json
 import logging
 import re
 import rlcompleter
@@ -346,6 +347,11 @@ def load_table(self, data):
     # Add the records.
     self.add_records(data.table_id, data.row_ids, columns)
 
+    if data.table_id == "_grist_DocInfo":
+      # when loading the DocInfo table, update the usercode module with the new doc settings.
+      # otherwise the custom code will not be loaded until the model is rebuilt due to other events.
+      self.rebuild_usercode()
+
   def load_done(self):
     """
     Finalizes the loading of data into this Engine.
@@ -1137,7 +1143,14 @@ def rebuild_usercode(self):
     if not self._should_rebuild_usercode:
       return
 
-    self.gencode.make_module(self.schema)
+    doc_info = None
+    try:
+      doc_info = self.docmodel.doc_info.lookupOne()
+      doc_settings = json.loads(doc_info.documentSettings)
+    except (AttributeError, ValueError) as e:
+      doc_settings = {"customCode": "## ERROR: " +
+                      repr(e).replace("\n", "\n# ")}
+    self.gencode.make_module(self.schema, doc_settings)
 
     # Re-populate self.tables, reusing existing tables whenever possible.
     old_tables = self.tables

sandbox/grist/gencode.py

@@ -164,7 +164,7 @@ def _make_table_model(self, table_info, summary_tables, filter_for_user=False):
 
     return textbuilder.Combiner(parts)
 
-  def make_module(self, schema):
+  def make_module(self, schema, doc_settings):
     """Regenerates the code text and usercode module from updated document schema."""
     # Collect summary tables to group them by source table.
     summary_tables = {}
@@ -176,6 +176,13 @@ def make_module(self, schema):
     fullparts = ["import grist\n" +
                  "from functions import *       # global uppercase functions\n" +
                  "import datetime, math, re     # modules commonly needed in formulas\n"]
+
+    user_code = doc_settings.get('customCode', '')
+    if user_code:
+      fullparts.append("\n### BEGIN CUSTOM USER CODE ###\n")
+      fullparts.append(user_code)
+      fullparts.append("\n### END CUSTOM USER CODE ###\n")
+
     userparts = fullparts[:]
     for table_info in six.itervalues(schema):
       fullparts.append("\n\n")