Make changes required for Desktop FS updates

.gitignore

@@ -12,6 +12,11 @@
 /sandbox_venv*
 /.vscode/
 
+# Files created by grist-desktop setup 
+/cpython.tar.gz
+/python
+/static_ext
+
 # Build helper files.
 /.build*
 
@@ -82,4 +87,5 @@ xunit.xml
 **/_build
 
 # ext directory can be overwritten
-ext/**
+/ext
+/ext/**

app/client/ui/AppUI.ts

@@ -91,6 +91,8 @@ function createMainPage(appModel: AppModel, appObj: App) {
 
 function pagePanelsHome(owner: IDisposableOwner, appModel: AppModel, app: App) {
   const pageModel = HomeModelImpl.create(owner, appModel, app.clientScope);
+  // Expose the HomeModel for grist-desktop.
+  (window as any).gristHomeModel = pageModel;
   const leftPanelOpen = Observable.create(owner, true);
 
   // Set document title to strings like "Home - Grist" or "Org Name - Grist".

app/client/ui/HomeIntro.ts

@@ -3,7 +3,7 @@ import {getLoginOrSignupUrl, getLoginUrl, getSignupUrl, urlState} from 'app/clie
 import {HomeModel} from 'app/client/models/HomeModel';
 import {productPill} from 'app/client/ui/AppHeader';
 import * as css from 'app/client/ui/DocMenuCss';
-import {createDocAndOpen, importDocAndOpen} from 'app/client/ui/HomeLeftPane';
+import {createDocAndOpen, importDocAndOpen} from 'app/client/ui/NewDocMethods';
 import {manageTeamUsersApp} from 'app/client/ui/OpenUserManager';
 import {bigBasicButton, cssButton} from 'app/client/ui2018/buttons';
 import {testId, theme, vars} from 'app/client/ui2018/cssVars';

app/client/ui/HomeLeftPane.ts

@@ -1,29 +1,27 @@
-import {makeT} from 'app/client/lib/localization';
-import {loadUserManager} from 'app/client/lib/imports';
-import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
-import {reportError} from 'app/client/models/AppModel';
-import {docUrl, urlState} from 'app/client/models/gristUrlState';
-import {HomeModel} from 'app/client/models/HomeModel';
-import {getWorkspaceInfo, workspaceName} from 'app/client/models/WorkspaceInfo';
-import {getAdminPanelName} from 'app/client/ui/AdminPanel';
+import * as roles from 'app/common/roles';
 import {addNewButton, cssAddNewButton} from 'app/client/ui/AddNewButton';
-import {docImport, importFromPlugin} from 'app/client/ui/HomeImports';
+import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls';
+import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs';
+import {createDocAndOpen, importDocAndOpen, importFromPluginAndOpen} from 'app/client/ui/NewDocMethods';
+import {createHelpTools, cssLeftPanel, cssScrollPane,
+  cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon';
 import {
   cssLinkText, cssMenuTrigger, cssPageEntry, cssPageIcon, cssPageLink, cssSpacer
 } from 'app/client/ui/LeftPanelCommon';
-import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour';
-import {transientInput} from 'app/client/ui/transientInput';
-import {testId, theme} from 'app/client/ui2018/cssVars';
-import {icon} from 'app/client/ui2018/icons';
+import {getWorkspaceInfo, workspaceName} from 'app/client/models/WorkspaceInfo';
 import {menu, menuIcon, menuItem, upgradableMenuItem, upgradeText} from 'app/client/ui2018/menus';
+import {testId, theme} from 'app/client/ui2018/cssVars';
 import {confirmModal} from 'app/client/ui2018/modals';
-import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls';
-import * as roles from 'app/common/roles';
+import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour';
+import {getAdminPanelName} from 'app/client/ui/AdminPanel';
 import {getGristConfig} from 'app/common/urlUtils';
+import {HomeModel} from 'app/client/models/HomeModel';
+import {icon} from 'app/client/ui2018/icons';
+import {loadUserManager} from 'app/client/lib/imports';
+import {makeT} from 'app/client/lib/localization';
+import {transientInput} from 'app/client/ui/transientInput';
+import {urlState} from 'app/client/models/gristUrlState';
 import {Workspace} from 'app/common/UserAPI';
-import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs';
-import {createHelpTools, cssLeftPanel, cssScrollPane,
-        cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon';
 
 const t = makeT('HomeLeftPane');
 
@@ -159,48 +157,6 @@ export function createHomeLeftPane(leftPanelOpen: Observable<boolean>, home: Hom
   );
 }
 
-export async function createDocAndOpen(home: HomeModel) {
-  const destWS = home.newDocWorkspace.get();
-  if (!destWS) { return; }
-  try {
-    const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id);
-    // Fetch doc information including urlId.
-    // TODO: consider changing API to return same response as a GET when creating an
-    // object, which is a semi-standard.
-    const doc = await home.app.api.getDoc(docId);
-    await urlState().pushUrl(docUrl(doc));
-  } catch (err) {
-    reportError(err);
-  }
-}
-
-export async function importDocAndOpen(home: HomeModel) {
-  const destWS = home.newDocWorkspace.get();
-  if (!destWS) { return; }
-  const docId = await docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id);
-  if (docId) {
-    const doc = await home.app.api.getDoc(docId);
-    await urlState().pushUrl(docUrl(doc));
-  }
-}
-
-export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) {
-  try {
-    const destWS = home.newDocWorkspace.get();
-    if (!destWS) { return; }
-    const docId = await importFromPlugin(
-      home.app,
-      destWS === "unsaved" ? "unsaved" : destWS.id,
-      source);
-    if (docId) {
-      const doc = await home.app.api.getDoc(docId);
-      await urlState().pushUrl(docUrl(doc));
-    }
-  } catch (err) {
-    reportError(err);
-  }
-}
-
 function addMenu(home: HomeModel, creating: Observable<boolean>): DomElementArg[] {
   const org = home.app.currentOrg;
   const orgAccess: roles.Role|null = org ? org.access : null;

app/client/ui/ImportProgress.ts

@@ -0,0 +1,58 @@
+import {IProgress} from 'app/client/models/NotifyModel';
+import {Disposable} from 'grainjs';
+
+export class ImportProgress extends Disposable {
+  // Import does upload first, then import. We show a single indicator, estimating which fraction
+  // of the time should be given to upload (whose progress we can report well), and which to the
+  // subsequent import (whose progress indicator is mostly faked).
+  private _uploadFraction: number;
+  private _estImportSeconds: number;
+
+  private _importTimer: null | ReturnType<typeof setInterval> = null;
+  private _importStart: number = 0;
+
+  constructor(private _progressUI: IProgress, file: File) {
+    super();
+    // We'll assume that for .grist files, the upload takes 90% of the total time, and for other
+    // files, 40%.
+    this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4;
+
+    // TODO: Import step should include a progress callback, to be combined with upload progress.
+    // Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and
+    // use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful,
+    // but does slow down for larger files, and is more comforting than a stuck indicator.
+    this._estImportSeconds = file.size / 1024 / 1024 * 2;
+
+    this._progressUI.setProgress(0);
+    this.onDispose(() => this._importTimer && clearInterval(this._importTimer));
+  }
+
+  // Once this reaches 100, the import stage begins.
+  public setUploadProgress(percentage: number) {
+    this._progressUI.setProgress(percentage * this._uploadFraction);
+    if (percentage >= 100 && !this._importTimer) {
+      this._importStart = Date.now();
+      this._importTimer = setInterval(() => this._onImportTimer(), 100);
+    }
+  }
+
+  public finish() {
+    if (this._importTimer) {
+      clearInterval(this._importTimer);
+    }
+    this._progressUI.setProgress(100);
+  }
+
+  /**
+   * Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically
+   * approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the
+   * estimate is good, and to keep showing slowing progress even if it's not.
+   */
+  private _onImportTimer() {
+    const elapsedSeconds = (Date.now() - this._importStart) / 1000;
+    const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds);
+    const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction);
+    this._progressUI.setProgress(100 * progress);
+  }
+}
+

app/gen-server/lib/homedb/HomeDBManager.ts

@@ -68,12 +68,12 @@ import {Request} from "express";
 import {defaultsDeep, flatten, pick} from 'lodash';
 import {
   Brackets,
-  Connection,
   DatabaseType,
+  DataSource,
   EntityManager,
   ObjectLiteral,
   SelectQueryBuilder,
-  WhereExpression
+  WhereExpressionBuilder
 } from "typeorm";
 import uuidv4 from "uuid/v4";
 
@@ -247,7 +247,7 @@ export type BillingOptions = Partial<Pick<BillingAccount,
  */
 export class HomeDBManager extends EventEmitter {
   private _usersManager = new UsersManager(this, this._runInTransaction.bind(this));
-  private _connection: Connection;
+  private _connection: DataSource;
   private _dbType: DatabaseType;
   private _exampleWorkspaceId: number;
   private _exampleOrgId: number;
@@ -985,6 +985,10 @@ export class HomeDBManager extends EventEmitter {
     return doc;
   }
 
+  public async getAllDocs() {
+    return this.connection.getRepository(Document).find();
+  }
+
   public async getRawDocById(docId: string, transaction?: EntityManager) {
     return await this.getDoc({
       urlId: docId,
@@ -3435,7 +3439,7 @@ export class HomeDBManager extends EventEmitter {
   // Adds a where clause to filter orgs by domain or id.
   // If org is null, filter for user's personal org.
   // if includeSupport is true, include the org of the support@ user (for the Samples workspace)
-  private _whereOrg<T extends WhereExpression>(qb: T, org: string|number, includeSupport = false): T {
+  private _whereOrg<T extends WhereExpressionBuilder>(qb: T, org: string|number, includeSupport = false): T {
     if (this.isMergedOrg(org)) {
       // Select from universe of personal orgs.
       // Don't panic though!  While this means that SQL can't use an organization id
@@ -3455,7 +3459,7 @@ export class HomeDBManager extends EventEmitter {
     }
   }
 
-  private _wherePlainOrg<T extends WhereExpression>(qb: T, org: string|number): T {
+  private _wherePlainOrg<T extends WhereExpressionBuilder>(qb: T, org: string|number): T {
     if (typeof org === 'number') {
       return qb.andWhere('orgs.id = :org', {org});
     }