From 9e6590065ba55b530d22a89043f2143ec6b32845 Mon Sep 17 00:00:00 2001 From: Nell Hardcastle Date: Thu, 7 Nov 2024 11:45:27 -0800 Subject: [PATCH] refactor(app): Decouple from @openneuro/client The client will no longer be needed with the move to deno CLI and this allows for frontend specific client configuration. Fixes error handling not bubbling up properly across the React app. --- packages/openneuro-app/package.json | 1 - packages/openneuro-app/src/client.jsx | 32 +++-- .../dataset/download/download-query.js | 39 +++++- .../scripts/uploader/file-upload-parallel.ts | 122 ++++++++++++++++++ .../src/scripts/uploader/file-upload.js | 6 +- .../src/scripts/uploader/hash-file-list.ts | 36 ++++++ .../src/scripts/uploader/upload-mutation.js | 36 +++++- .../src/scripts/uploader/uploader.jsx | 28 +++- 8 files changed, 268 insertions(+), 32 deletions(-) create mode 100644 packages/openneuro-app/src/scripts/uploader/file-upload-parallel.ts create mode 100644 packages/openneuro-app/src/scripts/uploader/hash-file-list.ts diff --git a/packages/openneuro-app/package.json b/packages/openneuro-app/package.json index 62019ce3aa..0fa910536a 100644 --- a/packages/openneuro-app/package.json +++ b/packages/openneuro-app/package.json @@ -19,7 +19,6 @@ "@emotion/react": "11.11.1", "@emotion/styled": "11.11.0", "@niivue/niivue": "0.45.1", - "@openneuro/client": "^4.28.3", "@openneuro/components": "^4.28.3", "@sentry/react": "^8.25.0", "@tanstack/react-table": "^8.9.3", diff --git a/packages/openneuro-app/src/client.jsx b/packages/openneuro-app/src/client.jsx index 74989cde27..9229f8b32c 100644 --- a/packages/openneuro-app/src/client.jsx +++ b/packages/openneuro-app/src/client.jsx @@ -3,14 +3,12 @@ */ import "./scripts/utils/global-polyfill" import "./scripts/sentry" -import { ApolloProvider, InMemoryCache } from "@apollo/client" -import { createClient } from "@openneuro/client" +import { ApolloClient, ApolloProvider, InMemoryCache } from "@apollo/client" import React from "react" import { createRoot } from "react-dom/client" import { BrowserRouter, Route, Routes } from "react-router-dom" import App from "./scripts/app" import Index from "./scripts/index" -import { version } from "./lerna.json" import { config } from "./scripts/config" import * as gtag from "./scripts/utils/gtag" import { relayStylePagination } from "@apollo/client/utilities" @@ -20,22 +18,22 @@ gtag.initialize(config.analytics.trackingIds) const mainElement = document.getElementById("main") const container = createRoot(mainElement) +const client = new ApolloClient({ + uri: `${config.url}/crn/graphql`, + cache: new InMemoryCache({ + typePolicies: { + Query: { + fields: { + advancedSearch: relayStylePagination(), + }, + }, + }, + }), +}) + container.render( - + } /> diff --git a/packages/openneuro-app/src/scripts/dataset/download/download-query.js b/packages/openneuro-app/src/scripts/dataset/download/download-query.js index 2d4190a7d7..b4e37b03ac 100644 --- a/packages/openneuro-app/src/scripts/dataset/download/download-query.js +++ b/packages/openneuro-app/src/scripts/dataset/download/download-query.js @@ -1,10 +1,43 @@ -import { datasets } from "@openneuro/client" +import { gql } from "@apollo/client" + +export const DOWNLOAD_DATASET = gql` +query downloadDraft($datasetId: ID!, $tree: String) { + dataset(id: $datasetId) { + id + draft { + id + files(tree: $tree) { + id + directory + filename + size + urls + } + } + } +} +` + +export const DOWNLOAD_SNAPSHOT = gql` + query downloadSnapshot($datasetId: ID!, $tag: String!, $tree: String) { + snapshot(datasetId: $datasetId, tag: $tag) { + id + files(tree: $tree) { + id + directory + filename + size + urls + } + } + } +` export const downloadDataset = (client) => async ({ datasetId, snapshotTag, tree = null }) => { if (snapshotTag) { const { data } = await client.query({ - query: datasets.downloadSnapshot, + query: DOWNLOAD_SNAPSHOT, variables: { datasetId, tag: snapshotTag, @@ -14,7 +47,7 @@ export const downloadDataset = return data.snapshot.files } else { const { data } = await client.query({ - query: datasets.downloadDataset, + query: DOWNLOAD_DATASET, variables: { datasetId, tree, diff --git a/packages/openneuro-app/src/scripts/uploader/file-upload-parallel.ts b/packages/openneuro-app/src/scripts/uploader/file-upload-parallel.ts new file mode 100644 index 0000000000..dc4d70ddcc --- /dev/null +++ b/packages/openneuro-app/src/scripts/uploader/file-upload-parallel.ts @@ -0,0 +1,122 @@ +/** + * Convert from a URL compatible path + * @param {String} path + */ +export const decodeFilePath = (path) => { + return path.replace(new RegExp(":", "g"), "/") +} + +/** + * Determine parallelism based on Request list + * @param {Array} requests + * @param {number} bytes expected total size of all requests + * @returns {number} + */ +export function uploadParallelism(requests, bytes) { + const averageSize = bytes / requests.length + const parallelism = averageSize / 524288 // 512KB + if (parallelism > 8) { + return 8 + } else if (parallelism < 2) { + return 2 + } else { + return Math.round(parallelism) + } +} + +/** + * Extract filename from Request URL + * @param {string} url .../a:path:to:a:file + */ +export function parseFilename(url) { + const filePath = url.substring(url.lastIndexOf("/") + 1) + return decodeFilePath(filePath) +} + +/** + * Control retry delay for upload file requests + * @param {number} step Attempt number + * @param {Request} request Active request + */ +export async function retryDelay(step, request) { + if (step <= 4) { + await new Promise((r) => setTimeout(r, step ** 2 * 1000)) + } else { + throw new Error( + `Failed to upload file after ${step} attempts - "${request.url}"`, + ) + } +} + +/** + * Repeatable function for single file upload fetch request + * @param {object} uploadProgress Progress controller instance + * @param {typeof fetch} fetch Fetch implementation to use - useful for environments without a native fetch + * @returns {function (Request, number): Promise} + */ +export const uploadFile = + (uploadProgress, fetch) => async (request, attempt = 1) => { + // Create a retry function with attempts incremented + const filename = parseFilename(request.url) + const handleFailure = async (failure) => { + const retryClone = request.clone() + // eslint-disable-next-line no-console + console.warn(`\nRetrying upload for ${filename}: ${failure}`) + try { + await retryDelay(attempt, request) + return uploadFile(uploadProgress, fetch)(retryClone, attempt + 1) + } catch (err) { + if ("failUpload" in uploadProgress) { + uploadProgress.failUpload(filename) + } + throw err + } + } + // This is needed to cancel the request in case of client errors + if ("startUpload" in uploadProgress) { + uploadProgress.startUpload(filename) + } + try { + // Clone before using the request to allow retries to reuse the body + const response = await fetch(request) + if (response.status === 200) { + // We need to wait for the response body or fetch-h2 may leave the connection open + await response.json() + if ("finishUpload" in uploadProgress) { + uploadProgress.finishUpload(filename) + } + uploadProgress.increment() + } else { + await handleFailure(response.statusText) + } + } catch (err) { + await handleFailure(err) + } + } + +/** + * @param {Request[]} requests + * @param {number} totalSize + * @param {object} uploadProgress + * @param {typeof fetch} fetch + */ +export async function uploadParallel( + requests, + totalSize, + uploadProgress, + fetch, +) { + // Array stride of parallel requests + const parallelism = uploadParallelism(requests, totalSize) + for ( + let rIndex = 0; + rIndex < requests.length; + rIndex = rIndex + parallelism + ) { + await Promise.allSettled( + requests + .slice(rIndex, rIndex + parallelism) + .map(uploadFile(uploadProgress, fetch)), + ) + } +} diff --git a/packages/openneuro-app/src/scripts/uploader/file-upload.js b/packages/openneuro-app/src/scripts/uploader/file-upload.js index 33eabbfac6..9df2093ca6 100644 --- a/packages/openneuro-app/src/scripts/uploader/file-upload.js +++ b/packages/openneuro-app/src/scripts/uploader/file-upload.js @@ -1,5 +1,5 @@ import { config } from "../config" -import { uploads } from "@openneuro/client" +import { uploadParallel } from "./file-upload-parallel" /** * Trim the webkitRelativePath value to only include the dataset relative path @@ -38,7 +38,7 @@ export const getRelativePath = ( */ export const encodeFilePath = (file, options = { stripRelativePath: false }) => file.webkitRelativePath - ? uploads.encodeFilePath(getRelativePath(file, options)) + ? getRelativePath(file, options).replace(new RegExp("/", "g"), ":") : file.name /** @@ -85,5 +85,5 @@ export async function uploadFiles({ // No background fetch // Parallelism is handled by the client in this case - return uploads.uploadParallel(requests, totalSize, uploadProgress, fetch) + return uploadParallel(requests, totalSize, uploadProgress, fetch) } diff --git a/packages/openneuro-app/src/scripts/uploader/hash-file-list.ts b/packages/openneuro-app/src/scripts/uploader/hash-file-list.ts new file mode 100644 index 0000000000..4e5f09ea2a --- /dev/null +++ b/packages/openneuro-app/src/scripts/uploader/hash-file-list.ts @@ -0,0 +1,36 @@ +/** + * Java hashcode implementation for browser and Node.js + * @param {string} str + */ +function hashCode(str) { + return str + .split("") + .reduce( + (prevHash, currVal) => + ((prevHash << 5) - prevHash + currVal.charCodeAt(0)) | 0, + 0, + ) +} + +/** + * Calculate a hash from a list of files to upload + * @param {string} datasetId Dataset namespace for this hash + * @param {Array} files Files being uploaded + * @returns {string} Hex string identity hash + */ +export function hashFileList(datasetId, files) { + return Math.abs( + hashCode( + datasetId + + files + .map( + (f) => + `${ + "webkitRelativePath" in f ? f.webkitRelativePath : f.filename + }:${f.size}`, + ) + .sort() + .join(":"), + ), + ).toString(16) +} diff --git a/packages/openneuro-app/src/scripts/uploader/upload-mutation.js b/packages/openneuro-app/src/scripts/uploader/upload-mutation.js index d53314d2be..f4d947e52e 100644 --- a/packages/openneuro-app/src/scripts/uploader/upload-mutation.js +++ b/packages/openneuro-app/src/scripts/uploader/upload-mutation.js @@ -1,6 +1,34 @@ -import { datasets, uploads } from "@openneuro/client" +import { gql } from "@apollo/client" import { SUBMIT_METADATA } from "../dataset/mutations/submit-metadata.jsx" +export const CREATE_DATASET = gql` + mutation createDataset($affirmedDefaced: Boolean, $affirmedConsent: Boolean) { + createDataset( + affirmedDefaced: $affirmedDefaced + affirmedConsent: $affirmedConsent + ) { + id + } + } +` + +export const PREPARE_UPLOAD = gql` + mutation prepareUpload($datasetId: ID!, $uploadId: ID!) { + prepareUpload(datasetId: $datasetId, uploadId: $uploadId) { + id + datasetId + token + endpoint + } + } +` + +export const FINISH_UPLOAD = gql` + mutation finishUpload($uploadId: ID!) { + finishUpload(uploadId: $uploadId) + } +` + /** * Create a dataset and update the label * @param {object} client Apollo client @@ -9,7 +37,7 @@ export const createDataset = (client) => ({ affirmedDefaced, affirmedConsent }) => { return client .mutate({ - mutation: datasets.createDataset, + mutation: CREATE_DATASET, variables: { affirmedDefaced, affirmedConsent }, errorPolicy: "all", }) @@ -22,7 +50,7 @@ export const createDataset = */ export const prepareUpload = (client) => ({ datasetId, uploadId }) => { return client.mutate({ - mutation: uploads.prepareUpload, + mutation: PREPARE_UPLOAD, variables: { datasetId, uploadId }, }) } @@ -33,7 +61,7 @@ export const prepareUpload = (client) => ({ datasetId, uploadId }) => { */ export const finishUpload = (client) => (uploadId) => { return client.mutate({ - mutation: uploads.finishUpload, + mutation: FINISH_UPLOAD, variables: { uploadId }, }) } diff --git a/packages/openneuro-app/src/scripts/uploader/uploader.jsx b/packages/openneuro-app/src/scripts/uploader/uploader.jsx index 8601e165c3..a42a052263 100644 --- a/packages/openneuro-app/src/scripts/uploader/uploader.jsx +++ b/packages/openneuro-app/src/scripts/uploader/uploader.jsx @@ -3,17 +3,37 @@ import { toast } from "react-toastify" import ToastContent from "../common/partials/toast-content.jsx" import React from "react" import PropTypes from "prop-types" -import { ApolloConsumer } from "@apollo/client" +import { ApolloConsumer, gql } from "@apollo/client" import * as gtag from "../utils/gtag" import UploaderContext from "./uploader-context.js" import FileSelect from "./file-select" import { locationFactory } from "./uploader-location.js" import * as mutation from "./upload-mutation.js" -import { datasets, uploads } from "@openneuro/client" import { useNavigate } from "react-router-dom" import { uploadFiles } from "./file-upload.js" import { UploadProgress } from "./upload-progress-class" import { addPathToFiles } from "./add-path-to-files.js" +import { hashFileList } from "./hash-file-list" + +// Get only working tree files +const GET_DRAFT_FILES = gql` + query dataset($id: ID!, $tree: String) { + dataset(id: $id) { + id + draft { + id + files(tree: $tree) { + filename + size + } + } + metadata { + affirmedDefaced + affirmedConsent + } + } + } +` /** * Stateful uploader workflow and status @@ -101,7 +121,7 @@ export class UploadClient extends React.Component { return ({ files }) => { this.props.client .query({ - query: datasets.getDraftFiles, + query: GET_DRAFT_FILES, variables: { id: datasetId }, }) .then( @@ -279,7 +299,7 @@ export class UploadClient extends React.Component { }, } = await mutation.prepareUpload(this.props.client)({ datasetId: this.state.datasetId, - uploadId: uploads.hashFileList(this.state.datasetId, filesToUpload), + uploadId: hashFileList(this.state.datasetId, filesToUpload), }) try {