From 2b67b2ff6709763c6452bfd3032bc3580099d738 Mon Sep 17 00:00:00 2001 From: Henry Allen <31718268+hmallen99@users.noreply.github.com> Date: Thu, 16 Jan 2025 16:51:55 -0500 Subject: [PATCH] Implement Texture Loading (#35) * Add PanResponder stub * Stub CanvasEventHandler * Add license * EventHandler stub * Add inspo * Update gesture events * Add Event Listener to canvas element * Create CustomLocationPage for testing * Render static template * Black background * Import input into CustomLocationPage * Add logging * Use pointer events instead of panresponder * Basic boundingClientRect * Add FlyControlSystem * Start adding joystick * Pass type to CanvasEventHandler * Set target entity * Tweak follow camera settings * Update LocationPage * Add pointer listen events * Add controller back * Revert render system changes * Import the rest of the systems * Add TODO * Comment out unusable ClientModules * use .native files instead of entirely new package * Remove client-core-mobile * Remove projects-mobile * Clean up LocationRoutes reference * revert engine change * Realign with upstream * Remove url search from AvatarSpawnSystem * Add react native basis universal * Start implementing KTX2 loader * Add todos * Import KTX2 loader * Copy from original file * remove nocheck * remove init * Replace worker code * Fix types * fix podfile * add debug logs * Add debug statements * Remove RNBridgeless check * texture loading fixes * Clean up logging * Fix asset type detection * fix sky station skybox type * add todos * Get some materials loading * Add URL polyfill * expo asset patch * add license * clean up unused changes * Add logging for missing meshopt decoder * Begin texture scaling impl * handle imports * Extract createTextureFromImage * Simplify texture loading * Align with web API * Fix texture color * clean up TODOs * Remove unused code * Undo Touch changes for this branch * Add back getClientBoundingRect * duplicate AssetLoader code * fix path stripping --- .../World/LoadLocationScene.native.tsx | 17 +- .../src/networking/AvatarSpawnSystem.tsx | 12 +- packages/clientNative/App.tsx | 1 - packages/clientNative/ios/Podfile | 23 + packages/clientNative/ios/Podfile.lock | 71 +- .../xcschemes/clientNative.xcscheme | 1 + packages/clientNative/package.json | 7 + .../src/polyfill/NativeHTMLCanvasElement.ts | 13 + .../engine/src/assets/classes/AssetLoader.ts | 8 +- .../engine/src/assets/constants/AssetType.ts | 20 +- .../src/assets/functions/createGLTFLoader.ts | 17 + .../assets/loaders/gltf/KTX2Loader.native.ts | 711 +++++++++++++++++- .../loaders/texture/TextureLoader.native.ts | 337 +++++++++ packages/engine/src/gltf/GLTFExtensions.ts | 7 +- .../engine/src/gltf/GLTFLoaderFunctions.ts | 29 +- .../public/scenes/sky-station.gltf | 2 +- patches/expo-asset+11.0.2.patch | 22 + 17 files changed, 1229 insertions(+), 69 deletions(-) create mode 100644 packages/engine/src/assets/loaders/texture/TextureLoader.native.ts create mode 100644 patches/expo-asset+11.0.2.patch diff --git a/packages/client-core/src/components/World/LoadLocationScene.native.tsx b/packages/client-core/src/components/World/LoadLocationScene.native.tsx index ed179ed895..d39906761f 100755 --- a/packages/client-core/src/components/World/LoadLocationScene.native.tsx +++ b/packages/client-core/src/components/World/LoadLocationScene.native.tsx @@ -71,12 +71,13 @@ export const useLoadLocation = (props: { locationName: string }) => { !locationState.currentLocation.location.sceneId.value || locationState.invalidLocation.value || locationState.currentLocation.selfNotAuthorized.value || - !scene + !locationState.currentLocation.location.sceneURL.value ) return - const sceneURL = scene.url - return GLTFAssetState.loadScene(sceneURL, scene.id) - }, [locationState.currentLocation.location.sceneId, scene]) + const sceneURL = locationState.currentLocation.location.sceneURL.value + const sceneID = locationState.currentLocation.location.sceneId.value + return GLTFAssetState.loadScene(sceneURL, sceneID) + }, [locationState.currentLocation.location.sceneId, locationState.currentLocation.location.sceneURL]) } export const useLoadScene = (props: { projectName: string; sceneName: string }) => { @@ -85,6 +86,12 @@ export const useLoadScene = (props: { projectName: string; sceneName: string }) const key = `${props.projectName}/${props.sceneName}` const url = getState(DomainConfigState).cloudDomain + `/projects/${key}` getMutableState(LocationState).currentLocation.location.sceneId.set(key) - return GLTFAssetState.loadScene(url, key) + getMutableState(LocationState).currentLocation.location.sceneURL.set(url) + const unload = GLTFAssetState.loadScene(url, key) + return () => { + getMutableState(LocationState).currentLocation.location.sceneId.set('') + getMutableState(LocationState).currentLocation.location.sceneURL.set('') + unload() + } }, []) } diff --git a/packages/client-core/src/networking/AvatarSpawnSystem.tsx b/packages/client-core/src/networking/AvatarSpawnSystem.tsx index eda321d018..ff2ed67bd6 100644 --- a/packages/client-core/src/networking/AvatarSpawnSystem.tsx +++ b/packages/client-core/src/networking/AvatarSpawnSystem.tsx @@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://github.com/ir-engine/ir-engine/blob/dev/LICENSE. The License is based on the Mozilla Public License Version 1.1, but Sections 14 -and 15 have been added to cover use of software over a computer network and -provide for limited attribution for the Original Developer. In addition, +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, Exhibit A has been modified to be consistent with Exhibit B. Software distributed under the License is distributed on an "AS IS" basis, @@ -19,13 +19,12 @@ The Original Code is Infinite Reality Engine. The Original Developer is the Initial Developer. The Initial Developer of the Original Code is the Infinite Reality Engine team. -All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 +All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 Infinite Reality Engine. All Rights Reserved. */ import React, { useEffect } from 'react' -import { getSearchParamFromURL } from '@ir-engine/common/src/utils/getSearchParamFromURL' import { spawnLocalAvatarInWorld } from '@ir-engine/common/src/world/receiveJoinWorld' import { defineSystem, @@ -74,7 +73,10 @@ export const AvatarSpawnReactor = (props: { sceneEntity: Entity }) => { useImmediateEffect(() => { const sceneSettingsSpectateEntity = getOptionalComponent(settingsQuery[0], SceneSettingsComponent)?.spectateEntity - spectateEntity.set(sceneSettingsSpectateEntity || (getSearchParamFromURL('spectate') as EntityUUID)) + // spectateEntity.set(sceneSettingsSpectateEntity || (getSearchParamFromURL('spectate') as EntityUUID)) + if (sceneSettingsSpectateEntity) { + spectateEntity.set(sceneSettingsSpectateEntity) + } }, [settingsQuery[0], searchParams.value['spectate']]) const isSpectating = typeof spectateEntity.value === 'string' diff --git a/packages/clientNative/App.tsx b/packages/clientNative/App.tsx index de52f2b45f..9eef417bda 100644 --- a/packages/clientNative/App.tsx +++ b/packages/clientNative/App.tsx @@ -34,7 +34,6 @@ Infinite Reality Engine. All Rights Reserved. // createHyperStore() import React, {Suspense, lazy, useEffect} from 'react'; import {SafeAreaView, View, Text} from 'react-native'; -import LocationRoutes from './src/pages/location/location'; import waitForClientAuthenticated from '@ir-engine/client-core/src/util/wait-for-client-authenticated'; import {pipeLogs} from '@ir-engine/common/src/logger'; import {API} from '@ir-engine/common'; diff --git a/packages/clientNative/ios/Podfile b/packages/clientNative/ios/Podfile index 3f246eb2e0..ea5c39c083 100644 --- a/packages/clientNative/ios/Podfile +++ b/packages/clientNative/ios/Podfile @@ -8,6 +8,29 @@ require Pod::Executable.execute_command('node', ['-p', {paths: [process.argv[1]]}, )', __dir__]).strip +$static_library = [ + 'react-native-basis-universal', +] + +pre_install do |installer| + Pod::Installer::Xcode::TargetValidator.send(:define_method, :verify_no_static_framework_transitive_dependencies) {} + + installer.pod_targets.each do |pod| + ## Skia pod correction + if $static_library.include?(pod.name) + puts "Overriding the build_type to static_library from static_framework for #{pod.name}" + def pod.build_type; + Pod::BuildType.static_library + end + end + + ## Firebase + bt = pod.send(:build_type) + puts "#{pod.name} (#{bt})" + puts " linkage: #{bt.send(:linkage)} packaging: #{bt.send(:packaging)}" + end +end + platform :ios, min_ios_version_supported prepare_react_native_project! diff --git a/packages/clientNative/ios/Podfile.lock b/packages/clientNative/ios/Podfile.lock index 01bed3f8f0..a852334875 100644 --- a/packages/clientNative/ios/Podfile.lock +++ b/packages/clientNative/ios/Podfile.lock @@ -1,13 +1,13 @@ PODS: - boost (1.84.0) - DoubleConversion (1.1.6) - - EXConstants (17.0.3): + - EXConstants (17.0.4): - ExpoModulesCore - Expo (52.0.8): - ExpoModulesCore - - ExpoAsset (11.0.1): + - ExpoAsset (11.0.2): - ExpoModulesCore - - ExpoFileSystem (18.0.4): + - ExpoFileSystem (18.0.7): - ExpoModulesCore - ExpoFont (13.0.1): - ExpoModulesCore @@ -1363,6 +1363,30 @@ PODS: - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - Yoga + - react-native-basis-universal (0.3.0): + - DoubleConversion + - glog + - hermes-engine + - RCT-Folly (= 2024.01.01.00) + - RCTRequired + - RCTTypeSafety + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-NativeModulesApple + - React-RCTFabric + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - Yoga + - react-native-blob-jsi-helper (0.3.1): + - React + - React-Core - react-native-draco (0.3.0): - DoubleConversion - glog @@ -1409,6 +1433,27 @@ PODS: - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - Yoga + - react-native-wgpu (0.1.22): + - DoubleConversion + - glog + - hermes-engine + - RCT-Folly (= 2024.01.01.00) + - RCTRequired + - RCTTypeSafety + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-NativeModulesApple + - React-RCTFabric + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - Yoga - React-nativeconfig (0.76.2) - React-NativeModulesApple (0.76.2): - glog @@ -1734,9 +1779,12 @@ DEPENDENCIES: - React-logger (from `../../../node_modules/react-native/ReactCommon/logger`) - React-Mapbuffer (from `../../../node_modules/react-native/ReactCommon`) - React-microtasksnativemodule (from `../../../node_modules/react-native/ReactCommon/react/nativemodule/microtasks`) + - "react-native-basis-universal (from `../../../node_modules/@callstack/react-native-basis-universal`)" + - react-native-blob-jsi-helper (from `../../../node_modules/react-native-blob-jsi-helper`) - "react-native-draco (from `../../../node_modules/@callstack/react-native-draco`)" - react-native-get-random-values (from `../../../node_modules/react-native-get-random-values`) - react-native-quick-crypto (from `../../../node_modules/react-native-quick-crypto`) + - react-native-wgpu (from `../../../node_modules/react-native-wgpu`) - React-nativeconfig (from `../../../node_modules/react-native/ReactCommon`) - React-NativeModulesApple (from `../../../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`) - React-perflogger (from `../../../node_modules/react-native/ReactCommon/reactperflogger`) @@ -1876,12 +1924,18 @@ EXTERNAL SOURCES: :path: "../../../node_modules/react-native/ReactCommon" React-microtasksnativemodule: :path: "../../../node_modules/react-native/ReactCommon/react/nativemodule/microtasks" + react-native-basis-universal: + :path: "../../../node_modules/@callstack/react-native-basis-universal" + react-native-blob-jsi-helper: + :path: "../../../node_modules/react-native-blob-jsi-helper" react-native-draco: :path: "../../../node_modules/@callstack/react-native-draco" react-native-get-random-values: :path: "../../../node_modules/react-native-get-random-values" react-native-quick-crypto: :path: "../../../node_modules/react-native-quick-crypto" + react-native-wgpu: + :path: "../../../node_modules/react-native-wgpu" React-nativeconfig: :path: "../../../node_modules/react-native/ReactCommon" React-NativeModulesApple: @@ -1946,10 +2000,10 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: boost: 1dca942403ed9342f98334bf4c3621f011aa7946 DoubleConversion: f16ae600a246532c4020132d54af21d0ddb2a385 - EXConstants: 277129d9a42ba2cf1fad375e7eaa9939005c60be + EXConstants: 6b8c5653492349b3c3fe6b905c556bc45b360405 Expo: 959ac1f7486354ee1d0e1c5332143b4203baf8e9 - ExpoAsset: 6f7a8887cbb9fb39fdb0808e7f6f74ba0e1ae9b6 - ExpoFileSystem: 83da9dbce2371cc72c3a3ef49e0df54a117310f1 + ExpoAsset: d2d2cbc6a4efadf51a3da27d85d589935abd0b98 + ExpoFileSystem: 818e82dbb71175414d1ca310e926c48ff0d07348 ExpoFont: 12b0217e42ac97029d0f317f0486039a8508cf52 ExpoGL: 9e4ac36b4dfeba548728f835c72fe895364ec3e7 ExpoKeepAwake: 22173f45d767c7d37403fdf48292726901d69ce7 @@ -2002,9 +2056,12 @@ SPEC CHECKSUMS: React-logger: addd140841248966c2547eb94836399cc1061f4d React-Mapbuffer: 1bc8e611871f4965dac0bc47a4561421a6e20f69 React-microtasksnativemodule: cff02bc87f8a1d5b9985c1c92ea8e84e854229d9 + react-native-basis-universal: a4f060a9b1118991a8a9c08157fb27b609861b3e + react-native-blob-jsi-helper: bd7509e50b0f906044c53ad7ab767786054424c9 react-native-draco: ca7c7ba9364c93a0a19f9197f66790d95abd852f react-native-get-random-values: d16467cf726c618e9c7a8c3c39c31faa2244bbba react-native-quick-crypto: 4921fde8af4321dd95df13963a80698c3a8fef34 + react-native-wgpu: a57435e58b0b5b94f5809d13414b02ce52f6cd91 React-nativeconfig: 7f8cd6cae21f8bb18c53b746c495e72795fc5cb0 React-NativeModulesApple: 3210b7177c11145bb8e0d6f24aae102a221c4ddc React-perflogger: c8860eaab4fe60d628b27bf0086a372c429fc74f @@ -2037,6 +2094,6 @@ SPEC CHECKSUMS: SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748 Yoga: b31c16dc8150acf90019a65ce2da4b47fab913cb -PODFILE CHECKSUM: 8bbb9ee26c0034dd601d94c137a968f3e9b5c0fe +PODFILE CHECKSUM: 4ac6aaeb919e40334eb4db2fd588dc1730d05e37 COCOAPODS: 1.15.2 diff --git a/packages/clientNative/ios/clientNative.xcodeproj/xcshareddata/xcschemes/clientNative.xcscheme b/packages/clientNative/ios/clientNative.xcodeproj/xcshareddata/xcschemes/clientNative.xcscheme index a5dc42384a..bd668084dc 100644 --- a/packages/clientNative/ios/clientNative.xcodeproj/xcshareddata/xcschemes/clientNative.xcscheme +++ b/packages/clientNative/ios/clientNative.xcodeproj/xcshareddata/xcschemes/clientNative.xcscheme @@ -49,6 +49,7 @@ ignoresPersistentStateOnLaunch = "NO" debugDocumentVersioning = "YES" debugServiceExtension = "internal" + enableGPUValidationMode = "1" allowLocationSimulation = "YES"> diff --git a/packages/clientNative/package.json b/packages/clientNative/package.json index e99d0ea325..1c4f7d4f0a 100644 --- a/packages/clientNative/package.json +++ b/packages/clientNative/package.json @@ -19,6 +19,7 @@ "react": "18.2.0" }, "dependencies": { + "@callstack/react-native-basis-universal": "^0.3.0", "@callstack/react-native-draco": "^0.3.0", "@expo/browser-polyfill": "^1.0.1", "@ir-engine/common": "^1.6.0", @@ -28,16 +29,21 @@ "@react-native-firebase/app": "^21.6.1", "@react-native-firebase/crashlytics": "^21.6.1", "@ungap/structured-clone": "^1.2.0", + "base-64": "^1.0.0", "expo": "^52.0.0", + "expo-asset": "^11.0.2", + "expo-asset-utils": "^3.0.0", "expo-gl": "^15.0.2", "expo-modules-core": "^2.0.4", "primus-client": "^7.3.4", "react": "18.2.0", "react-native": "0.76.2", + "react-native-blob-jsi-helper": "^0.3.1", "react-native-dotenv": "^3.4.11", "react-native-get-random-values": "^1.11.0", "react-native-nitro-modules": "^0.20.1", "react-native-quick-crypto": "^0.7.11", + "react-native-wgpu": "^0.1.22", "text-encoding-shim": "^1.0.5" }, "devDependencies": { @@ -52,6 +58,7 @@ "@react-native/metro-config": "0.76.2", "@react-native/typescript-config": "0.76.2", "@rnx-kit/metro-config": "^2.0.1", + "@types/base-64": "^1.0.2", "@types/react": "18.2.0", "@types/react-test-renderer": "18.0.0", "babel-jest": "^29.6.3", diff --git a/packages/clientNative/src/polyfill/NativeHTMLCanvasElement.ts b/packages/clientNative/src/polyfill/NativeHTMLCanvasElement.ts index 95ed938183..379291566c 100644 --- a/packages/clientNative/src/polyfill/NativeHTMLCanvasElement.ts +++ b/packages/clientNative/src/polyfill/NativeHTMLCanvasElement.ts @@ -54,4 +54,17 @@ export class NativeHTMLCanvasElement implements HTMLCanvasElement { } throw new Error(`Unsupported context: ${glContext}`); } + + public getBoundingClientRect() { + return { + x: 0, + y: 0, + left: 0, + top: 0, + right: this.width, + bottom: this.height, + width: this.width, + height: this.height, + }; + } } diff --git a/packages/engine/src/assets/classes/AssetLoader.ts b/packages/engine/src/assets/classes/AssetLoader.ts index 20150b77f1..00e7d26af9 100644 --- a/packages/engine/src/assets/classes/AssetLoader.ts +++ b/packages/engine/src/assets/classes/AssetLoader.ts @@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://github.com/ir-engine/ir-engine/blob/dev/LICENSE. The License is based on the Mozilla Public License Version 1.1, but Sections 14 -and 15 have been added to cover use of software over a computer network and -provide for limited attribution for the Original Developer. In addition, +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, Exhibit A has been modified to be consistent with Exhibit B. Software distributed under the License is distributed on an "AS IS" basis, @@ -19,7 +19,7 @@ The Original Code is Infinite Reality Engine. The Original Developer is the Initial Developer. The Initial Developer of the Original Code is the Infinite Reality Engine team. -All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 +All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 Infinite Reality Engine. All Rights Reserved. */ @@ -122,7 +122,7 @@ const loadAsset = async ( } try { - return loader.load(url, onLoad, onProgress, onError, signal) + return loader.load(url.replace('https', 'http'), onLoad, onProgress, onError, signal) } catch (error) { onError(error) } diff --git a/packages/engine/src/assets/constants/AssetType.ts b/packages/engine/src/assets/constants/AssetType.ts index ad8bce4d9f..3dffd37301 100644 --- a/packages/engine/src/assets/constants/AssetType.ts +++ b/packages/engine/src/assets/constants/AssetType.ts @@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://github.com/ir-engine/ir-engine/blob/dev/LICENSE. The License is based on the Mozilla Public License Version 1.1, but Sections 14 -and 15 have been added to cover use of software over a computer network and -provide for limited attribution for the Original Developer. In addition, +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, Exhibit A has been modified to be consistent with Exhibit B. Software distributed under the License is distributed on an "AS IS" basis, @@ -19,7 +19,7 @@ The Original Code is Infinite Reality Engine. The Original Developer is the Initial Developer. The Initial Developer of the Original Code is the Infinite Reality Engine team. -All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 +All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 Infinite Reality Engine. All Rights Reserved. */ @@ -131,11 +131,18 @@ export const FileExtToAssetExt = (fileExt: string): AssetExt | undefined => { return fileExt } +const getFileNameFromUrl = (path: string) => { + if (!global.RN$Bridgeless) { + const url = new URL(path) + return url.pathname.split('/').pop() as string + } + return path.split('/').pop() as string +} + const dataURLStart = 'data:image' export const FileToAssetExt = (file: string): AssetExt | undefined => { if (isURL(file)) { - const url = new URL(file) - file = url.pathname.split('/').pop() as string + file = getFileNameFromUrl(file) } // Check if image data url else if (file.startsWith(dataURLStart)) { @@ -155,8 +162,7 @@ export const FileToAssetType = (fileName: string): AssetType => { } if (isURL(fileName)) { - const url = new URL(fileName) - fileName = url.pathname.split('/').pop() as string + fileName = getFileNameFromUrl(fileName) } const split = fileName.split('.') diff --git a/packages/engine/src/assets/functions/createGLTFLoader.ts b/packages/engine/src/assets/functions/createGLTFLoader.ts index 6f2044d2a4..862a4286a8 100644 --- a/packages/engine/src/assets/functions/createGLTFLoader.ts +++ b/packages/engine/src/assets/functions/createGLTFLoader.ts @@ -48,6 +48,22 @@ export const initializeKTX2Loader = (loader: GLTFLoader) => { ktxLoader.setTranscoderPath(getState(DomainConfigState).publicDomain + '/loader_decoders/basis/') // FIXME: We are unable to spawn WebGLRenderer without Expo GL context. Is this required? if (global.RN$Bridgeless) { + ktxLoader.detectSupport({ + isWebGPURenderer: false, + hasFeature: () => false, + extensions: new Map([ + ['WEBGL_compressed_texture_astc', true], + ['WEBGL_compressed_texture_etc1', false], + ['WEBGL_compressed_texture_etc', true], + ['WEBGL_compressed_texture_s3tc', false], + ['EXT_texture_compression_bptc', false], + ['WEBGL_compressed_texture_pvrtc', true], + ['WEBKIT_WEBGL_compressed_texture_pvrtc', false] + ]), + capabilities: { + isWebGL2: true + } + }) loader.setKTX2Loader(ktxLoader) return } else { @@ -81,6 +97,7 @@ export const createGLTFLoader = (keepMaterials = false) => { // MeshoptDecoder.useWorkers(2) // } // loader.setMeshoptDecoder(MeshoptDecoder) + console.error('skip setting mesh optimizer decoder') // TODO: Detect React Native better if (global.RN$Bridgeless) { diff --git a/packages/engine/src/assets/loaders/gltf/KTX2Loader.native.ts b/packages/engine/src/assets/loaders/gltf/KTX2Loader.native.ts index dfa15748f9..e978d1a337 100644 --- a/packages/engine/src/assets/loaders/gltf/KTX2Loader.native.ts +++ b/packages/engine/src/assets/loaders/gltf/KTX2Loader.native.ts @@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://github.com/ir-engine/ir-engine/blob/dev/LICENSE. The License is based on the Mozilla Public License Version 1.1, but Sections 14 -and 15 have been added to cover use of software over a computer network and -provide for limited attribution for the Original Developer. In addition, +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, Exhibit A has been modified to be consistent with Exhibit B. Software distributed under the License is distributed on an "AS IS" basis, @@ -19,49 +19,704 @@ The Original Code is Infinite Reality Engine. The Original Developer is the Initial Developer. The Initial Developer of the Original Code is the Infinite Reality Engine team. -All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 +All portions of the code written by the Infinite Reality Engine team are Copyright � 2021-2023 Infinite Reality Engine. All Rights Reserved. */ -import { CompressedTexture, CompressedTextureLoader, LoadingManager, WebGLRenderer } from 'three' +/** + * Loader for KTX 2.0 GPU Texture containers. + * + * KTX 2.0 is a container format for various GPU texture formats. The loader + * supports Basis Universal GPU textures, which can be quickly transcoded to + * a wide variety of GPU texture compression formats. While KTX 2.0 also allows + * other hardware-specific formats, this loader does not yet parse them. + * + * References: + * - KTX: http://github.khronos.org/KTX-Specification/ + * - DFD: https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.html#basicdescriptor + */ -export class KTX2Loader extends CompressedTextureLoader { - constructor(manager?: LoadingManager) { +import * as BasisModule from '@callstack/react-native-basis-universal' +import { + CompressedArrayTexture, + CompressedCubeTexture, + CompressedTexture, + Data3DTexture, + DataTexture, + DisplayP3ColorSpace, + FloatType, + HalfFloatType, + LinearDisplayP3ColorSpace, + LinearFilter, + LinearMipmapLinearFilter, + LinearSRGBColorSpace, + NoColorSpace, + RGBAFormat, + RGBA_ASTC_4x4_Format, + RGBA_ASTC_6x6_Format, + RGBA_BPTC_Format, + RGBA_ETC2_EAC_Format, + RGBA_PVRTC_4BPPV1_Format, + RGBA_S3TC_DXT5_Format, + RGB_ETC1_Format, + RGB_ETC2_Format, + RGB_PVRTC_4BPPV1_Format, + RGB_S3TC_DXT1_Format, + RGFormat, + RedFormat, + SRGBColorSpace, + TypedArray, + UnsignedByteType +} from 'three' +import { FileLoader } from '../base/FileLoader' +import { Loader } from '../base/Loader' +import { + KHR_DF_FLAG_ALPHA_PREMULTIPLIED, + KHR_DF_PRIMARIES_BT709, + KHR_DF_PRIMARIES_DISPLAYP3, + KHR_DF_PRIMARIES_UNSPECIFIED, + KHR_DF_TRANSFER_SRGB, + KHR_SUPERCOMPRESSION_NONE, + KHR_SUPERCOMPRESSION_ZSTD, + VK_FORMAT_ASTC_6x6_SRGB_BLOCK, + VK_FORMAT_ASTC_6x6_UNORM_BLOCK, + VK_FORMAT_R16G16B16A16_SFLOAT, + VK_FORMAT_R16G16_SFLOAT, + VK_FORMAT_R16_SFLOAT, + VK_FORMAT_R32G32B32A32_SFLOAT, + VK_FORMAT_R32G32_SFLOAT, + VK_FORMAT_R32_SFLOAT, + VK_FORMAT_R8G8B8A8_SRGB, + VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_R8G8_SRGB, + VK_FORMAT_R8G8_UNORM, + VK_FORMAT_R8_SRGB, + VK_FORMAT_R8_UNORM, + VK_FORMAT_UNDEFINED, + read +} from './ktx-parse.module.js' +import { ZSTDDecoder } from './zstddec.module.js' + +const _taskCache = new WeakMap() + +let _activeLoaders = 0 + +let _zstd + +type BasisWorkerConfig = { + astcSupported: boolean + etc1Supported: boolean + etc2Supported: boolean + dxtSupported: boolean + bptcSupported: boolean + pvrtcSupported: boolean +} + +type MipMap = { + data: TypedArray + width?: number + height?: number + depth?: number +} + +type BasisWorker = { + init: (messageConfig: BasisWorkerConfig | null) => void + transcode: (buffer: TypedArray) => { + buffers: TypedArray[] + } +} + +class KTX2Loader extends Loader { + private basisWorker: BasisWorker + private workerConfig: BasisWorkerConfig | null + + constructor(manager) { super(manager) - console.warn('NATIVE KTX2 LOADER IS INITIALIZED') - return this + + this.workerConfig = null + + this.basisWorker = createBasisWorker() + + // if (typeof MSC_TRANSCODER !== 'undefined') { + // console.warn( + // 'THREE.KTX2Loader: Please update to latest "basis_transcoder".' + + // ' "msc_basis_transcoder" is no longer supported in three.js r125+.' + // ) + // } + console.warn('Initialized Native KTX2Loader') } - setTranscoderPath(path: string): KTX2Loader { + setTranscoderPath(path) { return this } - setWorkerLimit(limit: number): KTX2Loader { + + setWorkerLimit(num) { return this } - detectSupport(renderer: WebGLRenderer | null): KTX2Loader { + + detectSupport(renderer) { + if (renderer.isWebGPURenderer === true) { + this.workerConfig = { + astcSupported: renderer.hasFeature('texture-compression-astc'), + etc1Supported: false, + etc2Supported: renderer.hasFeature('texture-compression-etc2'), + dxtSupported: renderer.hasFeature('texture-compression-bc'), + bptcSupported: false, + pvrtcSupported: false + } + } else { + this.workerConfig = { + astcSupported: renderer.extensions.has('WEBGL_compressed_texture_astc'), + etc1Supported: renderer.extensions.has('WEBGL_compressed_texture_etc1'), + etc2Supported: renderer.extensions.has('WEBGL_compressed_texture_etc'), + dxtSupported: renderer.extensions.has('WEBGL_compressed_texture_s3tc'), + bptcSupported: renderer.extensions.has('EXT_texture_compression_bptc'), + pvrtcSupported: + renderer.extensions.has('WEBGL_compressed_texture_pvrtc') || + renderer.extensions.has('WEBKIT_WEBGL_compressed_texture_pvrtc') + } + + if (renderer.capabilities.isWebGL2) { + // https://github.com/mrdoob/three.js/pull/22928 + this.workerConfig.etc1Supported = false + } + } + return this } - dispose(): KTX2Loader { - return this + + init() { + this.basisWorker.init(this.workerConfig) + return Promise.resolve() + } + + load(url, onLoad, onProgress, onError, signal) { + if (this.workerConfig === null) { + throw new Error('THREE.KTX2Loader: Missing initialization with `.detectSupport( renderer )`.') + } + + const loader = new FileLoader(this.manager) + + loader.setResponseType('arraybuffer') + loader.setWithCredentials(this.withCredentials) + + loader.load( + url, + (buffer: any) => { + // Check for an existing task using this buffer. A transferred buffer cannot be transferred + // again from this thread. + if (_taskCache.has(buffer)) { + const cachedTask = _taskCache.get(buffer) + + return cachedTask.promise.then(onLoad).catch(onError) + } + + this._createTexture(buffer) + .then((texture) => (onLoad ? onLoad(texture) : null)) + .catch(onError) + }, + onProgress, + onError, + signal + ) + } + + _createTextureFrom(transcodeResult, container) { + const { faces, width, height, format, type, error, dfdFlags } = transcodeResult + + if (type === 'error') return Promise.reject(error) + + let texture + + if (container.faceCount === 6) { + texture = new CompressedCubeTexture(faces, format, UnsignedByteType) + } else { + const mipmaps = faces[0].mipmaps + + texture = + container.layerCount > 1 + ? new CompressedArrayTexture(mipmaps, width, height, container.layerCount, format, UnsignedByteType) + : new CompressedTexture(mipmaps, width, height, format, UnsignedByteType) + } + + texture.minFilter = faces[0].mipmaps.length === 1 ? LinearFilter : LinearMipmapLinearFilter + texture.magFilter = LinearFilter + texture.generateMipmaps = false + + texture.needsUpdate = true + texture.colorSpace = parseColorSpace(container) + texture.premultiplyAlpha = !!(dfdFlags & KHR_DF_FLAG_ALPHA_PREMULTIPLIED) + + return texture } - parse( - buffer: ArrayBuffer, - onLoad: (texture: CompressedTexture) => void, - onError?: (event: ErrorEvent) => void - ): KTX2Loader { - debugger + /** + * @param {ArrayBuffer} buffer + * @param {object?} config + * @return {Promise} + */ + async _createTexture(buffer, config = {}) { + const container = read(new Uint8Array(buffer)) + + if (container.vkFormat !== VK_FORMAT_UNDEFINED) { + return createRawTexture(container) + } + + const texturePending = this.init() + .then(() => { + return this.basisWorker.transcode(buffer) + }) + .then((e) => this._createTextureFrom(e, container)) + + // Cache the task result. + _taskCache.set(buffer, { promise: texturePending }) + + return texturePending + } + + dispose() { + _activeLoaders-- + return this } +} - load( - url: string, - onLoad: (texture: CompressedTexture) => void, - onProgress?: (requrest: ProgressEvent) => void | undefined, - onError?: ((event: ErrorEvent) => void) | undefined, - signal?: AbortSignal - ): CompressedTexture { - debugger - return null +/* CONSTANTS */ + +const _BasisFormat = { + ETC1S: 0, + UASTC_4x4: 1 +} + +const _TranscoderFormat = { + ETC1: 0, + ETC2: 1, + BC1: 2, + BC3: 3, + BC4: 4, + BC5: 5, + BC7_M6_OPAQUE_ONLY: 6, + BC7_M5: 7, + PVRTC1_4_RGB: 8, + PVRTC1_4_RGBA: 9, + ASTC_4x4: 10, + ATC_RGB: 11, + ATC_RGBA_INTERPOLATED_ALPHA: 12, + RGBA32: 13, + RGB565: 14, + BGR565: 15, + RGBA4444: 16 +} + +const _EngineFormat = { + RGBAFormat: RGBAFormat, + RGBA_ASTC_4x4_Format: RGBA_ASTC_4x4_Format, + RGBA_BPTC_Format: RGBA_BPTC_Format, + RGBA_ETC2_EAC_Format: RGBA_ETC2_EAC_Format, + RGBA_PVRTC_4BPPV1_Format: RGBA_PVRTC_4BPPV1_Format, + RGBA_S3TC_DXT5_Format: RGBA_S3TC_DXT5_Format, + RGB_ETC1_Format: RGB_ETC1_Format, + RGB_ETC2_Format: RGB_ETC2_Format, + RGB_PVRTC_4BPPV1_Format: RGB_PVRTC_4BPPV1_Format, + RGB_S3TC_DXT1_Format: RGB_S3TC_DXT1_Format +} + +/* WEB WORKER */ + +const createBasisWorker = (): BasisWorker => { + let config: BasisWorkerConfig | null = null + + const EngineFormat = _EngineFormat + const TranscoderFormat = _TranscoderFormat + const BasisFormat = _BasisFormat + + function init(messageConfig: BasisWorkerConfig | null) { + config = messageConfig + BasisModule.initializeBasis() } + + function transcode(buffer: TypedArray) { + const ktx2File = new BasisModule.KTX2File(new Uint8Array(buffer)) + + function cleanup() { + ktx2File.close() + ktx2File.delete() + } + + if (!ktx2File.isValid()) { + cleanup() + throw new Error('THREE.KTX2Loader: Invalid or unsupported .ktx2 file') + } + + const basisFormat = ktx2File.isUASTC() ? BasisFormat.UASTC_4x4 : BasisFormat.ETC1S + const width = ktx2File.getWidth() + const height = ktx2File.getHeight() + const layerCount = ktx2File.getLayers() || 1 + const levelCount = ktx2File.getLevels() + const faceCount = ktx2File.getFaces() + const hasAlpha = ktx2File.getHasAlpha() + const dfdFlags = ktx2File.getDFDFlags() + + const { transcoderFormat, engineFormat } = getTranscoderFormat(basisFormat, width, height, hasAlpha) + + if (!width || !height || !levelCount) { + cleanup() + throw new Error('THREE.KTX2Loader: Invalid texture') + } + + if (!ktx2File.startTranscoding()) { + cleanup() + throw new Error('THREE.KTX2Loader: .startTranscoding failed') + } + + const faces: { mipmaps: MipMap[]; width: number; height: number; format: any }[] = [] + const buffers: TypedArray[] = [] + + for (let face = 0; face < faceCount; face++) { + const mipmaps: MipMap[] = [] + + for (let mip = 0; mip < levelCount; mip++) { + const layerMips: Uint8Array[] = [] + + let mipWidth: number | undefined, mipHeight: number | undefined + + for (let layer = 0; layer < layerCount; layer++) { + const levelInfo = ktx2File.getImageLevelInfo(mip, layer, face) + + if ( + face === 0 && + mip === 0 && + layer === 0 && + (levelInfo.origWidth % 4 !== 0 || levelInfo.origHeight % 4 !== 0) + ) { + console.warn('THREE.KTX2Loader: ETC1S and UASTC textures should use multiple-of-four dimensions.') + } + + if (levelCount > 1) { + mipWidth = levelInfo.origWidth + mipHeight = levelInfo.origHeight + } else { + // Handles non-multiple-of-four dimensions in textures without mipmaps. Textures with + // mipmaps must use multiple-of-four dimensions, for some texture formats and APIs. + // See mrdoob/three.js#25908. + mipWidth = levelInfo.width + mipHeight = levelInfo.height + } + + const dst = new Uint8Array(ktx2File.getImageTranscodedSizeInBytes(mip, layer, 0, transcoderFormat)) + const status = ktx2File.transcodeImage(dst, mip, layer, face, transcoderFormat, 0, -1, -1) + + if (!status) { + cleanup() + throw new Error('THREE.KTX2Loader: .transcodeImage failed.') + } + + layerMips.push(dst) + } + + const mipData = concat(layerMips) + + mipmaps.push({ data: mipData, width: mipWidth, height: mipHeight }) + buffers.push(mipData.buffer) + } + + faces.push({ mipmaps, width, height, format: engineFormat }) + } + + cleanup() + + return { faces, buffers, width, height, hasAlpha, format: engineFormat, dfdFlags } + } + + // + + // Optimal choice of a transcoder target format depends on the Basis format (ETC1S or UASTC), + // device capabilities, and texture dimensions. The list below ranks the formats separately + // for ETC1S and UASTC. + // + // In some cases, transcoding UASTC to RGBA32 might be preferred for higher quality (at + // significant memory cost) compared to ETC1/2, BC1/3, and PVRTC. The transcoder currently + // chooses RGBA32 only as a last resort and does not expose that option to the caller. + const FORMAT_OPTIONS = [ + { + if: 'astcSupported', + basisFormat: [BasisFormat.UASTC_4x4], + transcoderFormat: [TranscoderFormat.ASTC_4x4, TranscoderFormat.ASTC_4x4], + engineFormat: [EngineFormat.RGBA_ASTC_4x4_Format, EngineFormat.RGBA_ASTC_4x4_Format], + priorityETC1S: Infinity, + priorityUASTC: 1, + needsPowerOfTwo: false + }, + { + if: 'bptcSupported', + basisFormat: [BasisFormat.ETC1S, BasisFormat.UASTC_4x4], + transcoderFormat: [TranscoderFormat.BC7_M5, TranscoderFormat.BC7_M5], + engineFormat: [EngineFormat.RGBA_BPTC_Format, EngineFormat.RGBA_BPTC_Format], + priorityETC1S: 3, + priorityUASTC: 2, + needsPowerOfTwo: false + }, + { + if: 'dxtSupported', + basisFormat: [BasisFormat.ETC1S, BasisFormat.UASTC_4x4], + transcoderFormat: [TranscoderFormat.BC1, TranscoderFormat.BC3], + engineFormat: [EngineFormat.RGB_S3TC_DXT1_Format, EngineFormat.RGBA_S3TC_DXT5_Format], + priorityETC1S: 4, + priorityUASTC: 5, + needsPowerOfTwo: false + }, + { + if: 'etc2Supported', + basisFormat: [BasisFormat.ETC1S, BasisFormat.UASTC_4x4], + transcoderFormat: [TranscoderFormat.ETC1, TranscoderFormat.ETC2], + engineFormat: [EngineFormat.RGB_ETC2_Format, EngineFormat.RGBA_ETC2_EAC_Format], + priorityETC1S: 1, + priorityUASTC: 3, + needsPowerOfTwo: false + }, + { + if: 'etc1Supported', + basisFormat: [BasisFormat.ETC1S, BasisFormat.UASTC_4x4], + transcoderFormat: [TranscoderFormat.ETC1], + engineFormat: [EngineFormat.RGB_ETC1_Format], + priorityETC1S: 2, + priorityUASTC: 4, + needsPowerOfTwo: false + }, + { + if: 'pvrtcSupported', + basisFormat: [BasisFormat.ETC1S, BasisFormat.UASTC_4x4], + transcoderFormat: [TranscoderFormat.PVRTC1_4_RGB, TranscoderFormat.PVRTC1_4_RGBA], + engineFormat: [EngineFormat.RGB_PVRTC_4BPPV1_Format, EngineFormat.RGBA_PVRTC_4BPPV1_Format], + priorityETC1S: 5, + priorityUASTC: 6, + needsPowerOfTwo: true + } + ] + + const ETC1S_OPTIONS = FORMAT_OPTIONS.sort(function (a, b) { + return a.priorityETC1S - b.priorityETC1S + }) + const UASTC_OPTIONS = FORMAT_OPTIONS.sort(function (a, b) { + return a.priorityUASTC - b.priorityUASTC + }) + + function getTranscoderFormat(basisFormat, width, height, hasAlpha) { + let transcoderFormat + let engineFormat + + const options = basisFormat === BasisFormat.ETC1S ? ETC1S_OPTIONS : UASTC_OPTIONS + + for (let i = 0; i < options.length; i++) { + const opt = options[i] + + if (!config?.[opt.if]) continue + if (!opt.basisFormat.includes(basisFormat)) continue + if (hasAlpha && opt.transcoderFormat.length < 2) continue + if (opt.needsPowerOfTwo && !(isPowerOfTwo(width) && isPowerOfTwo(height))) continue + + transcoderFormat = opt.transcoderFormat[hasAlpha ? 1 : 0] + engineFormat = opt.engineFormat[hasAlpha ? 1 : 0] + + return { transcoderFormat, engineFormat } + } + + console.warn('THREE.KTX2Loader: No suitable compressed texture format found. Decoding to RGBA32.') + + transcoderFormat = TranscoderFormat.RGBA32 + engineFormat = EngineFormat.RGBAFormat + + return { transcoderFormat, engineFormat } + } + + function isPowerOfTwo(value) { + if (value <= 2) return true + + return (value & (value - 1)) === 0 && value !== 0 + } + + /** Concatenates N byte arrays. */ + function concat(arrays) { + if (arrays.length === 1) return arrays[0] + + let totalByteLength = 0 + + for (let i = 0; i < arrays.length; i++) { + const array = arrays[i] + totalByteLength += array.byteLength + } + + const result = new Uint8Array(totalByteLength) + + let byteOffset = 0 + + for (let i = 0; i < arrays.length; i++) { + const array = arrays[i] + result.set(array, byteOffset) + + byteOffset += array.byteLength + } + + return result + } + + return { transcode, init } +} + +// +// Parsing for non-Basis textures. These textures are may have supercompression +// like Zstd, but they do not require transcoding. + +const UNCOMPRESSED_FORMATS = new Set([RGBAFormat, RGFormat, RedFormat]) + +const FORMAT_MAP = { + [VK_FORMAT_R32G32B32A32_SFLOAT]: RGBAFormat, + [VK_FORMAT_R16G16B16A16_SFLOAT]: RGBAFormat, + [VK_FORMAT_R8G8B8A8_UNORM]: RGBAFormat, + [VK_FORMAT_R8G8B8A8_SRGB]: RGBAFormat, + + [VK_FORMAT_R32G32_SFLOAT]: RGFormat, + [VK_FORMAT_R16G16_SFLOAT]: RGFormat, + [VK_FORMAT_R8G8_UNORM]: RGFormat, + [VK_FORMAT_R8G8_SRGB]: RGFormat, + + [VK_FORMAT_R32_SFLOAT]: RedFormat, + [VK_FORMAT_R16_SFLOAT]: RedFormat, + [VK_FORMAT_R8_SRGB]: RedFormat, + [VK_FORMAT_R8_UNORM]: RedFormat, + + [VK_FORMAT_ASTC_6x6_SRGB_BLOCK]: RGBA_ASTC_6x6_Format, + [VK_FORMAT_ASTC_6x6_UNORM_BLOCK]: RGBA_ASTC_6x6_Format } + +const TYPE_MAP = { + [VK_FORMAT_R32G32B32A32_SFLOAT]: FloatType, + [VK_FORMAT_R16G16B16A16_SFLOAT]: HalfFloatType, + [VK_FORMAT_R8G8B8A8_UNORM]: UnsignedByteType, + [VK_FORMAT_R8G8B8A8_SRGB]: UnsignedByteType, + + [VK_FORMAT_R32G32_SFLOAT]: FloatType, + [VK_FORMAT_R16G16_SFLOAT]: HalfFloatType, + [VK_FORMAT_R8G8_UNORM]: UnsignedByteType, + [VK_FORMAT_R8G8_SRGB]: UnsignedByteType, + + [VK_FORMAT_R32_SFLOAT]: FloatType, + [VK_FORMAT_R16_SFLOAT]: HalfFloatType, + [VK_FORMAT_R8_SRGB]: UnsignedByteType, + [VK_FORMAT_R8_UNORM]: UnsignedByteType, + + [VK_FORMAT_ASTC_6x6_SRGB_BLOCK]: UnsignedByteType, + [VK_FORMAT_ASTC_6x6_UNORM_BLOCK]: UnsignedByteType +} + +async function createRawTexture(container) { + const { vkFormat } = container + + if (FORMAT_MAP[vkFormat] === undefined) { + throw new Error('THREE.KTX2Loader: Unsupported vkFormat.') + } + + // + + let zstd + + if (container.supercompressionScheme === KHR_SUPERCOMPRESSION_ZSTD) { + if (!_zstd) { + _zstd = new Promise(async (resolve) => { + const zstd = new ZSTDDecoder() + await zstd.init() + resolve(zstd) + }) + } + + zstd = await _zstd + } + + // + + const mipmaps: MipMap[] = [] + + for (let levelIndex = 0; levelIndex < container.levels.length; levelIndex++) { + const levelWidth = Math.max(1, container.pixelWidth >> levelIndex) + const levelHeight = Math.max(1, container.pixelHeight >> levelIndex) + const levelDepth = container.pixelDepth ? Math.max(1, container.pixelDepth >> levelIndex) : 0 + + const level = container.levels[levelIndex] + + let levelData + + if (container.supercompressionScheme === KHR_SUPERCOMPRESSION_NONE) { + levelData = level.levelData + } else if (container.supercompressionScheme === KHR_SUPERCOMPRESSION_ZSTD) { + levelData = zstd.decode(level.levelData, level.uncompressedByteLength) + } else { + throw new Error('THREE.KTX2Loader: Unsupported supercompressionScheme.') + } + + let data + + if (TYPE_MAP[vkFormat] === FloatType) { + data = new Float32Array( + levelData.buffer, + levelData.byteOffset, + levelData.byteLength / Float32Array.BYTES_PER_ELEMENT + ) + } else if (TYPE_MAP[vkFormat] === HalfFloatType) { + data = new Uint16Array( + levelData.buffer, + levelData.byteOffset, + levelData.byteLength / Uint16Array.BYTES_PER_ELEMENT + ) + } else { + data = levelData + } + + mipmaps.push({ + data: data, + width: levelWidth, + height: levelHeight, + depth: levelDepth + }) + } + + let texture + + if (UNCOMPRESSED_FORMATS.has(FORMAT_MAP[vkFormat])) { + texture = + container.pixelDepth === 0 + ? new DataTexture(mipmaps[0].data, container.pixelWidth, container.pixelHeight) + : new Data3DTexture(mipmaps[0].data, container.pixelWidth, container.pixelHeight, container.pixelDepth) + } else { + if (container.pixelDepth > 0) throw new Error('THREE.KTX2Loader: Unsupported pixelDepth.') + + texture = new CompressedTexture(mipmaps as unknown as ImageData[], container.pixelWidth, container.pixelHeight) + } + + texture.mipmaps = mipmaps + + texture.type = TYPE_MAP[vkFormat] + texture.format = FORMAT_MAP[vkFormat] + texture.colorSpace = parseColorSpace(container) + texture.needsUpdate = true + + // + + return Promise.resolve(texture) +} + +function parseColorSpace(container) { + const dfd = container.dataFormatDescriptor[0] + + if (dfd.colorPrimaries === KHR_DF_PRIMARIES_BT709) { + return dfd.transferFunction === KHR_DF_TRANSFER_SRGB ? SRGBColorSpace : LinearSRGBColorSpace + } else if (dfd.colorPrimaries === KHR_DF_PRIMARIES_DISPLAYP3) { + return dfd.transferFunction === KHR_DF_TRANSFER_SRGB ? DisplayP3ColorSpace : LinearDisplayP3ColorSpace + } else if (dfd.colorPrimaries === KHR_DF_PRIMARIES_UNSPECIFIED) { + return NoColorSpace + } else { + console.warn(`THREE.KTX2Loader: Unsupported color primaries, "${dfd.colorPrimaries}"`) + return NoColorSpace + } +} + +export { KTX2Loader } diff --git a/packages/engine/src/assets/loaders/texture/TextureLoader.native.ts b/packages/engine/src/assets/loaders/texture/TextureLoader.native.ts new file mode 100644 index 0000000000..5220e19a4d --- /dev/null +++ b/packages/engine/src/assets/loaders/texture/TextureLoader.native.ts @@ -0,0 +1,337 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/ir-engine/ir-engine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Infinite Reality Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Infinite Reality Engine team. + +All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 +Infinite Reality Engine. All Rights Reserved. +*/ + +import { resolveAsync } from 'expo-asset-utils' +import { Image, Platform } from 'react-native' +import { GPUOffscreenCanvas } from 'react-native-wgpu' +import THREE, { DataTexture, LoadingManager, Texture } from 'three' +import { Loader } from '../base/Loader' + +const iOSMaxResolution = 1024 + +export class TextureLoader extends Loader { + maxResolution: number | undefined + autoDetectBitmap: boolean | undefined + + constructor(manager?: LoadingManager, autoDetectBitmap?: boolean, maxResolution?: number) { + super(manager) + if (maxResolution) this.maxResolution = maxResolution + else if (Platform.OS === 'ios') this.maxResolution = iOSMaxResolution + this.autoDetectBitmap = autoDetectBitmap + } + + override async load( + asset: any, + onLoad?: (texture: THREE.Texture) => void, + onProgress?: (event: ProgressEvent) => void, + onError?: (event: unknown) => void + ) { + if (!asset) { + throw new Error('ExpoTHREE.TextureLoader.load(): Cannot parse a null asset') + } + + const nativeAsset = await resolveAsync(asset) + + if (!nativeAsset.width || !nativeAsset.height) { + const { width, height } = await new Promise<{ + width: number + height: number + }>((res, rej) => { + Image.getSize(nativeAsset.localUri!, (width: number, height: number) => res({ width, height }), rej) + }) + nativeAsset.width = width + nativeAsset.height = height + } + + if (this.maxResolution && calculateResizingFactor(this.maxResolution, nativeAsset.height, nativeAsset.width) < 1) { + const data = await getScaledTextureData(nativeAsset.localUri!, this.maxResolution) + + try { + // TODO: why is the texture the wrong color? + const texture = new DataTexture(new Uint8Array(data.data), data.width, data.height) + texture.needsUpdate = true + console.log('success', texture.id) + + onLoad?.(texture) + } catch (err) { + console.error(err) + onError?.(err) + } + } else { + const texture = new THREE.Texture() + texture['isDataTexture'] = true // Forces passing to `gl.texImage2D(...)` verbatim + texture.image = { + data: nativeAsset, + width: nativeAsset.width, + height: nativeAsset.height + } + texture.needsUpdate = true + + if (onLoad !== undefined) { + onLoad(texture) + } + } + } +} + +const calculateResizingFactor = (maxResolution: number, originalHeight: number, originalWidth: number) => { + let resizingFactor = 1 + if (originalWidth >= originalHeight) { + if (originalWidth > maxResolution) { + resizingFactor = maxResolution / originalWidth + } + } else { + if (originalHeight > maxResolution) { + resizingFactor = maxResolution / originalHeight + } + } + return resizingFactor +} + +async function createTextureFromBase64(device: GPUDevice, imageURI: string) { + const response = await fetch(imageURI) + const blob = await response.blob() + + // Create ImageBitmap + const imageBitmap = await createImageBitmap(blob) + + // Create texture + const texture = device.createTexture({ + size: [imageBitmap.width, imageBitmap.height, 1], + format: 'rgba8unorm', + usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.RENDER_ATTACHMENT + }) + + device.queue.copyExternalImageToTexture({ source: imageBitmap }, { texture: texture }, [ + imageBitmap.width, + imageBitmap.height + ]) + + return texture +} + +const getScaledTextureData = async (imageURI: string, maxResolution: number) => { + const adapter = await navigator.gpu.requestAdapter() + if (!adapter) { + throw new Error('No adapter') + } + const device = await adapter.requestDevice() + + const canvas = new GPUOffscreenCanvas(maxResolution, maxResolution) + const context = canvas.getContext('webgpu') + if (!context) { + throw new Error('Could not get webgpu context') + } + + const presentationFormat = navigator.gpu.getPreferredCanvasFormat() + console.log(presentationFormat) + context?.configure({ + device: device!, + format: presentationFormat, + alphaMode: 'premultiplied' + }) + + const vertices = new Float32Array([ + // positions // texture coordinates + -1.0, + -1.0, + 0.0, + 1.0, // bottom left + 1.0, + -1.0, + 1.0, + 1.0, // bottom right + -1.0, + 1.0, + 0.0, + 0.0, // top left + 1.0, + 1.0, + 1.0, + 0.0 // top right + ]) + + const vertexBuffer = device.createBuffer({ + size: vertices.byteLength, + usage: GPUBufferUsage.VERTEX | GPUBufferUsage.COPY_DST, + mappedAtCreation: true + }) + new Float32Array(vertexBuffer.getMappedRange()).set(vertices) + vertexBuffer.unmap() + + // Load and create texture from base64 + + // Create sampler + const sampler = device.createSampler({ + magFilter: 'linear', + minFilter: 'linear', + mipmapFilter: 'linear' + }) + + // Create shader module + const shaderModule = device.createShaderModule({ + code: ` +struct VertexInput { +@location(0) position: vec2, +@location(1) texCoord: vec2, +}; + +struct VertexOutput { +@builtin(position) position: vec4, +@location(0) texCoord: vec2, +}; + +@vertex +fn vertexMain(input: VertexInput) -> VertexOutput { +var output: VertexOutput; +output.position = vec4(input.position, 0.0, 1.0); +output.texCoord = input.texCoord; +return output; +} + +@group(0) @binding(0) var texSampler: sampler; +@group(0) @binding(1) var tex: texture_2d; + +@fragment +fn fragmentMain(@location(0) texCoord: vec2) -> @location(0) vec4 { +return textureSample(tex, texSampler, texCoord); +} +` + }) + + const texture = await createTextureFromBase64(device, imageURI) + + // Create bind group layout + const bindGroupLayout = device.createBindGroupLayout({ + entries: [ + { + binding: 0, + visibility: GPUShaderStage.FRAGMENT, + sampler: { type: 'filtering' } + }, + { + binding: 1, + visibility: GPUShaderStage.FRAGMENT, + texture: { sampleType: 'float' } + } + ] + }) + + // Create pipeline layout + const pipelineLayout = device.createPipelineLayout({ + bindGroupLayouts: [bindGroupLayout] + }) + + // Create bind group + const bindGroup = device.createBindGroup({ + layout: bindGroupLayout, + entries: [ + { + binding: 0, + resource: sampler + }, + { + binding: 1, + resource: texture.createView() + } + ] + }) + + // Create render pipeline + const pipeline = device.createRenderPipeline({ + layout: pipelineLayout, + vertex: { + module: shaderModule, + entryPoint: 'vertexMain', + buffers: [ + { + arrayStride: 16, // 4 floats * 4 bytes + attributes: [ + { + shaderLocation: 0, + offset: 0, + format: 'float32x2' + }, + { + shaderLocation: 1, + offset: 8, + format: 'float32x2' + } + ] + } + ] + }, + fragment: { + module: shaderModule, + entryPoint: 'fragmentMain', + targets: [ + { + format: presentationFormat + } + ] + }, + primitive: { + topology: 'triangle-strip', + stripIndexFormat: 'uint32' + } + }) + + // Create command encoder and begin render pass + const commandEncoder = device.createCommandEncoder() + const textureView = context.getCurrentTexture().createView() + + const renderPassDescriptor = { + colorAttachments: [ + { + view: textureView, + clearValue: { r: 0.0, g: 0.0, b: 0.0, a: 1.0 }, + loadOp: 'clear', + storeOp: 'store' + } + ] + } + + const passEncoder = commandEncoder.beginRenderPass(renderPassDescriptor) + passEncoder.setPipeline(pipeline) + passEncoder.setBindGroup(0, bindGroup) + passEncoder.setVertexBuffer(0, vertexBuffer) + passEncoder.draw(4, 1, 0, 0) + passEncoder.end() + + // Submit commands to GPU + device.queue.submit([commandEncoder.finish()]) + + const data = await canvas.getImageData() + swizzleBrgaToRgba(data.data) + return data +} + +const swizzleBrgaToRgba = (data: number[]) => { + for (let i = 0; i < data.length; i += 4) { + const temp = data[i] + data[i] = data[i + 2] + data[i + 2] = temp + } +} diff --git a/packages/engine/src/gltf/GLTFExtensions.ts b/packages/engine/src/gltf/GLTFExtensions.ts index 41f0d1e88e..f1c8ff4b6c 100644 --- a/packages/engine/src/gltf/GLTFExtensions.ts +++ b/packages/engine/src/gltf/GLTFExtensions.ts @@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. The License is based on the Mozilla Public License Version 1.1, but Sections 14 -and 15 have been added to cover use of software over a computer network and -provide for limited attribution for the Original Developer. In addition, +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, Exhibit A has been modified to be consistent with Exhibit B. Software distributed under the License is distributed on an "AS IS" basis, @@ -19,7 +19,7 @@ The Original Code is Ethereal Engine. The Original Developer is the Initial Developer. The Initial Developer of the Original Code is the Ethereal Engine team. -All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 Ethereal Engine. All Rights Reserved. */ @@ -129,6 +129,7 @@ export const EXT_MESHOPT_COMPRESSION = { const decoder = getState(AssetLoaderState).gltfLoader.meshoptDecoder if (!decoder || !decoder.supported) { if (json.extensionsRequired && json.extensionsRequired.indexOf(EXTENSIONS.EXT_MESHOPT_COMPRESSION) >= 0) { + console.error('THREE.GLTFLoader: setMeshoptDecoder must be called before loading compressed files') return reject('THREE.GLTFLoader: setMeshoptDecoder must be called before loading compressed files') } else { // Assumes that the extension is optional and that fallback buffer data is present diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 7ab15b93c0..aeaaf94417 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. The License is based on the Mozilla Public License Version 1.1, but Sections 14 -and 15 have been added to cover use of software over a computer network and -provide for limited attribution for the Original Developer. In addition, +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, Exhibit A has been modified to be consistent with Exhibit B. Software distributed under the License is distributed on an "AS IS" basis, @@ -19,7 +19,7 @@ The Original Code is Ethereal Engine. The Original Developer is the Initial Developer. The Initial Developer of the Original Code is the Ethereal Engine team. -All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 Ethereal Engine. All Rights Reserved. */ @@ -44,6 +44,7 @@ import { ResourceManager, ResourceType } from '@ir-engine/spatial/src/resources/ import { useReferencedResource } from '@ir-engine/spatial/src/resources/resourceHooks' import { traverseEntityNode } from '@ir-engine/spatial/src/transform/components/EntityTree' import { useEffect } from 'react' +import { getBlobForArrayBuffer } from 'react-native-blob-jsi-helper' import { AnimationClip, Bone, @@ -926,13 +927,25 @@ const useLoadImageSource = ( } } - if (bufferViewSourceURI && !global.RN$Bridgeless) { + if (bufferViewSourceURI) { isObjectURL = true - const blob = new Blob([bufferViewSourceURI], { type: sourceDef.mimeType }) - const url = URL.createObjectURL(blob) - sourceURI.set(url) + const blob = getBlobForArrayBuffer(bufferViewSourceURI) + + const fileReaderInstance = new FileReader() + fileReaderInstance.onload = () => { + const url = fileReaderInstance.result + if (typeof url === 'string') { + sourceURI.set(url.replace('data:blob', `data:${sourceDef.mimeType!}`)) + } + } + + fileReaderInstance.onerror = (err) => { + console.log('failed to load uri', fileReaderInstance.error) + } + + fileReaderInstance.readAsDataURL(blob) + return () => { - URL.revokeObjectURL(url) sourceURI.set('') } } diff --git a/packages/projects/default-project/public/scenes/sky-station.gltf b/packages/projects/default-project/public/scenes/sky-station.gltf index e29fa5109e..43a9652957 100644 --- a/packages/projects/default-project/public/scenes/sky-station.gltf +++ b/packages/projects/default-project/public/scenes/sky-station.gltf @@ -317,7 +317,7 @@ "weight": 1 }, "EE_envmap": { - "type": "Texture", + "type": "Skybox", "envMapTextureType": "Equirectangular", "envMapSourceColor": 1193046, "envMapSourceURL": "__$project$__/ir-engine/default-project/assets/sky_skybox.jpg", diff --git a/patches/expo-asset+11.0.2.patch b/patches/expo-asset+11.0.2.patch new file mode 100644 index 0000000000..c2097561b2 --- /dev/null +++ b/patches/expo-asset+11.0.2.patch @@ -0,0 +1,22 @@ +diff --git a/node_modules/expo-asset/build/AssetUris.js b/node_modules/expo-asset/build/AssetUris.js +index 54cc343..d5eb1d1 100644 +--- a/node_modules/expo-asset/build/AssetUris.js ++++ b/node_modules/expo-asset/build/AssetUris.js +@@ -1,16 +1,5 @@ + export function getFilename(url) { +- const { pathname, searchParams } = new URL(url, 'https://e'); +- // When attached to a dev server, we use `unstable_path` to represent the file path. This ensures +- // the file name is not canonicalized by the browser. +- // NOTE(EvanBacon): This is technically not tied to `__DEV__` as it's possible to use this while bundling in production +- // mode. +- if (__DEV__) { +- if (searchParams.has('unstable_path')) { +- const encodedFilePath = decodeURIComponent(searchParams.get('unstable_path')); +- return getBasename(encodedFilePath); +- } +- } +- return getBasename(pathname); ++ return getBasename(url); + } + function getBasename(pathname) { + return pathname.substring(pathname.lastIndexOf('/') + 1);