Implement Texture Loading (#35)

* Add PanResponder stub

* Stub CanvasEventHandler

* Add license

* EventHandler stub

* Add inspo

* Update gesture events

* Add Event Listener to canvas element

* Create CustomLocationPage for testing

* Render static template

* Black background

* Import input into CustomLocationPage

* Add logging

* Use pointer events instead of panresponder

* Basic boundingClientRect

* Add FlyControlSystem

* Start adding joystick

* Pass type to CanvasEventHandler

* Set target entity

* Tweak follow camera settings

* Update LocationPage

* Add pointer listen events

* Add controller back

* Revert render system changes

* Import the rest of the systems

* Add TODO

* Comment out unusable ClientModules

* use .native files instead of entirely new package

* Remove client-core-mobile

* Remove projects-mobile

* Clean up LocationRoutes reference

* revert engine change

* Realign with upstream

* Remove url search from AvatarSpawnSystem

* Add react native basis universal

* Start implementing KTX2 loader

* Add todos

* Import KTX2 loader

* Copy from original file

* remove nocheck

* remove init

* Replace worker code

* Fix types

* fix podfile

* add debug logs

* Add debug statements

* Remove RNBridgeless check

* texture loading fixes

* Clean up logging

* Fix asset type detection

* fix sky station skybox type

* add todos

* Get some materials loading

* Add URL polyfill

* expo asset patch

* add license

* clean up unused changes

* Add logging for missing meshopt decoder

* Begin texture scaling impl

* handle imports

* Extract createTextureFromImage

* Simplify texture loading

* Align with web API

* Fix texture color

* clean up TODOs

* Remove unused code

* Undo Touch changes for this branch

* Add back getClientBoundingRect

* duplicate AssetLoader code

* fix path stripping

packages/client-core/src/components/World/LoadLocationScene.native.tsx

@@ -71,12 +71,13 @@ export const useLoadLocation = (props: { locationName: string }) => {
       !locationState.currentLocation.location.sceneId.value ||
       locationState.invalidLocation.value ||
       locationState.currentLocation.selfNotAuthorized.value ||
-      !scene
+      !locationState.currentLocation.location.sceneURL.value
     )
       return
-    const sceneURL = scene.url
-    return GLTFAssetState.loadScene(sceneURL, scene.id)
-  }, [locationState.currentLocation.location.sceneId, scene])
+    const sceneURL = locationState.currentLocation.location.sceneURL.value
+    const sceneID = locationState.currentLocation.location.sceneId.value
+    return GLTFAssetState.loadScene(sceneURL, sceneID)
+  }, [locationState.currentLocation.location.sceneId, locationState.currentLocation.location.sceneURL])
 }
 
 export const useLoadScene = (props: { projectName: string; sceneName: string }) => {
@@ -85,6 +86,12 @@ export const useLoadScene = (props: { projectName: string; sceneName: string })
     const key = `${props.projectName}/${props.sceneName}`
     const url = getState(DomainConfigState).cloudDomain + `/projects/${key}`
     getMutableState(LocationState).currentLocation.location.sceneId.set(key)
-    return GLTFAssetState.loadScene(url, key)
+    getMutableState(LocationState).currentLocation.location.sceneURL.set(url)
+    const unload = GLTFAssetState.loadScene(url, key)
+    return () => {
+      getMutableState(LocationState).currentLocation.location.sceneId.set('')
+      getMutableState(LocationState).currentLocation.location.sceneURL.set('')
+      unload()
+    }
   }, [])
 }

packages/client-core/src/networking/AvatarSpawnSystem.tsx

@@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance
 with the License. You may obtain a copy of the License at
 https://github.com/ir-engine/ir-engine/blob/dev/LICENSE.
 The License is based on the Mozilla Public License Version 1.1, but Sections 14
-and 15 have been added to cover use of software over a computer network and 
-provide for limited attribution for the Original Developer. In addition, 
+and 15 have been added to cover use of software over a computer network and
+provide for limited attribution for the Original Developer. In addition,
 Exhibit A has been modified to be consistent with Exhibit B.
 
 Software distributed under the License is distributed on an "AS IS" basis,
@@ -19,13 +19,12 @@ The Original Code is Infinite Reality Engine.
 The Original Developer is the Initial Developer. The Initial Developer of the
 Original Code is the Infinite Reality Engine team.
 
-All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 
+All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023
 Infinite Reality Engine. All Rights Reserved.
 */
 
 import React, { useEffect } from 'react'
 
-import { getSearchParamFromURL } from '@ir-engine/common/src/utils/getSearchParamFromURL'
 import { spawnLocalAvatarInWorld } from '@ir-engine/common/src/world/receiveJoinWorld'
 import {
   defineSystem,
@@ -74,7 +73,10 @@ export const AvatarSpawnReactor = (props: { sceneEntity: Entity }) => {
 
   useImmediateEffect(() => {
     const sceneSettingsSpectateEntity = getOptionalComponent(settingsQuery[0], SceneSettingsComponent)?.spectateEntity
-    spectateEntity.set(sceneSettingsSpectateEntity || (getSearchParamFromURL('spectate') as EntityUUID))
+    // spectateEntity.set(sceneSettingsSpectateEntity || (getSearchParamFromURL('spectate') as EntityUUID))
+    if (sceneSettingsSpectateEntity) {
+      spectateEntity.set(sceneSettingsSpectateEntity)
+    }
   }, [settingsQuery[0], searchParams.value['spectate']])
 
   const isSpectating = typeof spectateEntity.value === 'string'

packages/clientNative/App.tsx

@@ -34,7 +34,6 @@ Infinite Reality Engine. All Rights Reserved.
 // createHyperStore()
 import React, {Suspense, lazy, useEffect} from 'react';
 import {SafeAreaView, View, Text} from 'react-native';
-import LocationRoutes from './src/pages/location/location';
 import waitForClientAuthenticated from '@ir-engine/client-core/src/util/wait-for-client-authenticated';
 import {pipeLogs} from '@ir-engine/common/src/logger';
 import {API} from '@ir-engine/common';

packages/clientNative/ios/Podfile

@@ -8,6 +8,29 @@ require Pod::Executable.execute_command('node', ['-p',
     {paths: [process.argv[1]]},
   )', __dir__]).strip
 
+$static_library = [
+  'react-native-basis-universal',
+]
+
+pre_install do |installer|
+  Pod::Installer::Xcode::TargetValidator.send(:define_method, :verify_no_static_framework_transitive_dependencies) {}
+
+  installer.pod_targets.each do |pod|
+    ## Skia pod correction
+    if $static_library.include?(pod.name)
+      puts "Overriding the build_type to static_library from static_framework for #{pod.name}"
+      def pod.build_type;
+        Pod::BuildType.static_library
+      end
+    end
+
+    ## Firebase
+    bt = pod.send(:build_type)
+    puts "#{pod.name} (#{bt})"
+    puts "  linkage: #{bt.send(:linkage)} packaging: #{bt.send(:packaging)}"
+  end
+end
+
 platform :ios, min_ios_version_supported
 prepare_react_native_project!
 

packages/clientNative/ios/Podfile.lock

@@ -1,13 +1,13 @@
 PODS:
   - boost (1.84.0)
   - DoubleConversion (1.1.6)
-  - EXConstants (17.0.3):
+  - EXConstants (17.0.4):
     - ExpoModulesCore
   - Expo (52.0.8):
     - ExpoModulesCore
-  - ExpoAsset (11.0.1):
+  - ExpoAsset (11.0.2):
     - ExpoModulesCore
-  - ExpoFileSystem (18.0.4):
+  - ExpoFileSystem (18.0.7):
     - ExpoModulesCore
   - ExpoFont (13.0.1):
     - ExpoModulesCore
@@ -1363,6 +1363,30 @@ PODS:
     - ReactCommon/turbomodule/bridging
     - ReactCommon/turbomodule/core
     - Yoga
+  - react-native-basis-universal (0.3.0):
+    - DoubleConversion
+    - glog
+    - hermes-engine
+    - RCT-Folly (= 2024.01.01.00)
+    - RCTRequired
+    - RCTTypeSafety
+    - React-Core
+    - React-debug
+    - React-Fabric
+    - React-featureflags
+    - React-graphics
+    - React-ImageManager
+    - React-NativeModulesApple
+    - React-RCTFabric
+    - React-rendererdebug
+    - React-utils
+    - ReactCodegen
+    - ReactCommon/turbomodule/bridging
+    - ReactCommon/turbomodule/core
+    - Yoga
+  - react-native-blob-jsi-helper (0.3.1):
+    - React
+    - React-Core
   - react-native-draco (0.3.0):
     - DoubleConversion
     - glog
@@ -1409,6 +1433,27 @@ PODS:
     - ReactCommon/turbomodule/bridging
     - ReactCommon/turbomodule/core
     - Yoga
+  - react-native-wgpu (0.1.22):
+    - DoubleConversion
+    - glog
+    - hermes-engine
+    - RCT-Folly (= 2024.01.01.00)
+    - RCTRequired
+    - RCTTypeSafety
+    - React-Core
+    - React-debug
+    - React-Fabric
+    - React-featureflags
+    - React-graphics
+    - React-ImageManager
+    - React-NativeModulesApple
+    - React-RCTFabric
+    - React-rendererdebug
+    - React-utils
+    - ReactCodegen
+    - ReactCommon/turbomodule/bridging
+    - ReactCommon/turbomodule/core
+    - Yoga
   - React-nativeconfig (0.76.2)
   - React-NativeModulesApple (0.76.2):
     - glog
@@ -1734,9 +1779,12 @@ DEPENDENCIES:
   - React-logger (from `../../../node_modules/react-native/ReactCommon/logger`)
   - React-Mapbuffer (from `../../../node_modules/react-native/ReactCommon`)
   - React-microtasksnativemodule (from `../../../node_modules/react-native/ReactCommon/react/nativemodule/microtasks`)
+  - "react-native-basis-universal (from `../../../node_modules/@callstack/react-native-basis-universal`)"
+  - react-native-blob-jsi-helper (from `../../../node_modules/react-native-blob-jsi-helper`)
   - "react-native-draco (from `../../../node_modules/@callstack/react-native-draco`)"
   - react-native-get-random-values (from `../../../node_modules/react-native-get-random-values`)
   - react-native-quick-crypto (from `../../../node_modules/react-native-quick-crypto`)
+  - react-native-wgpu (from `../../../node_modules/react-native-wgpu`)
   - React-nativeconfig (from `../../../node_modules/react-native/ReactCommon`)
   - React-NativeModulesApple (from `../../../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`)
   - React-perflogger (from `../../../node_modules/react-native/ReactCommon/reactperflogger`)
@@ -1876,12 +1924,18 @@ EXTERNAL SOURCES:
     :path: "../../../node_modules/react-native/ReactCommon"
   React-microtasksnativemodule:
     :path: "../../../node_modules/react-native/ReactCommon/react/nativemodule/microtasks"
+  react-native-basis-universal:
+    :path: "../../../node_modules/@callstack/react-native-basis-universal"
+  react-native-blob-jsi-helper:
+    :path: "../../../node_modules/react-native-blob-jsi-helper"
   react-native-draco:
     :path: "../../../node_modules/@callstack/react-native-draco"
   react-native-get-random-values:
     :path: "../../../node_modules/react-native-get-random-values"
   react-native-quick-crypto:
     :path: "../../../node_modules/react-native-quick-crypto"
+  react-native-wgpu:
+    :path: "../../../node_modules/react-native-wgpu"
   React-nativeconfig:
     :path: "../../../node_modules/react-native/ReactCommon"
   React-NativeModulesApple:
@@ -1946,10 +2000,10 @@ EXTERNAL SOURCES:
 SPEC CHECKSUMS:
   boost: 1dca942403ed9342f98334bf4c3621f011aa7946
   DoubleConversion: f16ae600a246532c4020132d54af21d0ddb2a385
-  EXConstants: 277129d9a42ba2cf1fad375e7eaa9939005c60be
+  EXConstants: 6b8c5653492349b3c3fe6b905c556bc45b360405
   Expo: 959ac1f7486354ee1d0e1c5332143b4203baf8e9
-  ExpoAsset: 6f7a8887cbb9fb39fdb0808e7f6f74ba0e1ae9b6
-  ExpoFileSystem: 83da9dbce2371cc72c3a3ef49e0df54a117310f1
+  ExpoAsset: d2d2cbc6a4efadf51a3da27d85d589935abd0b98
+  ExpoFileSystem: 818e82dbb71175414d1ca310e926c48ff0d07348
   ExpoFont: 12b0217e42ac97029d0f317f0486039a8508cf52
   ExpoGL: 9e4ac36b4dfeba548728f835c72fe895364ec3e7
   ExpoKeepAwake: 22173f45d767c7d37403fdf48292726901d69ce7
@@ -2002,9 +2056,12 @@ SPEC CHECKSUMS:
   React-logger: addd140841248966c2547eb94836399cc1061f4d
   React-Mapbuffer: 1bc8e611871f4965dac0bc47a4561421a6e20f69
   React-microtasksnativemodule: cff02bc87f8a1d5b9985c1c92ea8e84e854229d9
+  react-native-basis-universal: a4f060a9b1118991a8a9c08157fb27b609861b3e
+  react-native-blob-jsi-helper: bd7509e50b0f906044c53ad7ab767786054424c9
   react-native-draco: ca7c7ba9364c93a0a19f9197f66790d95abd852f
   react-native-get-random-values: d16467cf726c618e9c7a8c3c39c31faa2244bbba
   react-native-quick-crypto: 4921fde8af4321dd95df13963a80698c3a8fef34
+  react-native-wgpu: a57435e58b0b5b94f5809d13414b02ce52f6cd91
   React-nativeconfig: 7f8cd6cae21f8bb18c53b746c495e72795fc5cb0
   React-NativeModulesApple: 3210b7177c11145bb8e0d6f24aae102a221c4ddc
   React-perflogger: c8860eaab4fe60d628b27bf0086a372c429fc74f
@@ -2037,6 +2094,6 @@ SPEC CHECKSUMS:
   SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748
   Yoga: b31c16dc8150acf90019a65ce2da4b47fab913cb
 
-PODFILE CHECKSUM: 8bbb9ee26c0034dd601d94c137a968f3e9b5c0fe
+PODFILE CHECKSUM: 4ac6aaeb919e40334eb4db2fd588dc1730d05e37
 
 COCOAPODS: 1.15.2

packages/clientNative/ios/clientNative.xcodeproj/xcshareddata/xcschemes/clientNative.xcscheme

@@ -49,6 +49,7 @@
       ignoresPersistentStateOnLaunch = "NO"
       debugDocumentVersioning = "YES"
       debugServiceExtension = "internal"
+      enableGPUValidationMode = "1"
       allowLocationSimulation = "YES">
       <BuildableProductRunnable
          runnableDebuggingMode = "0">

packages/clientNative/package.json

@@ -19,6 +19,7 @@
     "react": "18.2.0"
   },
   "dependencies": {
+    "@callstack/react-native-basis-universal": "^0.3.0",
     "@callstack/react-native-draco": "^0.3.0",
     "@expo/browser-polyfill": "^1.0.1",
     "@ir-engine/common": "^1.6.0",
@@ -28,16 +29,21 @@
     "@react-native-firebase/app": "^21.6.1",
     "@react-native-firebase/crashlytics": "^21.6.1",
     "@ungap/structured-clone": "^1.2.0",
+    "base-64": "^1.0.0",
     "expo": "^52.0.0",
+    "expo-asset": "^11.0.2",
+    "expo-asset-utils": "^3.0.0",
     "expo-gl": "^15.0.2",
     "expo-modules-core": "^2.0.4",
     "primus-client": "^7.3.4",
     "react": "18.2.0",
     "react-native": "0.76.2",
+    "react-native-blob-jsi-helper": "^0.3.1",
     "react-native-dotenv": "^3.4.11",
     "react-native-get-random-values": "^1.11.0",
     "react-native-nitro-modules": "^0.20.1",
     "react-native-quick-crypto": "^0.7.11",
+    "react-native-wgpu": "^0.1.22",
     "text-encoding-shim": "^1.0.5"
   },
   "devDependencies": {
@@ -52,6 +58,7 @@
     "@react-native/metro-config": "0.76.2",
     "@react-native/typescript-config": "0.76.2",
     "@rnx-kit/metro-config": "^2.0.1",
+    "@types/base-64": "^1.0.2",
     "@types/react": "18.2.0",
     "@types/react-test-renderer": "18.0.0",
     "babel-jest": "^29.6.3",

packages/clientNative/src/polyfill/NativeHTMLCanvasElement.ts

@@ -54,4 +54,17 @@ export class NativeHTMLCanvasElement implements HTMLCanvasElement {
     }
     throw new Error(`Unsupported context: ${glContext}`);
   }
+
+  public getBoundingClientRect() {
+    return {
+      x: 0,
+      y: 0,
+      left: 0,
+      top: 0,
+      right: this.width,
+      bottom: this.height,
+      width: this.width,
+      height: this.height,
+    };
+  }
 }

packages/engine/src/assets/classes/AssetLoader.ts

@@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance
 with the License. You may obtain a copy of the License at
 https://github.com/ir-engine/ir-engine/blob/dev/LICENSE.
 The License is based on the Mozilla Public License Version 1.1, but Sections 14
-and 15 have been added to cover use of software over a computer network and 
-provide for limited attribution for the Original Developer. In addition, 
+and 15 have been added to cover use of software over a computer network and
+provide for limited attribution for the Original Developer. In addition,
 Exhibit A has been modified to be consistent with Exhibit B.
 
 Software distributed under the License is distributed on an "AS IS" basis,
@@ -19,7 +19,7 @@ The Original Code is Infinite Reality Engine.
 The Original Developer is the Initial Developer. The Initial Developer of the
 Original Code is the Infinite Reality Engine team.
 
-All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 
+All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023
 Infinite Reality Engine. All Rights Reserved.
 */
 
@@ -122,7 +122,7 @@ const loadAsset = async <T>(
   }
 
   try {
-    return loader.load(url, onLoad, onProgress, onError, signal)
+    return loader.load(url.replace('https', 'http'), onLoad, onProgress, onError, signal)
   } catch (error) {
     onError(error)
   }

packages/engine/src/assets/constants/AssetType.ts

@@ -6,8 +6,8 @@ Version 1.0. (the "License"); you may not use this file except in compliance
 with the License. You may obtain a copy of the License at
 https://github.com/ir-engine/ir-engine/blob/dev/LICENSE.
 The License is based on the Mozilla Public License Version 1.1, but Sections 14
-and 15 have been added to cover use of software over a computer network and 
-provide for limited attribution for the Original Developer. In addition, 
+and 15 have been added to cover use of software over a computer network and
+provide for limited attribution for the Original Developer. In addition,
 Exhibit A has been modified to be consistent with Exhibit B.
 
 Software distributed under the License is distributed on an "AS IS" basis,
@@ -19,7 +19,7 @@ The Original Code is Infinite Reality Engine.
 The Original Developer is the Initial Developer. The Initial Developer of the
 Original Code is the Infinite Reality Engine team.
 
-All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023 
+All portions of the code written by the Infinite Reality Engine team are Copyright © 2021-2023
 Infinite Reality Engine. All Rights Reserved.
 */
 
@@ -131,11 +131,18 @@ export const FileExtToAssetExt = (fileExt: string): AssetExt | undefined => {
   return <AssetExt>fileExt
 }
 
+const getFileNameFromUrl = (path: string) => {
+  if (!global.RN$Bridgeless) {
+    const url = new URL(path)
+    return url.pathname.split('/').pop() as string
+  }
+  return path.split('/').pop() as string
+}
+
 const dataURLStart = 'data:image'
 export const FileToAssetExt = (file: string): AssetExt | undefined => {
   if (isURL(file)) {
-    const url = new URL(file)
-    file = url.pathname.split('/').pop() as string
+    file = getFileNameFromUrl(file)
   }
   // Check if image data url
   else if (file.startsWith(dataURLStart)) {
@@ -155,8 +162,7 @@ export const FileToAssetType = (fileName: string): AssetType => {
   }
 
   if (isURL(fileName)) {
-    const url = new URL(fileName)
-    fileName = url.pathname.split('/').pop() as string
+    fileName = getFileNameFromUrl(fileName)
   }
 
   const split = fileName.split('.')