Merge pull request #696 from LuxDL/ap/more_boltz_updates

More API updates
LuxDL · Jun 11, 2024 · edda7ec · edda7ec · github-actions · Jun 11, 2024
2 parents f7f4551 + 006b710
commit edda7ec
Show file tree

Hide file tree

Showing 19 changed files with 97 additions and 123 deletions.
diff --git a/docs/Project.toml b/docs/Project.toml
@@ -30,6 +30,6 @@ WeightInitializers = "d49dbf32-c5c2-4618-8acc-27bb2598ef2d"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [compat]
-Boltz = "0.3.5"
+Boltz = "0.3.7"
 DocumenterVitepress = "0.0.19"
 LuxDeviceUtils = "0.1.21"
diff --git a/docs/make.jl b/docs/make.jl
@@ -73,7 +73,10 @@ pages = [
             "api/Building_Blocks/WeightInitializers.md"
         ],
         "Domain Specific Modeling" => [
-            "api/Domain_Specific_Modeling/Boltz.md"
+            "api/Domain_Specific_Modeling/Boltz.md",
+            "api/Domain_Specific_Modeling/Boltz_Layers.md",
+            "api/Domain_Specific_Modeling/Boltz_Vision.md",
+            "api/Domain_Specific_Modeling/Boltz_Private.md"
         ],
         "Testing Functionality" => [
             "api/Testing_Functionality/LuxTestUtils.md"

diff --git a/docs/src/.vitepress/config.mts b/docs/src/.vitepress/config.mts
@@ -206,7 +206,9 @@ export default defineConfig({
                     },
                     {
                         text: 'Domain Specific Modeling', collapsed: false, items: [
-                            { text: 'Boltz', link: '/api/Domain_Specific_Modeling/Boltz' }]
+                            { text: 'Boltz', link: '/api/Domain_Specific_Modeling/Boltz' },
+                            { text: 'Computer Vision', link: '/api/Domain_Specific_Modeling/Boltz_Vision' },
+                            { text: 'Boltz Layers', link: '/api/Domain_Specific_Modeling/Boltz_Layers' }]
                     },
                     {
                         text: 'Testing Functionality', collapsed: false, items: [

diff --git a/docs/src/api/Accelerator_Support/LuxAMDGPU.md b/docs/src/api/Accelerator_Support/LuxAMDGPU.md
@@ -1,7 +1,3 @@
-```@meta
-CurrentModule = LuxAMDGPU
-```
-
 # LuxAMDGPU
 
 `LuxAMDGPU` is meant to be used as a trigger package for all `AMDGPU` dependencies in `Lux`.

diff --git a/docs/src/api/Accelerator_Support/LuxCUDA.md b/docs/src/api/Accelerator_Support/LuxCUDA.md
@@ -1,7 +1,3 @@
-```@meta
-CurrentModule = LuxCUDA
-```
-
 # LuxCUDA
 
 `LuxCUDA` is meant to be used as a trigger package for all `CUDA` dependencies in `Lux`.

diff --git a/docs/src/api/Accelerator_Support/LuxDeviceUtils.md b/docs/src/api/Accelerator_Support/LuxDeviceUtils.md
@@ -1,7 +1,3 @@
-```@meta
-CurrentModule = LuxDeviceUtils
-```
-
 # [LuxDeviceUtils](@id LuxDeviceUtils-API)
 
 `LuxDeviceUtils.jl` is a lightweight package defining rules for transferring data across

diff --git a/docs/src/api/Building_Blocks/LuxCore.md b/docs/src/api/Building_Blocks/LuxCore.md
@@ -1,7 +1,3 @@
-```@meta
-CurrentModule = Lux
-```
-
 # LuxCore
 
 `LuxCore.jl` defines the abstract layers for Lux. Allows users to be compatible with the

diff --git a/docs/src/api/Building_Blocks/LuxLib.md b/docs/src/api/Building_Blocks/LuxLib.md
@@ -2,10 +2,6 @@
 
 Backend for Lux.jl
 
-```@meta
-CurrentModule = LuxLib
-```
-
 ## Index
 
 ```@index

diff --git a/docs/src/api/Building_Blocks/WeightInitializers.md b/docs/src/api/Building_Blocks/WeightInitializers.md
@@ -1,7 +1,3 @@
-```@meta
-CurrentModule = WeightInitializers
-```
-
 # [WeightInitializers](@id WeightInitializers-API)
 
 This package is a light dependency providing common weight initialization schemes for deep

diff --git a/docs/src/api/Domain_Specific_Modeling/Boltz.md b/docs/src/api/Domain_Specific_Modeling/Boltz.md
@@ -1,80 +1,9 @@
-```@meta
-CurrentModule = Boltz
-```
-
 # Boltz
 
 Accelerate ⚡ your ML research using pre-built Deep Learning Models with Lux.
 
 ## Index
 
 ```@index
-Pages = ["Boltz.md"]
-```
-
-## `Layers` API
-
-```@docs
-Layers.ConvBatchNormActivation
-Layers.ClassTokens
-Layers.MultiHeadSelfAttention
-Layers.ViPosEmbedding
-Layers.VisionTransformerEncoder
-```
-
-## Computer Vision Models (`Vision` API)
-
-### Native Lux Models
-
-```@docs
-Vision.VGG
-Vision.VisionTransformer
-```
-
-### Imported from Metalhead.jl
-
-!!! tip
-
-    You need to load `Flux` and `Metalhead` before using these models.
-
-```@docs
-Vision.AlexNet
-Vision.ConvMixer
-Vision.DenseNet
-Vision.GoogLeNet
-Vision.MobileNet
-Vision.ResNet
-Vision.ResNeXt
-```
-
-### Pretrained Models
-
-!!! tip
-
-    Pass `pretrained=true` to the model constructor to load the pretrained weights.
-
-
-| MODEL                     | TOP 1 ACCURACY (%) | TOP 5 ACCURACY (%) |
-| :------------------------ | :----------------: | :----------------: |
-| `AlexNet()`               |       54.48        |       77.72        |
-| `VGG(11)`                 |       67.35        |       87.91        |
-| `VGG(13)`                 |       68.40        |       88.48        |
-| `VGG(16)`                 |       70.24        |       89.80        |
-| `VGG(19)`                 |       71.09        |       90.27        |
-| `VGG(11; batchnorm=true)` |       69.09        |       88.94        |
-| `VGG(13; batchnorm=true)` |       69.66        |       89.49        |
-| `VGG(16; batchnorm=true)` |       72.11        |       91.02        |
-| `VGG(19; batchnorm=true)` |       72.95        |       91.32        |
-
-#### Preprocessing
-
-All the pretrained models require that the images be normalized with the parameters
-`mean = [0.485f0, 0.456f0, 0.406f0]` and `std = [0.229f0, 0.224f0, 0.225f0]`.
-
-## Non-Public API
-
-```@docs
-Boltz._seconddimmean
-Boltz._fast_chunk
-Boltz._flatten_spatial
+Pages = ["Boltz.md", "Boltz_Layers.md", "Boltz_Vision.md"]
 ```
diff --git a/docs/src/api/Domain_Specific_Modeling/Boltz_Layers.md b/docs/src/api/Domain_Specific_Modeling/Boltz_Layers.md
@@ -0,0 +1,32 @@
+# `Boltz.Layers` and `Boltz.Basis` API Reference
+
+## `Layers` API
+
+```@docs
+Layers.ConvBatchNormActivation
+Layers.ConvNormActivation
+Layers.ClassTokens
+Layers.HamiltonianNN
+Layers.MultiHeadSelfAttention
+Layers.MLP
+Layers.TensorProductLayer
+Layers.ViPosEmbedding
+Layers.VisionTransformerEncoder
+```
+
+## Basis Functions
+
+!!! warning
+
+    The function calls for these basis functions should be considered experimental and are
+    subject to change without deprecation. However, the functions themselves are stable
+    and can be freely used in combination with the other Layers and Models.
+
+```@docs
+Basis.Cos
+Basis.Chebyshev
+Basis.Fourier
+Basis.Legendre
+Basis.Polynomial
+Basis.Sin
+```
diff --git a/docs/src/api/Domain_Specific_Modeling/Boltz_Private.md b/docs/src/api/Domain_Specific_Modeling/Boltz_Private.md
@@ -0,0 +1,8 @@
+# Boltz.jl Private API
+
+```@docs
+Boltz._seconddimmean
+Boltz._should_type_assert
+Boltz._fast_chunk
+Boltz._flatten_spatial
+```
diff --git a/docs/src/api/Domain_Specific_Modeling/Boltz_Vision.md b/docs/src/api/Domain_Specific_Modeling/Boltz_Vision.md
@@ -0,0 +1,48 @@
+# Computer Vision Models (`Vision` API)
+
+## Native Lux Models
+
+```@docs
+Vision.VGG
+Vision.VisionTransformer
+```
+
+## Imported from Metalhead.jl
+
+!!! tip
+
+    You need to load `Flux` and `Metalhead` before using these models.
+
+```@docs
+Vision.AlexNet
+Vision.ConvMixer
+Vision.DenseNet
+Vision.GoogLeNet
+Vision.MobileNet
+Vision.ResNet
+Vision.ResNeXt
+```
+
+## Pretrained Models
+
+!!! tip
+
+    Pass `pretrained=true` to the model constructor to load the pretrained weights.
+
+
+| MODEL                     | TOP 1 ACCURACY (%) | TOP 5 ACCURACY (%) |
+| :------------------------ | :----------------: | :----------------: |
+| `AlexNet()`               |       54.48        |       77.72        |
+| `VGG(11)`                 |       67.35        |       87.91        |
+| `VGG(13)`                 |       68.40        |       88.48        |
+| `VGG(16)`                 |       70.24        |       89.80        |
+| `VGG(19)`                 |       71.09        |       90.27        |
+| `VGG(11; batchnorm=true)` |       69.09        |       88.94        |
+| `VGG(13; batchnorm=true)` |       69.66        |       89.49        |
+| `VGG(16; batchnorm=true)` |       72.11        |       91.02        |
+| `VGG(19; batchnorm=true)` |       72.95        |       91.32        |
+
+### Preprocessing
+
+All the pretrained models require that the images be normalized with the parameters
+`mean = [0.485f0, 0.456f0, 0.406f0]` and `std = [0.229f0, 0.224f0, 0.225f0]`.
diff --git a/docs/src/api/Lux/autodiff.md b/docs/src/api/Lux/autodiff.md
@@ -4,10 +4,6 @@ Lux is not an AD package, but it composes well with most of the AD packages avai
 Julia ecosystem. This document lists the current level of support for various AD packages in
 Lux. Additionally, we provide some convenience functions for working with AD.
 
-```@meta
-CurrentModule = Lux
-```
-
 ## Overview
 
 | AD Package                                                         | CPU   | GPU   | Nested 2nd Order AD | Support Class |

diff --git a/docs/src/api/Lux/contrib.md b/docs/src/api/Lux/contrib.md
@@ -1,9 +1,5 @@
 # Experimental Features
 
-```@meta
-CurrentModule = Lux
-```
-
 All features listed on this page are **experimental** which means:
 
 1. No SemVer Guarantees. We use code here to iterate fast. That said, historically we have

diff --git a/docs/src/api/Lux/distributed_utils.md b/docs/src/api/Lux/distributed_utils.md
@@ -4,10 +4,6 @@
 
     These functionalities are available via the `Lux.DistributedUtils` module.
 
-```@meta
-CurrentModule = Lux
-```
-
 ## Index
 
 ```@index

diff --git a/docs/src/api/Lux/layers.md b/docs/src/api/Lux/layers.md
@@ -1,9 +1,5 @@
 # Built-In Layers
 
-```@meta
-CurrentModule = Lux
-```
-
 ## Index
 
 ```@index

diff --git a/docs/src/api/Lux/utilities.md b/docs/src/api/Lux/utilities.md
@@ -1,9 +1,5 @@
 # Utilities
 
-```@meta
-CurrentModule = Lux
-```
-
 ## Index
 
 ```@index

diff --git a/docs/src/api/Testing_Functionality/LuxTestUtils.md b/docs/src/api/Testing_Functionality/LuxTestUtils.md
@@ -1,7 +1,3 @@
-```@meta
-CurrentModule = LuxTestUtils
-```
-
 # LuxTestUtils
 
 !!! warning
Benchmark suite	Current: `edda7ec`	Previous: `f7f4551`	Ratio
`Dense(2 => 2)/cpu/reverse/ReverseDiff (compiled)/(2, 128)`	`3650.5` ns	`3700.75` ns	`0.99`
`Dense(2 => 2)/cpu/reverse/Zygote/(2, 128)`	`7205.4` ns	`7173.416666666666` ns	`1.00`
`Dense(2 => 2)/cpu/reverse/Tracker/(2, 128)`	`20689` ns	`21129` ns	`0.98`
`Dense(2 => 2)/cpu/reverse/ReverseDiff/(2, 128)`	`9711.1` ns	`9760.75` ns	`0.99`
`Dense(2 => 2)/cpu/reverse/Flux/(2, 128)`	`9014.8` ns	`9085` ns	`0.99`
`Dense(2 => 2)/cpu/reverse/SimpleChains/(2, 128)`	`4457.125` ns	`4523.5` ns	`0.99`
`Dense(2 => 2)/cpu/reverse/Enzyme/(2, 128)`	`1170.89453125` ns	`1158.1818181818182` ns	`1.01`
`Dense(2 => 2)/cpu/forward/NamedTuple/(2, 128)`	`1179.6129032258063` ns	`1117.313725490196` ns	`1.06`
`Dense(2 => 2)/cpu/forward/ComponentArray/(2, 128)`	`1174.5255474452554` ns	`1190.6204379562043` ns	`0.99`
`Dense(2 => 2)/cpu/forward/Flux/(2, 128)`	`1802.16` ns	`1784.6923076923076` ns	`1.01`
`Dense(2 => 2)/cpu/forward/SimpleChains/(2, 128)`	`180.28067700987307` ns	`180.12934631432546` ns	`1.00`
`Dense(20 => 20)/cpu/reverse/ReverseDiff (compiled)/(20, 128)`	`17292` ns	`17322` ns	`1.00`
`Dense(20 => 20)/cpu/reverse/Zygote/(20, 128)`	`16891` ns	`17012` ns	`0.99`
`Dense(20 => 20)/cpu/reverse/Tracker/(20, 128)`	`39424` ns	`39274` ns	`1.00`
`Dense(20 => 20)/cpu/reverse/ReverseDiff/(20, 128)`	`29575` ns	`29480` ns	`1.00`
`Dense(20 => 20)/cpu/reverse/Flux/(20, 128)`	`20247` ns	`21670.5` ns	`0.93`
`Dense(20 => 20)/cpu/reverse/SimpleChains/(20, 128)`	`17543` ns	`17322` ns	`1.01`
`Dense(20 => 20)/cpu/reverse/Enzyme/(20, 128)`	`4343.857142857143` ns	`4363.857142857143` ns	`1.00`
`Dense(20 => 20)/cpu/forward/NamedTuple/(20, 128)`	`3877.25` ns	`3877.125` ns	`1.00`
`Dense(20 => 20)/cpu/forward/ComponentArray/(20, 128)`	`3976.125` ns	`3962.375` ns	`1.00`
`Dense(20 => 20)/cpu/forward/Flux/(20, 128)`	`4979.285714285715` ns	`4940.571428571428` ns	`1.01`
`Dense(20 => 20)/cpu/forward/SimpleChains/(20, 128)`	`1666.2` ns	`1673.2` ns	`1.00`
`Conv((3, 3), 3 => 3)/cpu/reverse/ReverseDiff (compiled)/(64, 64, 3, 128)`	`38674535` ns	`49957369` ns	`0.77`
`Conv((3, 3), 3 => 3)/cpu/reverse/Zygote/(64, 64, 3, 128)`	`57855464` ns	`57581648` ns	`1.00`
`Conv((3, 3), 3 => 3)/cpu/reverse/Tracker/(64, 64, 3, 128)`	`76267517` ns	`112009767.5` ns	`0.68`
`Conv((3, 3), 3 => 3)/cpu/reverse/ReverseDiff/(64, 64, 3, 128)`	`83618661.5` ns	`106965049` ns	`0.78`
`Conv((3, 3), 3 => 3)/cpu/reverse/Flux/(64, 64, 3, 128)`	`72679934` ns	`105768801` ns	`0.69`
`Conv((3, 3), 3 => 3)/cpu/reverse/SimpleChains/(64, 64, 3, 128)`	`11680641.5` ns	`11747527` ns	`0.99`
`Conv((3, 3), 3 => 3)/cpu/reverse/Enzyme/(64, 64, 3, 128)`	`17856952` ns	`17741547` ns	`1.01`
`Conv((3, 3), 3 => 3)/cpu/forward/NamedTuple/(64, 64, 3, 128)`	`7013553` ns	`7001933.5` ns	`1.00`
`Conv((3, 3), 3 => 3)/cpu/forward/ComponentArray/(64, 64, 3, 128)`	`6977149.5` ns	`7000243.5` ns	`1.00`
`Conv((3, 3), 3 => 3)/cpu/forward/Flux/(64, 64, 3, 128)`	`10041170` ns	`18531441` ns	`0.54`
`Conv((3, 3), 3 => 3)/cpu/forward/SimpleChains/(64, 64, 3, 128)`	`6389008` ns	`6394361` ns	`1.00`
`vgg16/cpu/reverse/Zygote/(32, 32, 3, 16)`	`746726487` ns	`727979542` ns	`1.03`
`vgg16/cpu/reverse/Zygote/(32, 32, 3, 64)`	`2585135934` ns	`2579077526` ns	`1.00`
`vgg16/cpu/reverse/Zygote/(32, 32, 3, 2)`	`145337500` ns	`131352834.5` ns	`1.11`
`vgg16/cpu/reverse/Tracker/(32, 32, 3, 16)`	`855270478` ns	`969586753` ns	`0.88`
`vgg16/cpu/reverse/Tracker/(32, 32, 3, 64)`	`3032822955` ns	`3263278361` ns	`0.93`
`vgg16/cpu/reverse/Tracker/(32, 32, 3, 2)`	`191513330` ns	`226498384` ns	`0.85`
`vgg16/cpu/reverse/Flux/(32, 32, 3, 16)`	`628663470` ns	`873641593` ns	`0.72`
`vgg16/cpu/reverse/Flux/(32, 32, 3, 64)`	`2431719966` ns	`3046764407` ns	`0.80`
`vgg16/cpu/reverse/Flux/(32, 32, 3, 2)`	`128430302` ns	`131007272.5` ns	`0.98`
`vgg16/cpu/forward/NamedTuple/(32, 32, 3, 16)`	`174231960` ns	`174047082` ns	`1.00`
`vgg16/cpu/forward/NamedTuple/(32, 32, 3, 64)`	`644942314.5` ns	`645727264.5` ns	`1.00`
`vgg16/cpu/forward/NamedTuple/(32, 32, 3, 2)`	`45780988.5` ns	`45534194` ns	`1.01`
`vgg16/cpu/forward/ComponentArray/(32, 32, 3, 16)`	`164658201` ns	`164324255` ns	`1.00`
`vgg16/cpu/forward/ComponentArray/(32, 32, 3, 64)`	`633987268` ns	`642792641` ns	`0.99`
`vgg16/cpu/forward/ComponentArray/(32, 32, 3, 2)`	`30500114` ns	`29977065` ns	`1.02`
`vgg16/cpu/forward/Flux/(32, 32, 3, 16)`	`188572145` ns	`201738658` ns	`0.93`
`vgg16/cpu/forward/Flux/(32, 32, 3, 64)`	`724357861.5` ns	`918807028` ns	`0.79`
`vgg16/cpu/forward/Flux/(32, 32, 3, 2)`	`35976345` ns	`40275483` ns	`0.89`
`Conv((3, 3), 64 => 64)/cpu/reverse/ReverseDiff (compiled)/(64, 64, 64, 128)`	`1237030180` ns	`1293544406.5` ns	`0.96`
`Conv((3, 3), 64 => 64)/cpu/reverse/Zygote/(64, 64, 64, 128)`	`1865644001` ns	`1861728870` ns	`1.00`
`Conv((3, 3), 64 => 64)/cpu/reverse/Tracker/(64, 64, 64, 128)`	`2355646541` ns	`2468521111` ns	`0.95`
`Conv((3, 3), 64 => 64)/cpu/reverse/ReverseDiff/(64, 64, 64, 128)`	`2519641730` ns	`2631348316` ns	`0.96`
`Conv((3, 3), 64 => 64)/cpu/reverse/Flux/(64, 64, 64, 128)`	`1871255667` ns	`1895453908` ns	`0.99`
`Conv((3, 3), 64 => 64)/cpu/reverse/Enzyme/(64, 64, 64, 128)`	`559261598` ns	`565985350` ns	`0.99`
`Conv((3, 3), 64 => 64)/cpu/forward/NamedTuple/(64, 64, 64, 128)`	`321769270` ns	`321376035` ns	`1.00`
`Conv((3, 3), 64 => 64)/cpu/forward/ComponentArray/(64, 64, 64, 128)`	`322333048` ns	`320502632` ns	`1.01`
`Conv((3, 3), 64 => 64)/cpu/forward/Flux/(64, 64, 64, 128)`	`355818057.5` ns	`486344616` ns	`0.73`
`Conv((3, 3), 1 => 1)/cpu/reverse/ReverseDiff (compiled)/(64, 64, 1, 128)`	`11750378` ns	`11994505.5` ns	`0.98`
`Conv((3, 3), 1 => 1)/cpu/reverse/Zygote/(64, 64, 1, 128)`	`17892421` ns	`17906521` ns	`1.00`
`Conv((3, 3), 1 => 1)/cpu/reverse/Tracker/(64, 64, 1, 128)`	`19139788` ns	`19110550.5` ns	`1.00`
`Conv((3, 3), 1 => 1)/cpu/reverse/ReverseDiff/(64, 64, 1, 128)`	`23859946.5` ns	`23813093` ns	`1.00`
`Conv((3, 3), 1 => 1)/cpu/reverse/Flux/(64, 64, 1, 128)`	`17922065` ns	`17831210` ns	`1.01`
`Conv((3, 3), 1 => 1)/cpu/reverse/SimpleChains/(64, 64, 1, 128)`	`1162839` ns	`1157282` ns	`1.00`
`Conv((3, 3), 1 => 1)/cpu/reverse/Enzyme/(64, 64, 1, 128)`	`5778842` ns	`5758236.5` ns	`1.00`
`Conv((3, 3), 1 => 1)/cpu/forward/NamedTuple/(64, 64, 1, 128)`	`2053111` ns	`2047310.5` ns	`1.00`
`Conv((3, 3), 1 => 1)/cpu/forward/ComponentArray/(64, 64, 1, 128)`	`2034766` ns	`2030949` ns	`1.00`
`Conv((3, 3), 1 => 1)/cpu/forward/Flux/(64, 64, 1, 128)`	`2073890` ns	`2077663` ns	`1.00`
`Conv((3, 3), 1 => 1)/cpu/forward/SimpleChains/(64, 64, 1, 128)`	`197649` ns	`201106` ns	`0.98`
`Dense(200 => 200)/cpu/reverse/ReverseDiff (compiled)/(200, 128)`	`295075.5` ns	`293318` ns	`1.01`
`Dense(200 => 200)/cpu/reverse/Zygote/(200, 128)`	`267590` ns	`265260.5` ns	`1.01`
`Dense(200 => 200)/cpu/reverse/Tracker/(200, 128)`	`372184.5` ns	`366414` ns	`1.02`
`Dense(200 => 200)/cpu/reverse/ReverseDiff/(200, 128)`	`411758` ns	`408473` ns	`1.01`
`Dense(200 => 200)/cpu/reverse/Flux/(200, 128)`	`275684` ns	`273792` ns	`1.01`
`Dense(200 => 200)/cpu/reverse/SimpleChains/(200, 128)`	`410075` ns	`406058` ns	`1.01`
`Dense(200 => 200)/cpu/reverse/Enzyme/(200, 128)`	`83125` ns	`83516` ns	`1.00`
`Dense(200 => 200)/cpu/forward/NamedTuple/(200, 128)`	`81833` ns	`81513` ns	`1.00`
`Dense(200 => 200)/cpu/forward/ComponentArray/(200, 128)`	`83385` ns	`81592` ns	`1.02`
`Dense(200 => 200)/cpu/forward/Flux/(200, 128)`	`87153` ns	`86632` ns	`1.01`
`Dense(200 => 200)/cpu/forward/SimpleChains/(200, 128)`	`104926` ns	`104725` ns	`1.00`
`Conv((3, 3), 16 => 16)/cpu/reverse/ReverseDiff (compiled)/(64, 64, 16, 128)`	`195097943` ns	`199636923` ns	`0.98`
`Conv((3, 3), 16 => 16)/cpu/reverse/Zygote/(64, 64, 16, 128)`	`326683151.5` ns	`324677516.5` ns	`1.01`
`Conv((3, 3), 16 => 16)/cpu/reverse/Tracker/(64, 64, 16, 128)`	`383270904` ns	`437484335` ns	`0.88`
`Conv((3, 3), 16 => 16)/cpu/reverse/ReverseDiff/(64, 64, 16, 128)`	`456636762` ns	`526587672` ns	`0.87`
`Conv((3, 3), 16 => 16)/cpu/reverse/Flux/(64, 64, 16, 128)`	`370731185` ns	`410366601.5` ns	`0.90`
`Conv((3, 3), 16 => 16)/cpu/reverse/SimpleChains/(64, 64, 16, 128)`	`314203441.5` ns	`323744332.5` ns	`0.97`
`Conv((3, 3), 16 => 16)/cpu/reverse/Enzyme/(64, 64, 16, 128)`	`101771532` ns	`102515100.5` ns	`0.99`
`Conv((3, 3), 16 => 16)/cpu/forward/NamedTuple/(64, 64, 16, 128)`	`44354247.5` ns	`43799747` ns	`1.01`
`Conv((3, 3), 16 => 16)/cpu/forward/ComponentArray/(64, 64, 16, 128)`	`44239473.5` ns	`43534417` ns	`1.02`
`Conv((3, 3), 16 => 16)/cpu/forward/Flux/(64, 64, 16, 128)`	`60347251` ns	`63896063` ns	`0.94`
`Conv((3, 3), 16 => 16)/cpu/forward/SimpleChains/(64, 64, 16, 128)`	`28310724.5` ns	`28176419` ns	`1.00`
`Dense(2000 => 2000)/cpu/reverse/ReverseDiff (compiled)/(2000, 128)`	`18982153.5` ns	`18931882` ns	`1.00`
`Dense(2000 => 2000)/cpu/reverse/Zygote/(2000, 128)`	`19570435.5` ns	`19487991` ns	`1.00`
`Dense(2000 => 2000)/cpu/reverse/Tracker/(2000, 128)`	`23634115` ns	`23372438.5` ns	`1.01`
`Dense(2000 => 2000)/cpu/reverse/ReverseDiff/(2000, 128)`	`24235439` ns	`24063046.5` ns	`1.01`
`Dense(2000 => 2000)/cpu/reverse/Flux/(2000, 128)`	`19730324` ns	`19611168.5` ns	`1.01`
`Dense(2000 => 2000)/cpu/reverse/Enzyme/(2000, 128)`	`6543374` ns	`6520318` ns	`1.00`
`Dense(2000 => 2000)/cpu/forward/NamedTuple/(2000, 128)`	`6559820` ns	`6498792` ns	`1.01`
`Dense(2000 => 2000)/cpu/forward/ComponentArray/(2000, 128)`	`6519284` ns	`6477998.5` ns	`1.01`
`Dense(2000 => 2000)/cpu/forward/Flux/(2000, 128)`	`6512090` ns	`6511497` ns	`1.00`