+ /** \brief Set DynamicOptions for EPs (Execution Providers)
+ *
+ * Valid options can be found in {@code include\onnxruntime\core\session\onnxruntime_session_options_config_keys.h}
+ * Look for {@code kOrtEpDynamicOptions}
+ *
+ * @param sess [in] OrtSession
+ * @param keys [in] Array of null terminated UTF8 encoded strings of EP dynamic option keys
+ * @param values [in] Array of null terminated UTF8 encoded string of EP dynamic option values
+ * @param kv_len [in] Number of elements in the keys and values arrays
+ *
+ * \snippet{doc} snippets.dox OrtStatus Return Value
+ */
+ public native OrtStatus SetEpDynamicOptions( OrtSession sess, @Cast("const char*const*") PointerPointer keys,
+ @Cast("const char*const*") PointerPointer values, @Cast("size_t") long kv_len);
+ public native OrtStatus SetEpDynamicOptions( OrtSession sess, @Cast("const char*const*") @ByPtrPtr BytePointer keys,
+ @Cast("const char*const*") @ByPtrPtr BytePointer values, @Cast("size_t") long kv_len);
+ public native OrtStatus SetEpDynamicOptions( OrtSession sess, @Cast("const char*const*") @ByPtrPtr ByteBuffer keys,
+ @Cast("const char*const*") @ByPtrPtr ByteBuffer values, @Cast("size_t") long kv_len);
+ public native OrtStatus SetEpDynamicOptions( OrtSession sess, @Cast("const char*const*") @ByPtrPtr byte[] keys,
+ @Cast("const char*const*") @ByPtrPtr byte[] values, @Cast("size_t") long kv_len);
}
diff --git a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtLoraAdapter.java b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtLoraAdapter.java
new file mode 100644
index 0000000000..814d4bc1b8
--- /dev/null
+++ b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtLoraAdapter.java
@@ -0,0 +1,23 @@
+// Targeted by JavaCPP version 1.5.11-SNAPSHOT: DO NOT EDIT THIS FILE
+
+package org.bytedeco.onnxruntime;
+
+import java.nio.*;
+import org.bytedeco.javacpp.*;
+import org.bytedeco.javacpp.annotation.*;
+
+import static org.bytedeco.javacpp.presets.javacpp.*;
+import org.bytedeco.opencl.*;
+import static org.bytedeco.opencl.global.OpenCL.*;
+import org.bytedeco.dnnl.*;
+import static org.bytedeco.dnnl.global.dnnl.*;
+
+import static org.bytedeco.onnxruntime.global.onnxruntime.*;
+
+@Opaque @Properties(inherit = org.bytedeco.onnxruntime.presets.onnxruntime.class)
+public class OrtLoraAdapter extends Pointer {
+ /** Empty constructor. Calls {@code super((Pointer)null)}. */
+ public OrtLoraAdapter() { super((Pointer)null); }
+ /** Pointer cast constructor. Invokes {@link Pointer#Pointer(Pointer)}. */
+ public OrtLoraAdapter(Pointer p) { super(p); }
+}
diff --git a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtMIGraphXProviderOptions.java b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtMIGraphXProviderOptions.java
index 4a75c6b211..d63068551d 100644
--- a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtMIGraphXProviderOptions.java
+++ b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtMIGraphXProviderOptions.java
@@ -46,4 +46,5 @@ public class OrtMIGraphXProviderOptions extends Pointer {
public native @Cast("const char*") BytePointer migraphx_save_model_path(); public native OrtMIGraphXProviderOptions migraphx_save_model_path(BytePointer setter); // migraphx model path name
public native int migraphx_load_compiled_model(); public native OrtMIGraphXProviderOptions migraphx_load_compiled_model(int setter); // migraphx int8 cal table. Default 0 = false, noznero = true
public native @Cast("const char*") BytePointer migraphx_load_model_path(); public native OrtMIGraphXProviderOptions migraphx_load_model_path(BytePointer setter); // migraphx model path name
+ public native @Cast("bool") boolean migraphx_exhaustive_tune(); public native OrtMIGraphXProviderOptions migraphx_exhaustive_tune(boolean setter); // migraphx tuned compile Default = false
}
diff --git a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtOpenVINOProviderOptions.java b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtOpenVINOProviderOptions.java
index 433c1da949..ec48b1a88d 100644
--- a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtOpenVINOProviderOptions.java
+++ b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/OrtOpenVINOProviderOptions.java
@@ -43,7 +43,6 @@ public class OrtOpenVINOProviderOptions extends Pointer {
* Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16"
*/
public native @Cast("const char*") BytePointer device_type(); public native OrtOpenVINOProviderOptions device_type(BytePointer setter);
- /** 0 = disabled, nonzero = enabled */
public native @Cast("unsigned char") byte enable_npu_fast_compile(); public native OrtOpenVINOProviderOptions enable_npu_fast_compile(byte setter);
public native @Cast("const char*") BytePointer device_id(); public native OrtOpenVINOProviderOptions device_id(BytePointer setter);
/** 0 = Use default number of threads */
diff --git a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/RunOptions.java b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/RunOptions.java
index f4f08350d1..6e7add02d2 100644
--- a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/RunOptions.java
+++ b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/RunOptions.java
@@ -68,4 +68,12 @@ public class RunOptions extends BaseRunOptions {
* Wraps OrtApi::RunOptionsUnsetTerminate
*/
public native @ByRef RunOptions UnsetTerminate();
+
+ /** \brief Add the LoraAdapter to the list of active adapters.
+ * The setting does not affect RunWithBinding() calls.
+ *
+ * Wraps OrtApi::RunOptionsAddActiveLoraAdapter
+ * @param adapter The LoraAdapter to be used as the active adapter
+ */
+ public native @ByRef RunOptions AddActiveLoraAdapter(@Const @ByRef LoraAdapter adapter);
}
diff --git a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/SessionImpl.java b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/SessionImpl.java
index 37ecc69d02..9421b63b82 100644
--- a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/SessionImpl.java
+++ b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/SessionImpl.java
@@ -111,4 +111,20 @@ public native void RunAsync(@Const @ByRef RunOptions run_options, @Cast("const c
*/
/** Wraps OrtApi::SessionEndProfiling */
public native @UniquePtr("char, Ort::detail::AllocatedFree") @Cast("char*") BytePointer EndProfilingAllocated(OrtAllocator allocator);
+
+ /** \brief Set DynamicOptions for EPs (Execution Providers)
+ *
+ * Wraps OrtApi::SetEpDynamicOptions
+ *
+ * Valid options can be found in {@code include\onnxruntime\core\session\onnxruntime_session_options_config_keys.h}
+ * Look for {@code kOrtEpDynamicOptions}
+ *
+ * @param keys [in] Array of null terminated UTF8 encoded strings of EP dynamic option keys
+ * @param values [in] Array of null terminated UTF8 encoded string of EP dynamic option values
+ * @param kv_len [in] Number of elements in the keys and values arrays
+ */
+ public native void SetEpDynamicOptions(@Cast("const char*const*") PointerPointer keys, @Cast("const char*const*") PointerPointer values, @Cast("size_t") long kv_len);
+ public native void SetEpDynamicOptions(@Cast("const char*const*") @ByPtrPtr BytePointer keys, @Cast("const char*const*") @ByPtrPtr BytePointer values, @Cast("size_t") long kv_len);
+ public native void SetEpDynamicOptions(@Cast("const char*const*") @ByPtrPtr ByteBuffer keys, @Cast("const char*const*") @ByPtrPtr ByteBuffer values, @Cast("size_t") long kv_len);
+ public native void SetEpDynamicOptions(@Cast("const char*const*") @ByPtrPtr byte[] keys, @Cast("const char*const*") @ByPtrPtr byte[] values, @Cast("size_t") long kv_len);
}
diff --git a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/global/onnxruntime.java b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/global/onnxruntime.java
index 90c725945d..4fa658652f 100644
--- a/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/global/onnxruntime.java
+++ b/onnxruntime/src/gen/java/org/bytedeco/onnxruntime/global/onnxruntime.java
@@ -74,7 +74,7 @@ public class onnxruntime extends org.bytedeco.onnxruntime.presets.onnxruntime {
*
* This value is used by some API functions to behave as this version of the header expects.
*/
-public static final int ORT_API_VERSION = 19;
+public static final int ORT_API_VERSION = 20;
// #ifdef __cplusplus
// #endif
@@ -395,6 +395,9 @@ public enum OrtSparseIndicesFormat {
// Targeting ../OrtShapeInferContext.java
+// Targeting ../OrtLoraAdapter.java
+
+
// #ifdef _WIN32
// #else
@@ -736,6 +739,7 @@ public enum OrtSparseIndicesFormat {
@Namespace("Ort::detail") public static native void OrtRelease(OrtThreadingOptions ptr);
@Namespace("Ort::detail") public static native void OrtRelease(OrtEnv ptr);
@Namespace("Ort::detail") public static native void OrtRelease(OrtRunOptions ptr);
+@Namespace("Ort::detail") public static native void OrtRelease(OrtLoraAdapter ptr);
@Namespace("Ort::detail") public static native void OrtRelease(OrtSession ptr);
@Namespace("Ort::detail") public static native void OrtRelease(OrtSessionOptions ptr);
@Namespace("Ort::detail") public static native void OrtRelease(OrtTensorTypeAndShapeInfo ptr);
@@ -848,6 +852,9 @@ public enum OrtSparseIndicesFormat {
// Targeting ../BaseThreadingOptions.java
+// Targeting ../BaseOrtLoraAdapter.java
+
+
// Undefined. For const types use Base
*
@@ -824,730 +602,648 @@ public class nvinfer extends org.bytedeco.tensorrt.presets.nvinfer {
*
* \brief The TensorRT API version 1 namespace.
* */
+// Targeting ../nvinfer/Dims2.java
-/**
- * \brief PluginFormat is reserved for backward compatibility.
- *
- * @see IPluginV2::supportsFormat()
- * */
+// Targeting ../nvinfer/DimsHW.java
-//!
-//!
-/**
- * \brief Bit at the plugin version to identify that it is a plugin.
- * */
+// Targeting ../nvinfer/Dims3.java
-//!
-//!
-//!
-//!
-//!
-@Namespace("nvinfer1") @MemberGetter public static native int kPLUGIN_VERSION_PYTHON_BIT();
-public static final int kPLUGIN_VERSION_PYTHON_BIT = kPLUGIN_VERSION_PYTHON_BIT();
-// Targeting ../nvinfer/PluginTensorDesc.java
+// Targeting ../nvinfer/Dims4.java
-/**
- * \struct PluginVersion
- *
- * \brief Definition of plugin versions.
- *
- * Tag for plug-in versions. Used in upper byte of getTensorRTVersion().
- * */
-@Namespace("nvinfer1") public enum PluginVersion {
- /** IPluginV2 */
- kV2((byte)(0)),
- /** IPluginV2Ext */
- kV2_EXT((byte)(1)),
- /** IPluginV2IOExt */
- kV2_IOEXT((byte)(2)),
- /** IPluginV2DynamicExt */
- kV2_DYNAMICEXT((byte)(3)),
- /** IPluginV2DynamicExt-based Python plugins */
- kV2_DYNAMICEXT_PYTHON((byte)(kPLUGIN_VERSION_PYTHON_BIT | 3));
+ // namespace nvinfer1
- public final byte value;
- private PluginVersion(byte v) { this.value = v; }
- private PluginVersion(PluginVersion e) { this.value = e.value; }
- public PluginVersion intern() { for (PluginVersion e : values()) if (e.value == value) return e; return this; }
- @Override public String toString() { return intern().name(); }
-}
+// #endif // NV_INFER_LEGCY_DIMS_H
+
+
+// Parsed from NvInferRuntime.h
+
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// #ifndef NV_INFER_RUNTIME_H
+
+
+//!
+//!
+//!
+// #define NV_INFER_RUNTIME_H
/**
- * \enum PluginCreatorVersion
+ * \file NvInferRuntime.h
*
- * \brief Enum to identify version of the plugin creator.
+ * This is the top-level API file for TensorRT extended runtime library.
* */
-@Namespace("nvinfer1") public enum PluginCreatorVersion {
- /** IPluginCreator */
- kV1(0),
- /** IPluginCreator-based Python plugin creators */
- kV1_PYTHON(kPLUGIN_VERSION_PYTHON_BIT);
-
- public final int value;
- private PluginCreatorVersion(int v) { this.value = v; }
- private PluginCreatorVersion(PluginCreatorVersion e) { this.value = e.value; }
- public PluginCreatorVersion intern() { for (PluginCreatorVersion e : values()) if (e.value == value) return e; return this; }
- @Override public String toString() { return intern().name(); }
-}
-// Targeting ../nvinfer/IPluginV2.java
-
-// Targeting ../nvinfer/IPluginV2Ext.java
+// #include "NvInferImpl.h"
+// #include "NvInferPluginBase.h"
+// #undef NV_INFER_INTERNAL_INCLUDE
+// #include "NvInferRuntimeCommon.h"
+// Targeting ../nvinfer/IPluginFactory.java
-// Targeting ../nvinfer/IPluginV2IOExt.java
+// Targeting ../nvinfer/INoCopy.java
/**
- * \enum PluginFieldType
+ * \enum EngineCapability
*
- * \brief The possible field types for custom layer.
+ * \brief List of supported engine capability flows.
+ *
+ * \details The EngineCapability determines the restrictions of a network during build time and what runtime
+ * it targets. When BuilderFlag::kSAFETY_SCOPE is not set (by default), EngineCapability::kSTANDARD does not provide
+ * any restrictions on functionality and the resulting serialized engine can be executed with TensorRT's standard
+ * runtime APIs in the nvinfer1 namespace. EngineCapability::kSAFETY provides a restricted subset of network
+ * operations that are safety certified and the resulting serialized engine can be executed with TensorRT's safe
+ * runtime APIs in the nvinfer1::safe namespace. EngineCapability::kDLA_STANDALONE provides a restricted subset of
+ * network operations that are DLA compatible and the resulting serialized engine can be executed using standalone
+ * DLA runtime APIs. See sampleCudla for an example of integrating cuDLA APIs with TensorRT APIs.
* */
-@Namespace("nvinfer1") public enum PluginFieldType {
- /** FP16 field type. */
- kFLOAT16(0),
- /** FP32 field type. */
- kFLOAT32(1),
- /** FP64 field type. */
- kFLOAT64(2),
- /** INT8 field type. */
- kINT8(3),
- /** INT16 field type. */
- kINT16(4),
- /** INT32 field type. */
- kINT32(5),
- /** char field type. */
- kCHAR(6),
- /** nvinfer1::Dims field type. */
- kDIMS(7),
- /** Unknown field type. */
- kUNKNOWN(8),
- /** BF16 field type. */
- kBF16(9),
- /** INT64 field type. */
- kINT64(10),
- /** FP8 field type. */
- kFP8(11),
- /** INT4 field type. */
- kINT4(12);
+@Namespace("nvinfer1") public enum EngineCapability {
+ /**
+ * Standard: TensorRT flow without targeting the safety runtime.
+ * This flow supports both DeviceType::kGPU and DeviceType::kDLA.
+ * */
+
+
+//!
+//!
+ kSTANDARD(0),
+
+ /**
+ * Safety: TensorRT flow with restrictions targeting the safety runtime.
+ * See safety documentation for list of supported layers and formats.
+ * This flow supports only DeviceType::kGPU.
+ *
+ * This flag is only supported in NVIDIA Drive(R) products. */
+
+
+//!
+//!
+ kSAFETY(1),
+
+ /**
+ * DLA Standalone: TensorRT flow with restrictions targeting external, to TensorRT, DLA runtimes.
+ * See DLA documentation for list of supported layers and formats.
+ * This flow supports only DeviceType::kDLA.
+ * */
+ kDLA_STANDALONE(2);
public final int value;
- private PluginFieldType(int v) { this.value = v; }
- private PluginFieldType(PluginFieldType e) { this.value = e.value; }
- public PluginFieldType intern() { for (PluginFieldType e : values()) if (e.value == value) return e; return this; }
+ private EngineCapability(int v) { this.value = v; }
+ private EngineCapability(EngineCapability e) { this.value = e.value; }
+ public EngineCapability intern() { for (EngineCapability e : values()) if (e.value == value) return e; return this; }
@Override public String toString() { return intern().name(); }
}
-// Targeting ../nvinfer/PluginField.java
+/** Maximum number of elements in EngineCapability enum. @see EngineCapability */
+// Targeting ../nvinfer/Weights.java
-// Targeting ../nvinfer/PluginFieldCollection.java
+
+// Targeting ../nvinfer/IHostMemory.java
/**
- * \enum PluginCapabilityType
+ * \enum DimensionOperation
*
- * \brief Enumerates the different capability types a IPluginV3 object may have
- * */
-@Namespace("nvinfer1") public enum PluginCapabilityType {
- /** Core capability. Every IPluginV3 object must have this. */
- kCORE(0),
- /** Build capability. IPluginV3 objects provided to TensorRT build phase must have this. */
- kBUILD(1),
- /** Runtime capability. IPluginV3 objects provided to TensorRT build and execution phases must have this. */
- kRUNTIME(2);
+ * \brief An operation on two IDimensionExpr, which represent integer expressions used in dimension computations.
+ *
+ * For example, given two IDimensionExpr x and y and an IExprBuilder& eb,
+ * eb.operation(DimensionOperation::kSUM, x, y) creates a representation of x+y.
+ *
+ * @see IDimensionExpr, IExprBuilder
+ * */
+@Namespace("nvinfer1") public enum DimensionOperation {
+ /** Sum of the two operands. */
+ kSUM(0),
+ /** Product of the two operands. */
+ kPROD(1),
+ /** Maximum of the two operands. */
+ kMAX(2),
+ /** Minimum of the two operands. */
+ kMIN(3),
+ /** Substract the second element from the first. */
+ kSUB(4),
+ /** 1 if operands are equal, 0 otherwise. */
+ kEQUAL(5),
+ /** 1 if first operand is less than second operand, 0 otherwise. */
+ kLESS(6),
+ /** Floor division of the first element by the second. */
+ kFLOOR_DIV(7),
+ /** Division rounding up */
+ kCEIL_DIV(8);
public final int value;
- private PluginCapabilityType(int v) { this.value = v; }
- private PluginCapabilityType(PluginCapabilityType e) { this.value = e.value; }
- public PluginCapabilityType intern() { for (PluginCapabilityType e : values()) if (e.value == value) return e; return this; }
+ private DimensionOperation(int v) { this.value = v; }
+ private DimensionOperation(DimensionOperation e) { this.value = e.value; }
+ public DimensionOperation intern() { for (DimensionOperation e : values()) if (e.value == value) return e; return this; }
@Override public String toString() { return intern().name(); }
}
+/** Maximum number of elements in DimensionOperation enum. @see DimensionOperation */
+
+
/**
- * \enum TensorRTPhase
+ * \enum TensorLocation
*
- * \brief Indicates a phase of operation of TensorRT
+ * \brief The location for tensor data storage, device or host.
* */
-@Namespace("nvinfer1") public enum TensorRTPhase {
- /** Build phase of TensorRT */
- kBUILD(0),
- /** Execution phase of TensorRT */
- kRUNTIME(1);
+@Namespace("nvinfer1") public enum TensorLocation {
+ /** Data stored on device. */
+ kDEVICE(0),
+ /** Data stored on host. */
+ kHOST(1);
public final int value;
- private TensorRTPhase(int v) { this.value = v; }
- private TensorRTPhase(TensorRTPhase e) { this.value = e.value; }
- public TensorRTPhase intern() { for (TensorRTPhase e : values()) if (e.value == value) return e; return this; }
+ private TensorLocation(int v) { this.value = v; }
+ private TensorLocation(TensorLocation e) { this.value = e.value; }
+ public TensorLocation intern() { for (TensorLocation e : values()) if (e.value == value) return e; return this; }
@Override public String toString() { return intern().name(); }
}
-// Targeting ../nvinfer/IPluginCreatorInterface.java
+/** Maximum number of elements in TensorLocation enum. @see TensorLocation */
+// Targeting ../nvinfer/IDimensionExpr.java
-// Targeting ../nvinfer/IPluginCreator.java
+
+// Targeting ../nvinfer/IExprBuilder.java
+
+
+// Targeting ../nvinfer/DimsExprs.java
+
+
+// Targeting ../nvinfer/DynamicPluginTensorDesc.java
+
+
+// Targeting ../nvinfer/IPluginV2DynamicExt.java
+
+
+// Targeting ../nvinfer/IStreamReader.java
// namespace v_1_0
/**
- * \class IPluginCreatorInterface
+ * \class IStreamReader
*
- * \brief Base class for all plugin creator versions.
+ * \brief Application-implemented class for reading data in a stream-based manner.
*
- * @see IPluginCreator and IPluginRegistry
+ * \note To ensure compatibility of source code with future versions of TensorRT, use IStreamReader, not
+ * v_1_0::IStreamReader
* */
+
//!
//!
//!
//!
//!
+//!
+// Targeting ../nvinfer/IPluginResourceContext.java
-/**
- * \class IPluginCreator
- *
- * \brief Plugin creator class for user implemented layers.
- *
- * @see IPlugin and IPluginFactory
- *
- * @deprecated Deprecated in TensorRT 10.0. Please implement IPluginCreatorV3One instead along with IPluginV3 plugins
- * instead.
- * */
- // namespace nvinfer1
+// Targeting ../nvinfer/IPluginV3OneCore.java
-// #endif // NV_INFER_RUNTIME_PLUGIN_H
+// Targeting ../nvinfer/IPluginV3OneBuild.java
-// Parsed from NvInferRuntimeCommon.h
-/*
- * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+// Targeting ../nvinfer/IPluginV3OneRuntime.java
-// #ifndef NV_INFER_RUNTIME_COMMON_H
+
+ // namespace v_1_0
+
+ // namespace v_2_0
+
+/**
+ * \class IPluginV3OneCore
+ *
+ * \brief A plugin capability interface that enables the core capability (PluginCapabilityType::kCORE).
+ *
+ * @see IPluginCapability
+ * @see PluginCapabilityType
+ * @see IPluginV3::getCapabilityInterface()
+ * */
//!
//!
//!
//!
-//!
-// #define NV_INFER_RUNTIME_COMMON_H
/**
- * \file NvInferRuntimeCommon.h
- *
- * This file provides the nvinfer1::IPluginRegistry interface, which will be moved to the NvInferRuntime.h header
- * in a future release.
+ * \class IPluginV3OneBuild
*
- * \warning This file will be removed in a future release.
+ * \brief A plugin capability interface that enables the build capability (PluginCapabilityType::kBUILD). Exposes
+ * methods that allow the expression of the build time properties and behavior of a plugin.
*
- * \warning Do not directly include this file. Instead include NvInferRuntime.h
+ * @see IPluginCapability
+ * @see PluginCapabilityType
+ * @see IPluginV3::getCapabilityInterface()
* */
-// #include "NvInferRuntimeBase.h"
-// #undef NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE
-// #include "NvInferRuntimePlugin.h"
-// Targeting ../nvinfer/IPluginRegistry.java
-
-
-
- // namespace nvinfer1
-
-// #endif /* NV_INFER_RUNTIME_COMMON_H */
-
-
-// Parsed from NvInferLegacyDims.h
-
-/*
- * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// #ifndef NV_INFER_LEGACY_DIMS_H
-// #define NV_INFER_LEGACY_DIMS_H
-// #include "NvInferRuntimeBase.h"
//!
//!
-//!
-
//!
//!
-//!
-// #undef NV_INFER_INTERNAL_INCLUDE_RUNTIME_BASE
/**
- * \file NvInferLegacyDims.h
- *
- * This file contains declarations of legacy dimensions types which use channel
- * semantics in their names, and declarations on which those types rely.
- *
-
+ * \class IPluginV3OneRuntime
*
- * \namespace nvinfer1
+ * \brief A plugin capability interface that enables the runtime capability (PluginCapabilityType::kRUNTIME). Exposes
+ * methods that allow the expression of the runtime properties and behavior of a plugin.
*
- * \brief The TensorRT API version 1 namespace.
+ * @see IPluginCapability
+ * @see PluginCapabilityType
+ * @see IPluginV3::getCapabilityInterface()
* */
-// Targeting ../nvinfer/Dims2.java
-
-
-// Targeting ../nvinfer/DimsHW.java
-
-
-// Targeting ../nvinfer/Dims3.java
-
-
-// Targeting ../nvinfer/Dims4.java
-
-
-
- // namespace nvinfer1
-
-// #endif // NV_INFER_LEGCY_DIMS_H
-
-
-// Parsed from NvInferRuntime.h
-
-/*
- * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
- * SPDX-License-Identifier: Apache-2.0
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// #ifndef NV_INFER_RUNTIME_H
//!
//!
//!
-// #define NV_INFER_RUNTIME_H
+//!
/**
- * \file NvInferRuntime.h
+ * \class IPluginV3OneBuildV2
*
- * This is the top-level API file for TensorRT extended runtime library.
+ * \brief A plugin capability interface that extends IPluginV3OneBuild by providing I/O aliasing functionality.
+ *
+ * @see IPluginV3OneBuild
* */
-
-// #include "NvInferImpl.h"
-// #include "NvInferRuntimeCommon.h"
-// Targeting ../nvinfer/IPluginFactory.java
-
-
-// Targeting ../nvinfer/INoCopy.java
+// Targeting ../nvinfer/IProfiler.java
+ // namespace v_1_0
/**
- * \enum EngineCapability
+ * \class IProfiler
*
- * \brief List of supported engine capability flows.
+ * \brief Application-implemented interface for profiling.
*
- * \details The EngineCapability determines the restrictions of a network during build time and what runtime
- * it targets. When BuilderFlag::kSAFETY_SCOPE is not set (by default), EngineCapability::kSTANDARD does not provide
- * any restrictions on functionality and the resulting serialized engine can be executed with TensorRT's standard
- * runtime APIs in the nvinfer1 namespace. EngineCapability::kSAFETY provides a restricted subset of network
- * operations that are safety certified and the resulting serialized engine can be executed with TensorRT's safe
- * runtime APIs in the nvinfer1::safe namespace. EngineCapability::kDLA_STANDALONE provides a restricted subset of
- * network operations that are DLA compatible and the resulting serialized engine can be executed using standalone
- * DLA runtime APIs. See sampleCudla for an example of integrating cuDLA APIs with TensorRT APIs.
+ * When this class is added to an execution context, the profiler will be called once per layer for each invocation of
+ * executeV2()/enqueueV3().
+ *
+ * It is not recommended to run inference with profiler enabled when the inference execution time is critical since the
+ * profiler may affect execution time negatively.
* */
-@Namespace("nvinfer1") public enum EngineCapability {
- /**
- * Standard: TensorRT flow without targeting the safety runtime.
- * This flow supports both DeviceType::kGPU and DeviceType::kDLA.
- * */
-
+
//!
//!
- kSTANDARD(0),
-
- /**
- * Safety: TensorRT flow with restrictions targeting the safety runtime.
- * See safety documentation for list of supported layers and formats.
- * This flow supports only DeviceType::kGPU.
- *
- * This flag is only supported in NVIDIA Drive(R) products. */
-
-
//!
//!
- kSAFETY(1),
-
- /**
- * DLA Standalone: TensorRT flow with restrictions targeting external, to TensorRT, DLA runtimes.
- * See DLA documentation for list of supported layers and formats.
- * This flow supports only DeviceType::kDLA.
- * */
- kDLA_STANDALONE(2);
-
- public final int value;
- private EngineCapability(int v) { this.value = v; }
- private EngineCapability(EngineCapability e) { this.value = e.value; }
- public EngineCapability intern() { for (EngineCapability e : values()) if (e.value == value) return e; return this; }
- @Override public String toString() { return intern().name(); }
-}
-/** Maximum number of elements in EngineCapability enum. @see EngineCapability */
-
-// Targeting ../nvinfer/Weights.java
-
-
-// Targeting ../nvinfer/IHostMemory.java
-
-
/**
- * \enum DimensionOperation
- *
- * \brief An operation on two IDimensionExpr, which represent integer expressions used in dimension computations.
+ * \enum WeightsRole
*
- * For example, given two IDimensionExpr x and y and an IExprBuilder& eb,
- * eb.operation(DimensionOperation::kSUM, x, y) creates a representation of x+y.
+ * \brief How a layer uses particular Weights.
*
- * @see IDimensionExpr, IExprBuilder
+ * The power weights of an IScaleLayer are omitted. Refitting those is not supported.
* */
-@Namespace("nvinfer1") public enum DimensionOperation {
- /** Sum of the two operands. */
- kSUM(0),
- /** Product of the two operands. */
- kPROD(1),
- /** Maximum of the two operands. */
- kMAX(2),
- /** Minimum of the two operands. */
- kMIN(3),
- /** Substract the second element from the first. */
- kSUB(4),
- /** 1 if operands are equal, 0 otherwise. */
- kEQUAL(5),
- /** 1 if first operand is less than second operand, 0 otherwise. */
- kLESS(6),
- /** Floor division of the first element by the second. */
- kFLOOR_DIV(7),
- /** Division rounding up */
- kCEIL_DIV(8);
+@Namespace("nvinfer1") public enum WeightsRole {
+ /** kernel for IConvolutionLayer or IDeconvolutionLayer */
+ kKERNEL(0),
+ /** bias for IConvolutionLayer or IDeconvolutionLayer */
+ kBIAS(1),
+ /** shift part of IScaleLayer */
+ kSHIFT(2),
+ /** scale part of IScaleLayer */
+ kSCALE(3),
+ /** weights for IConstantLayer */
+ kCONSTANT(4),
+ /** Any other weights role */
+ kANY(5);
public final int value;
- private DimensionOperation(int v) { this.value = v; }
- private DimensionOperation(DimensionOperation e) { this.value = e.value; }
- public DimensionOperation intern() { for (DimensionOperation e : values()) if (e.value == value) return e; return this; }
+ private WeightsRole(int v) { this.value = v; }
+ private WeightsRole(WeightsRole e) { this.value = e.value; }
+ public WeightsRole intern() { for (WeightsRole e : values()) if (e.value == value) return e; return this; }
@Override public String toString() { return intern().name(); }
}
-/** Maximum number of elements in DimensionOperation enum. @see DimensionOperation */
+/** Maximum number of elements in WeightsRole enum. @see WeightsRole */
/**
- * \enum TensorLocation
+ * \enum DeviceType
+ * \brief The device that this layer/network will execute on.
*
- * \brief The location for tensor data storage, device or host.
* */
-@Namespace("nvinfer1") public enum TensorLocation {
- /** Data stored on device. */
- kDEVICE(0),
- /** Data stored on host. */
- kHOST(1);
+@Namespace("nvinfer1") public enum DeviceType {
+ /** GPU Device */
+ kGPU(0),
+ /** DLA Core */
+ kDLA(1);
public final int value;
- private TensorLocation(int v) { this.value = v; }
- private TensorLocation(TensorLocation e) { this.value = e.value; }
- public TensorLocation intern() { for (TensorLocation e : values()) if (e.value == value) return e; return this; }
+ private DeviceType(int v) { this.value = v; }
+ private DeviceType(DeviceType e) { this.value = e.value; }
+ public DeviceType intern() { for (DeviceType e : values()) if (e.value == value) return e; return this; }
@Override public String toString() { return intern().name(); }
}
-/** Maximum number of elements in TensorLocation enum. @see TensorLocation */
-
-// Targeting ../nvinfer/IDimensionExpr.java
-
-
-// Targeting ../nvinfer/IExprBuilder.java
-
-
-// Targeting ../nvinfer/DimsExprs.java
-
-
-// Targeting ../nvinfer/DynamicPluginTensorDesc.java
-
-
-// Targeting ../nvinfer/IPluginV2DynamicExt.java
-
-
-// Targeting ../nvinfer/IPluginResourceContext.java
-
-
-// Targeting ../nvinfer/IPluginCapability.java
-
-
- // namespace v_1_0
-
-/**
- * \class IPluginCapability
- *
- * \brief Base class for plugin capability interfaces
- *
- * IPluginCapability represents a split in TensorRT V3 plugins to sub-objects that expose different types of
- * capabilites a plugin may have, as opposed to a single interface which defines all capabilities and behaviors of a
- * plugin.
- *
- * \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
- *
- * @see PluginCapabilityType
- * */
-// Targeting ../nvinfer/IPluginV3.java
-
+/** Maximum number of elements in DeviceType enum. @see DeviceType */
- // namespace v_1_0
/**
- * \class IPluginV3
- *
- * \brief Plugin class for the V3 generation of user-implemented layers.
+ * \enum TempfileControlFlag
*
- * IPluginV3 acts as a wrapper around the plugin capability interfaces that define the actual behavior of the plugin.
+ * \brief Flags used to control TensorRT's behavior when creating executable temporary files.
*
- * @see IPluginCapability
- * @see IPluginCreatorV3One
- * @see IPluginRegistry
+ * On some platforms the TensorRT runtime may need to create files in a temporary directory or use platform-specific
+ * APIs to create files in-memory to load temporary DLLs that implement runtime code. These flags allow the
+ * application to explicitly control TensorRT's use of these files. This will preclude the use of certain TensorRT
+ * APIs for deserializing and loading lean runtimes.
* */
-// Targeting ../nvinfer/IPluginV3OneCore.java
-
-
-// Targeting ../nvinfer/IPluginV3OneBuild.java
-
+@Namespace("nvinfer1") public enum TempfileControlFlag {
+ /** Allow creating and loading files in-memory (or unnamed files). */
+
+//!
+ kALLOW_IN_MEMORY_FILES(0),
-// Targeting ../nvinfer/IPluginV3OneRuntime.java
+ /** Allow creating and loading named files in a temporary directory on the filesystem.
+ *
+ * @see IRuntime::setTemporaryDirectory() */
+ kALLOW_TEMPORARY_FILES(1);
+ public final int value;
+ private TempfileControlFlag(int v) { this.value = v; }
+ private TempfileControlFlag(TempfileControlFlag e) { this.value = e.value; }
+ public TempfileControlFlag intern() { for (TempfileControlFlag e : values()) if (e.value == value) return e; return this; }
+ @Override public String toString() { return intern().name(); }
+}
- // namespace v_1_0
+/** Maximum number of elements in TempfileControlFlag enum. @see TempfileControlFlag */
- // namespace v_2_0
/**
- * \class IPluginV3OneCore
- *
- * \brief A plugin capability interface that enables the core capability (PluginCapabilityType::kCORE).
+ * \brief Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operations.
*
- * @see IPluginCapability
- * @see PluginCapabilityType
- * @see IPluginV3::getCapabilityInterface()
- * */
+ * @see TempfileControlFlag,
+ * IRuntime::setTempfileControlFlags(),
+ * IRuntime::getTempfileControlFlags() */
//!
//!
//!
//!
-
-/**
- * \class IPluginV3OneBuild
- *
- * \brief A plugin capability interface that enables the build capability (PluginCapabilityType::kBUILD). Exposes
- * methods that allow the expression of the build time properties and behavior of a plugin.
- *
- * @see IPluginCapability
- * @see PluginCapabilityType
- * @see IPluginV3::getCapabilityInterface()
- * */
-
-
+//!
+//!
//!
//!
//!
//!
/**
- * \class IPluginV3OneRuntime
+ * \enum TensorFormat
*
- * \brief A plugin capability interface that enables the runtime capability (PluginCapabilityType::kRUNTIME). Exposes
- * methods that allow the expression of the runtime properties and behavior of a plugin.
+ * \brief Format of the input/output tensors.
*
- * @see IPluginCapability
- * @see PluginCapabilityType
- * @see IPluginV3::getCapabilityInterface()
+ * This enum is used by both plugins and network I/O tensors.
+ *
+ * @see IPluginV2::supportsFormat(), safe::ICudaEngine::getBindingFormat()
+ *
+ * Many of the formats are **vector-major** or **vector-minor**. These formats specify
+ * a vector dimension and scalars per vector.
+ * For example, suppose that the tensor has has dimensions [M,N,C,H,W],
+ * the vector dimension is C and there are V scalars per vector.
+ *
+ * * A **vector-major** format splits the vectorized dimension into two axes in the
+ * memory layout. The vectorized dimension is replaced by an axis of length ceil(C/V)
+ * and a new dimension of length V is appended. For the example tensor, the memory layout
+ * is equivalent to an array with dimensions [M][N][ceil(C/V)][H][W][V].
+ * Tensor coordinate (m,n,c,h,w) maps to array location [m][n][c/V][h][w][c\%V].
+ *
+ * * A **vector-minor** format moves the vectorized dimension to become the last axis
+ * in the memory layout. For the example tensor, the memory layout is equivalent to an
+ * array with dimensions [M][N][H][W][ceil(C/V)*V]. Tensor coordinate (m,n,c,h,w) maps
+ * array location subscript [m][n][h][w][c].
+ *
+ * In interfaces that refer to "components per element", that's the value of V above.
+ *
+ * For more information about data formats, see the topic "Data Format Description" located in the
+ * TensorRT Developer Guide. https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#data-format-desc
* */
+@Namespace("nvinfer1") public enum TensorFormat {
+ /** Memory layout is similar to an array in C or C++.
+ * The stride of each dimension is the product of the dimensions after it.
+ * The last dimension has unit stride.
+ *
+ * For DLA usage, the tensor sizes are limited to C,H,W in the range [1,8192]. */
+
+//!
+ kLINEAR(0),
+ /** Vector-major format with two scalars per vector.
+ * Vector dimension is third to last.
+ *
+ * This format requires FP16 and at least three dimensions. */
+ kCHW2(1),
+ /** Vector-minor format with eight scalars per vector.
+ * Vector dimension is third to last.
+ * This format requires FP16 or BF16 and at least three dimensions. */
+
//!
//!
//!
-//!
-
-/**
- * \class IPluginV3OneBuildV2
- *
- * \brief A plugin capability interface that extends IPluginV3OneBuild by providing I/O aliasing functionality.
- *
- * @see IPluginV3OneBuild
- * */
-// Targeting ../nvinfer/IPluginCreatorV3One.java
-
-
- // namespace v_1_0
-
-/**
- * \class IPluginCreatorV3One
- *
- * \brief A plugin creator class capable of producing IPluginV3 objects
- *
- * @see IPluginV3
- * @see IPluginRegistry
- * */
-// Targeting ../nvinfer/IProfiler.java
+ kHWC8(2),
+ /** Vector-major format with four scalars per vector.
+ * Vector dimension is third to last.
+ *
+ * This format requires INT8 or FP16 and at least three dimensions.
+ * For INT8, the length of the vector dimension must be a build-time constant.
+ *
+ * Deprecated usage:
+ *
+ * If running on the DLA, this format can be used for acceleration
+ * with the caveat that C must be less than or equal to 4.
+ * If used as DLA input and the build option kGPU_FALLBACK is not specified,
+ * it needs to meet line stride requirement of DLA format. Column stride in
+ * bytes must be a multiple of 64 on Orin. */
+
+//!
+//!
+ kCHW4(3),
- // namespace v_1_0
+ /** Vector-major format with 16 scalars per vector.
+ * Vector dimension is third to last.
+ *
+ * This format requires FP16 and at least three dimensions.
+ *
+ * For DLA usage, this format maps to the native feature format for FP16,
+ * and the tensor sizes are limited to C,H,W in the range [1,8192]. */
+
+//!
+//!
+ kCHW16(4),
-/**
- * \class IProfiler
- *
- * \brief Application-implemented interface for profiling.
- *
- * When this class is added to an execution context, the profiler will be called once per layer for each invocation of
- * executeV2()/enqueueV3().
- *
- * It is not recommended to run inference with profiler enabled when the inference execution time is critical since the
- * profiler may affect execution time negatively.
- * */
+ /** Vector-major format with 32 scalars per vector.
+ * Vector dimension is third to last.
+ *
+ * This format requires at least three dimensions.
+ *
+ * For DLA usage, this format maps to the native feature format for INT8,
+ * and the tensor sizes are limited to C,H,W in the range [1,8192]. */
+
+//!
+ kCHW32(5),
+ /** Vector-minor format with eight scalars per vector.
+ * Vector dimension is fourth to last.
+ *
+ * This format requires FP16 or BF16 and at least four dimensions. */
+
+//!
+ kDHWC8(6),
+ /** Vector-major format with 32 scalars per vector.
+ * Vector dimension is fourth to last.
+ *
+ * This format requires FP16 or INT8 and at least four dimensions. */
+
//!
+ kCDHW32(7),
+
+ /** Vector-minor format where channel dimension is third to last and unpadded.
+ *
+ * This format requires either FP32, FP16, UINT8, INT64 or BF16 and at least three dimensions. */
+
//!
+ kHWC(8),
+
+ /** DLA planar format. For a tensor with dimension {N, C, H, W}, the W axis
+ * always has unit stride. The stride for stepping along the H axis is
+ * rounded up to 64 bytes.
+ *
+ * The memory layout is equivalent to a C array with dimensions
+ * [N][C][H][roundUp(W, 64/elementSize)] where elementSize is
+ * 2 for FP16 and 1 for Int8, with the tensor coordinates (n, c, h, w)
+ * mapping to array subscript [n][c][h][w]. */
+
//!
+ kDLA_LINEAR(9),
+
+ /** DLA image format. For a tensor with dimension {N, C, H, W} the C axis
+ * always has unit stride. The stride for stepping along the H axis is rounded up
+ * to 64 bytes on Orin. C can only be 1, 3 or 4.
+ * If C == 1, it will map to grayscale format.
+ * If C == 3 or C == 4, it will map to color image format. And if C == 3,
+ * the stride for stepping along the W axis needs to be padded to 4 in elements.
+ *
+ * When C is {1, 3, 4}, then C' is {1, 4, 4} respectively,
+ * the memory layout is equivalent to a C array with dimensions
+ * [N][H][roundUp(W, 64/C'/elementSize)][C'] on Orin
+ * where elementSize is 2 for FP16
+ * and 1 for Int8. The tensor coordinates (n, c, h, w) mapping to array
+ * subscript [n][h][w][c]. */
+
//!
+ kDLA_HWC4(10),
-/**
- * \enum WeightsRole
- *
- * \brief How a layer uses particular Weights.
- *
- * The power weights of an IScaleLayer are omitted. Refitting those is not supported.
- * */
-@Namespace("nvinfer1") public enum WeightsRole {
- /** kernel for IConvolutionLayer or IDeconvolutionLayer */
- kKERNEL(0),
- /** bias for IConvolutionLayer or IDeconvolutionLayer */
- kBIAS(1),
- /** shift part of IScaleLayer */
- kSHIFT(2),
- /** scale part of IScaleLayer */
- kSCALE(3),
- /** weights for IConstantLayer */
- kCONSTANT(4),
- /** Any other weights role */
- kANY(5);
+ /** Vector-minor format with 16 scalars per vector.
+ * Vector dimension is third to last.
+ *
+ * This requires FP16 or INT8 and at least three dimensions. */
+
+//!
+ kHWC16(11),
+
+ /** Vector-minor format with one scalar per vector.
+ * Vector dimension is fourth to last.
+ *
+ * This format requires FP32 and at least four dimensions. */
+ kDHWC(12);
public final int value;
- private WeightsRole(int v) { this.value = v; }
- private WeightsRole(WeightsRole e) { this.value = e.value; }
- public WeightsRole intern() { for (WeightsRole e : values()) if (e.value == value) return e; return this; }
+ private TensorFormat(int v) { this.value = v; }
+ private TensorFormat(TensorFormat e) { this.value = e.value; }
+ public TensorFormat intern() { for (TensorFormat e : values()) if (e.value == value) return e; return this; }
@Override public String toString() { return intern().name(); }
}
-
-/** Maximum number of elements in WeightsRole enum. @see WeightsRole */
-
+/** Maximum number of elements in TensorFormat enum. @see TensorFormat */
+ // namespace impl
/**
- * \enum DeviceType
- * \brief The device that this layer/network will execute on.
+ * \enum AllocatorFlag
*
+ * \brief Allowed type of memory allocation.
* */
-@Namespace("nvinfer1") public enum DeviceType {
- /** GPU Device */
- kGPU(0),
- /** DLA Core */
- kDLA(1);
+@Namespace("nvinfer1") public enum AllocatorFlag {
+ /** TensorRT may call realloc() on this allocation. */
+ kRESIZABLE(0);
public final int value;
- private DeviceType(int v) { this.value = v; }
- private DeviceType(DeviceType e) { this.value = e.value; }
- public DeviceType intern() { for (DeviceType e : values()) if (e.value == value) return e; return this; }
+ private AllocatorFlag(int v) { this.value = v; }
+ private AllocatorFlag(AllocatorFlag e) { this.value = e.value; }
+ public AllocatorFlag intern() { for (AllocatorFlag e : values()) if (e.value == value) return e; return this; }
@Override public String toString() { return intern().name(); }
}
+/** Maximum number of elements in AllocatorFlag enum. @see AllocatorFlag */
+ // namespace impl
-/** Maximum number of elements in DeviceType enum. @see DeviceType */
-/**
- * \enum TempfileControlFlag
- *
- * \brief Flags used to control TensorRT's behavior when creating executable temporary files.
- *
- * On some platforms the TensorRT runtime may need to create files in a temporary directory or use platform-specific
- * APIs to create files in-memory to load temporary DLLs that implement runtime code. These flags allow the
- * application to explicitly control TensorRT's use of these files. This will preclude the use of certain TensorRT
- * APIs for deserializing and loading lean runtimes.
- * */
-@Namespace("nvinfer1") public enum TempfileControlFlag {
- /** Allow creating and loading files in-memory (or unnamed files). */
-
//!
- kALLOW_IN_MEMORY_FILES(0),
+//!
+//!
+//!
+//!
+// Targeting ../nvinfer/ILogger.java
- /** Allow creating and loading named files in a temporary directory on the filesystem.
- *
- * @see IRuntime::setTemporaryDirectory() */
- kALLOW_TEMPORARY_FILES(1);
- public final int value;
- private TempfileControlFlag(int v) { this.value = v; }
- private TempfileControlFlag(TempfileControlFlag e) { this.value = e.value; }
- public TempfileControlFlag intern() { for (TempfileControlFlag e : values()) if (e.value == value) return e; return this; }
- @Override public String toString() { return intern().name(); }
-}
+/** Maximum number of elements in ILogger::Severity enum. @see ILogger::Severity */
+
+// Targeting ../nvinfer/IGpuAllocator.java
-/** Maximum number of elements in TempfileControlFlag enum. @see TempfileControlFlag */
+ // namespace v_1_0
+
/**
- * \brief Represents a collection of one or more TempfileControlFlag values combined using bitwise-OR operations.
+ * \class IGpuAllocator
*
- * @see TempfileControlFlag,
- * IRuntime::setTempfileControlFlags(),
- * IRuntime::getTempfileControlFlags() */
+ * \brief Application-implemented class for controlling allocation on the GPU.
+ *
+ * \warning The lifetime of an IGpuAllocator object must exceed that of all objects that use it.
+ *
+ * This class is intended as a base class for allocators that implement synchronous allocation.
+ * If you want the benefits of asynchronous allocation, you can do either of:
+ *
+ * * Derive your class from IGpuAllocator and override all four of its virtual methods
+ * for allocation/deallocation, including the two deprecated methods.
+ *
+ * * Derive your class from IGpuAsyncAllocator and override its two pure virtual
+ * methods for allocation/deallocation.
+ *
+ * The latter style is preferred because it does not tie code to deprecated methods.
+ *
+ * @see IGpuAsyncAllocator.
+ * */
//!
@@ -1904,7 +1600,6 @@ public class nvinfer extends org.bytedeco.tensorrt.presets.nvinfer {
* including two deprecated methods.
*
* @see IGpuAllocator */
-
// namespace nvinfer1
/**
@@ -3566,7 +3261,20 @@ public class nvinfer extends org.bytedeco.tensorrt.presets.nvinfer {
* control over which weights are refittable or not using INetworkDefinition::markWeightsRefittable and
* INetworkDefinition::unmarkWeightsRefittable. By default, all weights are non-refittable when this flag is
* enabled. This flag cannot be used together with kREFIT or kREFIT_IDENTICAL. */
- kREFIT_INDIVIDUAL(23);
+ kREFIT_INDIVIDUAL(23),
+
+ /** Disable floating-point optimizations: 0*x => 0, x-x => 0, or x/x => 1. These identities are
+ * not true when x is a NaN or Inf, and thus might hide propagation or generation of NaNs. This flag is typically
+ * used in combination with kSPARSE_WEIGHTS.
+ * There are three valid sparsity configurations.
+ * 1. Disable all sparsity. Both kSPARSE_WEIGHTS and kSTRICT_NANS are unset
+ * 2. Enable sparsity only where it does not affect propagation/generation of NaNs. Both kSPARSE_WEIGHTS and
+ * kSTRICT_NANS are set
+ * 3. Enable all sparsity. kSPARSE_WEIGHTS is set and kSTRICT_NANS is unset */
+ kSTRICT_NANS(24),
+
+ /** Enable memory monitor during build time. */
+ kMONITOR_MEMORY(25);
public final int value;
private BuilderFlag(int v) { this.value = v; }
@@ -3944,6 +3652,7 @@ public class nvinfer extends org.bytedeco.tensorrt.presets.nvinfer {
// namespace v_1_0
+ // namespace v_1_0
/** enum class nvinfer1::ActivationType */
;
@@ -4156,103 +3865,466 @@ public class nvinfer extends org.bytedeco.tensorrt.presets.nvinfer {
// Targeting ../nvinfer/VRecurrenceLayer.java
-// Targeting ../nvinfer/VLoopOutputLayer.java
+// Targeting ../nvinfer/VLoopOutputLayer.java
+
+
+// Targeting ../nvinfer/VTripLimitLayer.java
+
+
+// Targeting ../nvinfer/VIteratorLayer.java
+
+
+// Targeting ../nvinfer/VLoop.java
+
+
+// Targeting ../nvinfer/VConditionalBoundaryLayer.java
+
+
+// Targeting ../nvinfer/VConditionLayer.java
+
+
+// Targeting ../nvinfer/VConditionalInputLayer.java
+
+
+// Targeting ../nvinfer/VConditionalOutputLayer.java
+
+
+// Targeting ../nvinfer/VIfConditional.java
+
+
+// Targeting ../nvinfer/VSelectLayer.java
+
+
+// Targeting ../nvinfer/VAssertionLayer.java
+
+
+// Targeting ../nvinfer/VFillLayer.java
+
+
+// Targeting ../nvinfer/VQuantizeLayer.java
+
+
+// Targeting ../nvinfer/VDequantizeLayer.java
+
+
+// Targeting ../nvinfer/VScatterLayer.java
+
+
+// Targeting ../nvinfer/VEinsumLayer.java
+
+
+// Targeting ../nvinfer/VOneHotLayer.java
+
+
+// Targeting ../nvinfer/VGridSampleLayer.java
+
+
+// Targeting ../nvinfer/VNMSLayer.java
+
+
+// Targeting ../nvinfer/VReverseSequenceLayer.java
+
+
+// Targeting ../nvinfer/VNormalizationLayer.java
+
+
+// Targeting ../nvinfer/VNetworkDefinition.java
+
+
+// Targeting ../nvinfer/VAlgorithmIOInfo.java
+
+
+// Targeting ../nvinfer/VAlgorithmVariant.java
+
+
+// Targeting ../nvinfer/VAlgorithmContext.java
+
+
+// Targeting ../nvinfer/VAlgorithm.java
+
+
+// Targeting ../nvinfer/VTimingCache.java
+
+
+// Targeting ../nvinfer/VBuilderConfig.java
+
+
+// Targeting ../nvinfer/VSerializationConfig.java
+
+
+// Targeting ../nvinfer/VBuilder.java
+
+
+
+ // namespace apiv
+ // namespace nvinfer1
+
+// @endcond
+
+// #endif // NV_INFER_RUNTIME_IMPL_H
+
+
+// Parsed from NvInferPluginBase.h
+
+/*
+* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+* SPDX-License-Identifier: LicenseRef-NvidiaProprietary
+*
+* NVIDIA CORPORATION, its affiliates and licensors retain all intellectual
+* property and proprietary rights in and to this material, related
+* documentation and any modifications thereto. Any use, reproduction,
+* disclosure or distribution of this material and related documentation
+* without an express license agreement from NVIDIA CORPORATION or
+* its affiliates is strictly prohibited.
+*/
+
+// #ifndef NV_INFER_PLUGIN_BASE_H
+// #define NV_INFER_PLUGIN_BASE_H
+
+// #if !defined(NV_INFER_INTERNAL_INCLUDE)
+// #endif
+// #include "NvInferRuntimeBase.h"
+// #undef NV_INFER_INTERNAL_INCLUDE
+
+/**
+ * \enum PluginFieldType
+ *
+ * \brief The possible field types for custom layer.
+ * */
+@Namespace("nvinfer1") public enum PluginFieldType {
+ /** FP16 field type. */
+ kFLOAT16(0),
+ /** FP32 field type. */
+ kFLOAT32(1),
+ /** FP64 field type. */
+ kFLOAT64(2),
+ /** INT8 field type. */
+ kINT8(3),
+ /** INT16 field type. */
+ kINT16(4),
+ /** INT32 field type. */
+ kINT32(5),
+ /** char field type. */
+ kCHAR(6),
+ /** nvinfer1::Dims field type. */
+ kDIMS(7),
+ /** Unknown field type. */
+ kUNKNOWN(8),
+ /** BF16 field type. */
+ kBF16(9),
+ /** INT64 field type. */
+ kINT64(10),
+ /** FP8 field type. */
+ kFP8(11),
+ /** INT4 field type. */
+ kINT4(12);
+
+ public final int value;
+ private PluginFieldType(int v) { this.value = v; }
+ private PluginFieldType(PluginFieldType e) { this.value = e.value; }
+ public PluginFieldType intern() { for (PluginFieldType e : values()) if (e.value == value) return e; return this; }
+ @Override public String toString() { return intern().name(); }
+}
+// Targeting ../nvinfer/PluginField.java
+
+
+// Targeting ../nvinfer/PluginFieldCollection.java
+
+
+
+/**
+ * \enum TensorRTPhase
+ *
+ * \brief Indicates a phase of operation of TensorRT
+ * */
+@Namespace("nvinfer1") public enum TensorRTPhase {
+ /** Build phase of TensorRT */
+ kBUILD(0),
+ /** Execution phase of TensorRT */
+ kRUNTIME(1);
+
+ public final int value;
+ private TensorRTPhase(int v) { this.value = v; }
+ private TensorRTPhase(TensorRTPhase e) { this.value = e.value; }
+ public TensorRTPhase intern() { for (TensorRTPhase e : values()) if (e.value == value) return e; return this; }
+ @Override public String toString() { return intern().name(); }
+}
+/**
+ * \enum PluginCapabilityType
+ *
+ * \brief Enumerates the different capability types a IPluginV3 object may have
+ * */
+@Namespace("nvinfer1") public enum PluginCapabilityType {
+ /** Core capability. Every IPluginV3 object must have this. */
+ kCORE(0),
+ /** Build capability. IPluginV3 objects provided to TensorRT build phase must have this. */
+ kBUILD(1),
+ /** Runtime capability. IPluginV3 objects provided to TensorRT build and execution phases must have this. */
+ kRUNTIME(2);
-// Targeting ../nvinfer/VTripLimitLayer.java
+ public final int value;
+ private PluginCapabilityType(int v) { this.value = v; }
+ private PluginCapabilityType(PluginCapabilityType e) { this.value = e.value; }
+ public PluginCapabilityType intern() { for (PluginCapabilityType e : values()) if (e.value == value) return e; return this; }
+ @Override public String toString() { return intern().name(); }
+}
+// Targeting ../nvinfer/IPluginCapability.java
-// Targeting ../nvinfer/VIteratorLayer.java
+// Targeting ../nvinfer/IPluginResource.java
-// Targeting ../nvinfer/VLoop.java
+// Targeting ../nvinfer/IPluginCreatorInterface.java
-// Targeting ../nvinfer/VConditionalBoundaryLayer.java
+// Targeting ../nvinfer/IPluginV3.java
-// Targeting ../nvinfer/VConditionLayer.java
+// Targeting ../nvinfer/IPluginCreatorV3One.java
-// Targeting ../nvinfer/VConditionalInputLayer.java
+ // namespace v_1_0
-// Targeting ../nvinfer/VConditionalOutputLayer.java
+/**
+ * \class IPluginCreatorV3One
+ *
+ * \brief A plugin creator class capable of producing IPluginV3 objects
+ *
+ * @see IPluginV3
+ * @see IPluginRegistry
+ * */
-// Targeting ../nvinfer/VIfConditional.java
+//!
+//!
+//!
+//!
+/**
+ * \class IPluginResource
+ *
+ * \brief Interface for plugins to define custom resources that could be shared through the plugin registry
+ *
+ * @see IPluginRegistry::acquirePluginResource
+ * @see IPluginRegistry::releasePluginResource
+ * */
-// Targeting ../nvinfer/VSelectLayer.java
+//!
+//!
+//!
+//!
-// Targeting ../nvinfer/VAssertionLayer.java
+/**
+ * \class IPluginCreatorInterface
+ *
+ * \brief Base class for all plugin creator versions.
+ *
+ * @see IPluginCreator and IPluginRegistry
+ * */
-// Targeting ../nvinfer/VFillLayer.java
+//!
+//!
+//!
+//!
+//!
+/**
+ * \class IPluginV3
+ *
+ * \brief Plugin class for the V3 generation of user-implemented layers.
+ *
+ * IPluginV3 acts as a wrapper around the plugin capability interfaces that define the actual behavior of the plugin.
+ *
+ * @see IPluginCapability
+ * @see IPluginCreatorV3One
+ * @see IPluginRegistry
+ * */
-// Targeting ../nvinfer/VQuantizeLayer.java
+//!
+//!
+//!
+//!
+//!
+//!
-// Targeting ../nvinfer/VDequantizeLayer.java
+/**
+ * \class IPluginCapability
+ *
+ * \brief Base class for plugin capability interfaces
+ *
+ * IPluginCapability represents a split in TensorRT V3 plugins to sub-objects that expose different types of
+ * capabilites a plugin may have, as opposed to a single interface which defines all capabilities and behaviors of a
+ * plugin.
+ *
+ * \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
+ *
+ * @see PluginCapabilityType
+ * */
+ // namespace nvinfer1
+// #endif /* NV_INFER_PLUGIN_BASE_H */
-// Targeting ../nvinfer/VScatterLayer.java
+// Parsed from NvInferRuntimePlugin.h
-// Targeting ../nvinfer/VEinsumLayer.java
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// #ifndef NV_INFER_RUNTIME_PLUGIN_H
+// #define NV_INFER_RUNTIME_PLUGIN_H
+// #include "NvInferPluginBase.h"
-// Targeting ../nvinfer/VOneHotLayer.java
+//!
+//!
+//!
+//!
-// Targeting ../nvinfer/VGridSampleLayer.java
+//!
+//!
+//!
+// #undef NV_INFER_INTERNAL_INCLUDE
+/**
+ * \file NvInferRuntimePlugin.h
+ *
+ * This file contains common definitions, data structures and interfaces that relate to plugins and are shared
+ * between the standard and safe runtime.
+ *
+ * \warning Do not directly include this file. Instead include NvInferRuntime.h
+ *
+
+ *
+ * \namespace nvinfer1
+ *
+ * \brief The TensorRT API version 1 namespace.
+ * */
-// Targeting ../nvinfer/VNMSLayer.java
+/** enum class nvinfer1::TensorFormat */
+;
-// Targeting ../nvinfer/VReverseSequenceLayer.java
+//!
+//!
+//!
-// Targeting ../nvinfer/VNormalizationLayer.java
+/**
+ * \brief PluginFormat is reserved for backward compatibility.
+ *
+ * @see IPluginV2::supportsFormat()
+ * */
-// Targeting ../nvinfer/VNetworkDefinition.java
+//!
+//!
+/**
+ * \brief Bit at the plugin version to identify that it is a plugin.
+ * */
-// Targeting ../nvinfer/VAlgorithmIOInfo.java
+//!
+//!
+//!
+//!
+//!
+@Namespace("nvinfer1") @MemberGetter public static native int kPLUGIN_VERSION_PYTHON_BIT();
+public static final int kPLUGIN_VERSION_PYTHON_BIT = kPLUGIN_VERSION_PYTHON_BIT();
+// Targeting ../nvinfer/PluginTensorDesc.java
-// Targeting ../nvinfer/VAlgorithmVariant.java
-// Targeting ../nvinfer/VAlgorithmContext.java
+/**
+ * \struct PluginVersion
+ *
+ * \brief Definition of plugin versions.
+ *
+ * Tag for plug-in versions. Used in upper byte of getTensorRTVersion().
+ * */
+@Namespace("nvinfer1") public enum PluginVersion {
+ /** IPluginV2 */
+ kV2((byte)(0)),
+ /** IPluginV2Ext */
+ kV2_EXT((byte)(1)),
+ /** IPluginV2IOExt */
+ kV2_IOEXT((byte)(2)),
+ /** IPluginV2DynamicExt */
+ kV2_DYNAMICEXT((byte)(3)),
+ /** IPluginV2DynamicExt-based Python plugins */
+ kV2_DYNAMICEXT_PYTHON((byte)(kPLUGIN_VERSION_PYTHON_BIT | 3));
+ public final byte value;
+ private PluginVersion(byte v) { this.value = v; }
+ private PluginVersion(PluginVersion e) { this.value = e.value; }
+ public PluginVersion intern() { for (PluginVersion e : values()) if (e.value == value) return e; return this; }
+ @Override public String toString() { return intern().name(); }
+}
-// Targeting ../nvinfer/VAlgorithm.java
+/**
+ * \enum PluginCreatorVersion
+ *
+ * \brief Enum to identify version of the plugin creator.
+ * */
+@Namespace("nvinfer1") public enum PluginCreatorVersion {
+ /** IPluginCreator */
+ kV1(0),
+ /** IPluginCreator-based Python plugin creators */
+ kV1_PYTHON(kPLUGIN_VERSION_PYTHON_BIT);
+ public final int value;
+ private PluginCreatorVersion(int v) { this.value = v; }
+ private PluginCreatorVersion(PluginCreatorVersion e) { this.value = e.value; }
+ public PluginCreatorVersion intern() { for (PluginCreatorVersion e : values()) if (e.value == value) return e; return this; }
+ @Override public String toString() { return intern().name(); }
+}
+// Targeting ../nvinfer/IPluginV2.java
-// Targeting ../nvinfer/VTimingCache.java
+// Targeting ../nvinfer/IPluginV2Ext.java
-// Targeting ../nvinfer/VBuilderConfig.java
+// Targeting ../nvinfer/IPluginV2IOExt.java
-// Targeting ../nvinfer/VSerializationConfig.java
+// Targeting ../nvinfer/IPluginCreator.java
-// Targeting ../nvinfer/VBuilder.java
+ // namespace v_1_0
+/**
+ * \class IPluginCreator
+ *
+ * \brief Plugin creator class for user implemented layers.
+ *
+ * @see IPlugin and IPluginFactory
+ *
+ * @deprecated Deprecated in TensorRT 10.0. Please implement IPluginCreatorV3One instead along with IPluginV3 plugins
+ * instead.
+ * */
- // namespace apiv
// namespace nvinfer1
-// @endcond
-
-// #endif // NV_INFER_RUNTIME_IMPL_H
+// #endif // NV_INFER_RUNTIME_PLUGIN_H
}
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IBuilder.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IBuilder.java
index f27e1a3125..06ffe336b2 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IBuilder.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IBuilder.java
@@ -250,7 +250,7 @@ public class IBuilder extends INoCopy {
*
* @return A pointer to a IHostMemory object that contains a serialized network.
*
- * \note This function will synchronize the cuda stream returned by \p config.getProfileStream() before returning.
+ * \note This function will synchronize the CUDA stream returned by \p config.getProfileStream() before returning.
*
* @see INetworkDefinition, IBuilderConfig, IHostMemory
* */
@@ -262,8 +262,34 @@ public class IBuilder extends INoCopy {
//!
//!
//!
+ //!
public native @NoException(true) IHostMemory buildSerializedNetwork(@ByRef INetworkDefinition network, @ByRef IBuilderConfig config);
+ /**
+ * \brief Builds a network for the given INetworkDefinition and IBuilderConfig.
+ *
+ * @param network Network definition.
+ * @param config Builder configuration.
+ *
+ * @return A pointer to a ICudaEngine object that contains an engine.
+ *
+ * \note This function will synchronize the CUDA stream returned by \p config.getProfileStream() before returning.
+ *
+ * \note This function does not support \p BuilderFlag::kVERSION_COMPATIBLE.
+ * Please use \p buildSerializedNetwork to get a version compatible engine.
+ *
+ * @see INetworkDefinition, IBuilderConfig, ICudaEngine
+ * */
+
+
+ //!
+ //!
+ //!
+ //!
+ //!
+ //!
+ public native @NoException(true) ICudaEngine buildEngineWithConfig(@ByRef INetworkDefinition network, @ByRef IBuilderConfig config);
+
/**
* \brief Checks that a network is within the scope of the IBuilderConfig settings.
*
@@ -279,7 +305,7 @@ public class IBuilder extends INoCopy {
* @return True if network is within the scope of the restrictions specified by the builder config,
* false otherwise.
*
- * \note This function will synchronize the cuda stream returned by \p config.getProfileStream() before returning.
+ * \note This function will synchronize the CUDA stream returned by \p config.getProfileStream() before returning.
* */
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IBuilderConfig.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IBuilderConfig.java
index 6a0da55a28..38d173b53e 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IBuilderConfig.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IBuilderConfig.java
@@ -365,9 +365,9 @@ public class IBuilderConfig extends INoCopy {
public native @NoException(true) void reset();
/**
- * \brief Set the cuda stream that is used to profile this network.
+ * \brief Set the CUDA stream that is used to profile this network.
*
- * @param stream The cuda stream used for profiling by the builder.
+ * @param stream The CUDA stream used for profiling by the builder.
*
* @see getProfileStream()
* */
@@ -380,9 +380,9 @@ public class IBuilderConfig extends INoCopy {
public native @NoException(true) void setProfileStream(CUstream_st stream);
/**
- * \brief Get the cuda stream that is used to profile this network.
+ * \brief Get the CUDA stream that is used to profile this network.
*
- * @return The cuda stream set by setProfileStream, nullptr if setProfileStream has not been called.
+ * @return The CUDA stream set by setProfileStream, nullptr if setProfileStream has not been called.
*
* @see setProfileStream()
* */
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IDebugListener.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IDebugListener.java
index 958d98cedf..256b6b2d87 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IDebugListener.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IDebugListener.java
@@ -44,7 +44,7 @@ public class IDebugListener extends IVersionedInterface {
* @param type data Type of the tensor.
* @param shape shape of the tensor.
* @param name name of the tensor.
- * @param stream Cuda stream object.
+ * @param stream CUDA stream object.
*
* @return True on success, false otherwise.
* */
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IExecutionContext.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IExecutionContext.java
index c0d00e3ab6..d643b2e503 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IExecutionContext.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IExecutionContext.java
@@ -148,7 +148,7 @@ public class IExecutionContext extends INoCopy {
/**
* \brief Set the device memory for use by this execution context.
*
- * The memory must be aligned with cuda memory alignment property (using cudaGetDeviceProperties()), and its size
+ * The memory must be aligned with CUDA memory alignment property (using cudaGetDeviceProperties()), and its size
* must be large enough for performing inference with the given network inputs. getDeviceMemorySize() and
* getDeviceMemorySizeForProfile() report upper bounds of the size. Setting memory to nullptr is acceptable if the
* reported size is 0. If using enqueueV3() to run the network, the memory is in use from the invocation of
@@ -178,7 +178,7 @@ public class IExecutionContext extends INoCopy {
/**
* \brief Set the device memory and its corresponding size for use by this execution context.
*
- * The memory must be aligned with cuda memory alignment property (using cudaGetDeviceProperties()), and its size
+ * The memory must be aligned with CUDA memory alignment property (using cudaGetDeviceProperties()), and its size
* must be large enough for performing inference with the given network inputs. getDeviceMemorySize() and
* getDeviceMemorySizeForProfile() report upper bounds of the size. Setting memory to nullptr is acceptable if the
* reported size is 0. If using enqueueV3() to run the network, the memory is in use from the invocation of
@@ -431,7 +431,7 @@ public class IExecutionContext extends INoCopy {
* @param profileIndex Index of the profile. The value must lie between 0 and
* getEngine().getNbOptimizationProfiles() - 1
*
- * @param stream A cuda stream on which the cudaMemcpyAsyncs may be
+ * @param stream A CUDA stream on which the cudaMemcpyAsyncs may be
* enqueued
*
* When an optimization profile is switched via this API, TensorRT may
@@ -778,7 +778,7 @@ public class IExecutionContext extends INoCopy {
/**
* \brief Mark input as consumed.
*
- * @param event The cuda event that is triggered after all input tensors have been consumed.
+ * @param event The CUDA event that is triggered after all input tensors have been consumed.
*
* \warning The set event must be valid during the inferece.
*
@@ -796,7 +796,7 @@ public class IExecutionContext extends INoCopy {
/**
* \brief The event associated with consuming the input.
*
- * @return The cuda event. Nullptr will be returned if the event is not set yet.
+ * @return The CUDA event. Nullptr will be returned if the event is not set yet.
* */
@@ -915,7 +915,7 @@ public class IExecutionContext extends INoCopy {
/**
* \brief Enqueue inference on a stream.
*
- * @param stream A cuda stream on which the inference kernels will be enqueued.
+ * @param stream A CUDA stream on which the inference kernels will be enqueued.
*
* @return True if the kernels were enqueued successfully, false otherwise.
*
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IGpuAllocator.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IGpuAllocator.java
index d3ff014add..d1952b6495 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IGpuAllocator.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IGpuAllocator.java
@@ -17,10 +17,7 @@
import static org.bytedeco.cuda.global.nvrtc.*;
import static org.bytedeco.tensorrt.global.nvinfer.*;
-
-
-/** DO NOT REFER TO namespace v_1_0 IN CODE. ALWAYS USE nvinfer1 INSTEAD.
- * The name v_1_0 may change in future versions of TensoRT. */
+ // namespace impl
@Namespace("nvinfer1::v_1_0") @Properties(inherit = org.bytedeco.tensorrt.presets.nvinfer.class)
public class IGpuAllocator extends IVersionedInterface {
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/ILogger.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/ILogger.java
index db186e21cd..e6cc3ca428 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/ILogger.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/ILogger.java
@@ -19,7 +19,10 @@
import static org.bytedeco.tensorrt.global.nvinfer.*;
-/**
+/** DO NOT REFER TO namespace v_1_0 IN CODE. ALWAYS USE nvinfer1 INSTEAD.
+ * The name v_1_0 may change in future versions of TensoRT.
+
+ *
* \class ILogger
*
* \brief Application-implemented logging interface for the builder, refitter and runtime.
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreator.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreator.java
index 02820857d2..112db030bd 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreator.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreator.java
@@ -18,7 +18,6 @@
import static org.bytedeco.tensorrt.global.nvinfer.*;
-
@Namespace("nvinfer1::v_1_0") @Properties(inherit = org.bytedeco.tensorrt.presets.nvinfer.class)
public class IPluginCreator extends IPluginCreatorInterface {
static { Loader.load(); }
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreatorInterface.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreatorInterface.java
index 995150e5f6..abd504a022 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreatorInterface.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreatorInterface.java
@@ -17,6 +17,7 @@
import static org.bytedeco.cuda.global.nvrtc.*;
import static org.bytedeco.tensorrt.global.nvinfer.*;
+ // class IPluginResource
@Namespace("nvinfer1::v_1_0") @Properties(inherit = org.bytedeco.tensorrt.presets.nvinfer.class)
public class IPluginCreatorInterface extends IVersionedInterface {
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreatorV3One.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreatorV3One.java
index ac753b0147..8cdd654ad4 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreatorV3One.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginCreatorV3One.java
@@ -18,6 +18,7 @@
import static org.bytedeco.tensorrt.global.nvinfer.*;
+
@Namespace("nvinfer1::v_1_0") @Properties(inherit = org.bytedeco.tensorrt.presets.nvinfer.class)
public class IPluginCreatorV3One extends IPluginCreatorInterface {
static { Loader.load(); }
@@ -34,6 +35,7 @@ public class IPluginCreatorV3One extends IPluginCreatorInterface {
//!
//!
//!
+ //!
public native @ByVal @NoException(true) InterfaceInfo getInterfaceInfo();
/**
@@ -47,6 +49,10 @@ public class IPluginCreatorV3One extends IPluginCreatorInterface {
* runtime interface will be passed as fc.
*
* \note The returned plugin object must be in an initialized state
+ *
+ * \note If invoked by the user (e.g. with TensorRTPhase::kBUILD, to add to the network defintion with
+ * addPluginV3()), it is the user's responsibility to delete the plugin object. If invoked by TensorRT (e.g. during
+ * engine deserialization), TensorRT will delete any objects it creates.
* */
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginResourceContext.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginResourceContext.java
index 7df6b53b7e..09c1d43e4b 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginResourceContext.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginResourceContext.java
@@ -19,6 +19,7 @@
import static org.bytedeco.tensorrt.global.nvinfer.*;
+
/**
* \class IPluginResourceContext
*
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginV3.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginV3.java
index 0c1acc074c..3233e2b6e7 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginV3.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginV3.java
@@ -18,6 +18,7 @@
import static org.bytedeco.tensorrt.global.nvinfer.*;
+
@Namespace("nvinfer1::v_1_0") @Properties(inherit = org.bytedeco.tensorrt.presets.nvinfer.class)
public class IPluginV3 extends IVersionedInterface {
static { Loader.load(); }
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginV3OneBuild.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginV3OneBuild.java
index 6ce6988079..10c762e260 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginV3OneBuild.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IPluginV3OneBuild.java
@@ -47,6 +47,7 @@ public class IPluginV3OneBuild extends IPluginCapability {
//!
//!
//!
+ //!
public native @ByVal @NoException(true) InterfaceInfo getInterfaceInfo();
/**
@@ -65,6 +66,8 @@ public class IPluginV3OneBuild extends IPluginCapability {
* @param nbInputs Number of input tensors.
* @param out The output tensors attributes that are used for configuration.
* @param nbOutputs Number of output tensors.
+ *
+ * @return 0 for success, else non-zero (which will cause engine termination, if invoked by TensorRT).
* */
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IRuntime.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IRuntime.java
index bfb8cd472a..0a665bdb3e 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IRuntime.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/IRuntime.java
@@ -182,11 +182,13 @@ public class IRuntime extends INoCopy {
* */
+
//!
//!
//!
public native ICudaEngine deserializeCudaEngine(@ByRef IStreamReader streamReader);
+
/**
* \brief get the logger with which the runtime was created
*
diff --git a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/VBuilder.java b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/VBuilder.java
index 5d6d417749..ac98c0e77c 100644
--- a/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/VBuilder.java
+++ b/tensorrt/src/gen/java/org/bytedeco/tensorrt/nvinfer/VBuilder.java
@@ -37,10 +37,12 @@ public class VBuilder extends VRoot {
public native @NoException(true) IErrorRecorder getErrorRecorder();
public native @NoException(true) void reset();
public native @Cast("bool") @NoException(true) boolean platformHasTf32();
- public native @NoException(true) IHostMemory buildSerializedNetwork(@ByRef INetworkDefinition network, @ByRef IBuilderConfig config);
+ public native @NoException(true) IHostMemory buildSerializedNetwork(
+ @ByRef INetworkDefinition network, @ByRef IBuilderConfig config);
public native @Cast("bool") @NoException(true) boolean isNetworkSupported(@Const @ByRef INetworkDefinition network, @Const @ByRef IBuilderConfig config);
public native @NoException(true) ILogger getLogger();
public native @Cast("bool") @NoException(true) boolean setMaxThreads(int maxThreads);
public native @NoException(true) int getMaxThreads();
public native @ByRef @NoException(true) IPluginRegistry getPluginRegistry();
+ public native @NoException(true) ICudaEngine buildEngineWithConfig(@ByRef INetworkDefinition network, @ByRef IBuilderConfig config);
}
diff --git a/tensorrt/src/main/java/org/bytedeco/tensorrt/presets/nvinfer.java b/tensorrt/src/main/java/org/bytedeco/tensorrt/presets/nvinfer.java
index 631ed304ab..cd38dcc533 100644
--- a/tensorrt/src/main/java/org/bytedeco/tensorrt/presets/nvinfer.java
+++ b/tensorrt/src/main/java/org/bytedeco/tensorrt/presets/nvinfer.java
@@ -47,11 +47,12 @@
@Platform(
value = {"linux-arm64", "linux-ppc64le", "linux-x86_64", "windows-x86_64"},
compiler = "cpp11",
- include = {"NvInferVersion.h", "NvInferRuntimeBase.h", "NvInferRuntimePlugin.h", "NvInferRuntimeCommon.h",
- "NvInferLegacyDims.h", "NvInferRuntime.h", "NvInfer.h", "NvInferImpl.h"/*, "NvUtils.h"*/},
+ include = {"NvInferVersion.h", "NvInferRuntimeBase.h", "NvInferRuntimeCommon.h",
+ "NvInferLegacyDims.h", "NvInferRuntime.h", "NvInfer.h", "NvInferImpl.h",
+ "NvInferPluginBase.h", "NvInferRuntimePlugin.h", /*, "NvUtils.h"*/},
exclude = "NvInferRuntimeBase.h",
link = "nvinfer@.10",
- preload = "nvinfer_builder_resource@.10.5.0"
+ preload = "nvinfer_builder_resource@.10.6.0"
),
@Platform(
value = "linux-arm64",
diff --git a/tritonserver/samples/unsupported/pom.xml b/tritonserver/samples/unsupported/pom.xml
index b4b253724c..4246bd5bcf 100644
--- a/tritonserver/samples/unsupported/pom.xml
+++ b/tritonserver/samples/unsupported/pom.xml
@@ -18,7 +18,7 @@