Skip to content

Commit

Permalink
Merge branch 'master' into feature/load_kmodel_from_stream
Browse files Browse the repository at this point in the history
  • Loading branch information
@sunnycase
sunnycase authored Jul 11, 2023
2 parents d61c21c + 7d9ac5d commit fa0adb2
Show file tree
Hide file tree
Showing 5 changed files with 82 additions and 65 deletions.
67 changes: 43 additions & 24 deletions src/Nncase.Importer/Onnx/DataGatter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,33 +99,52 @@ private Tensor GetTensor(TensorProto tensor)
{
return Tensor.FromBytes(type, tensor.RawData.ToByteArray(), shape);
}
else

// model size > 2G
// https://github.com/onnx/onnx/blob/main/docs/ExternalData.md
var externalDataCount = tensor.ExternalData.Count;
if (externalDataCount != 0)
{
return dt switch
if (externalDataCount < 3 && externalDataCount > 5)
{
// todo:not directly supported type should convert
// TensorProto.Types.DataType.Bool => Tensor.FromSpan(),
// TensorProto.Types.DataType.Float16 => Tensor.FromSpan(),
TensorProto.Types.DataType.Float => Tensor.From<float>(tensor.FloatData.ToArray(), shape),
TensorProto.Types.DataType.Double => Tensor.From<double>(tensor.DoubleData.ToArray(), shape),

// TensorProto.Types.DataType.Int16 => Tensor.FromSpan(),
TensorProto.Types.DataType.Int32 => Tensor.From<int>(tensor.Int32Data.ToArray(), shape),
TensorProto.Types.DataType.Int64 => Tensor.From<long>(tensor.Int64Data.ToArray(), shape),

TensorProto.Types.DataType.Int8 => Tensor.From<sbyte>(
tensor.Int32Data.Select(x => (sbyte)x).ToArray(),
shape),

// TensorProto.Types.DataType.String => Tensor.FromSpan(),
// TensorProto.Types.DataType.Uint32 => Tensor.FromSpan(),
// TensorProto.Types.DataType.Uint64 => Tensor.FromSpan<ulong>(tensor.Uint64Data.ToArray(), shape),
TensorProto.Types.DataType.Uint8 => Tensor.From<byte>(
tensor.Int32Data.Select(x => (byte)x).ToArray(),
shape),
_ => throw new NotSupportedException($"Not supported onnx constant data type{dt}"),
};
throw new NotSupportedException("NotSupport ExternalData format, only support location, offset, length, checksum");
}

var parent = Directory.GetParent(CompileSession.CompileOptions.InputFile)?.FullName;
var externalData = tensor.ExternalData;
var location = Path.Join(parent, externalData[0].Value);
var offset = long.Parse(externalData[1].Value);
var length = int.Parse(externalData[2].Value);
using var br = new BinaryReader(new FileStream(location, FileMode.Open));
br.BaseStream.Seek(offset, SeekOrigin.Begin);
var buffer = br.ReadBytes(length);
return Tensor.FromBytes(type, buffer, shape);
}

return dt switch
{
// todo:not directly supported type should convert
// TensorProto.Types.DataType.Bool => Tensor.FromSpan(),
// TensorProto.Types.DataType.Float16 => Tensor.FromSpan(),
TensorProto.Types.DataType.Float => Tensor.From<float>(tensor.FloatData.ToArray(), shape),
TensorProto.Types.DataType.Double => Tensor.From<double>(tensor.DoubleData.ToArray(), shape),

// TensorProto.Types.DataType.Int16 => Tensor.FromSpan(),
TensorProto.Types.DataType.Int32 => Tensor.From<int>(tensor.Int32Data.ToArray(), shape),
TensorProto.Types.DataType.Int64 => Tensor.From<long>(tensor.Int64Data.ToArray(), shape),

TensorProto.Types.DataType.Int8 => Tensor.From<sbyte>(
tensor.Int32Data.Select(x => (sbyte)x).ToArray(),
shape),

// TensorProto.Types.DataType.String => Tensor.FromSpan(),
// TensorProto.Types.DataType.Uint32 => Tensor.FromSpan(),
// TensorProto.Types.DataType.Uint64 => Tensor.FromSpan<ulong>(tensor.Uint64Data.ToArray(), shape),
TensorProto.Types.DataType.Uint8 => Tensor.From<byte>(
tensor.Int32Data.Select(x => (byte)x).ToArray(),
shape),
_ => throw new NotSupportedException($"Not supported onnx constant data type{dt}"),
};
}

private Expr GetInputExpr(NodeProto n, int index)
Expand Down
17 changes: 2 additions & 15 deletions src/Nncase.Tests/Rules/Neutral/UnitTestFoldQuant.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ public class UnitTestFoldQuant : TransformTestBase
{ 2, false, new[] { 1, 2, 3, 4 }, DataTypes.UInt8, new QuantParam(0, 0.043f), DataTypes.UInt8, new QuantParam(0, 0.043f), DataTypes.Int8 },
};

[Theory(Skip = "Quant loss threshold")]
[Theory]
[MemberData(nameof(FoldQuantDequantData))]
public void TestFoldQuantDequant(int count, bool is_pos, int[] shape, DataType input_dtype, QuantParam q_param, DataType quant_type, QuantParam deq_param, DataType dequant_type)
{
using var dumpScope = new DumpScope($"{count}");
var pre = IR.F.Math.Dequantize(IR.F.Math.Quantize(Random.Normal(input_dtype, 0, 1, 0, shape), q_param, quant_type), deq_param, dequant_type);
var pre = IR.F.Math.Dequantize(IR.F.Math.Quantize(Random.Uniform(DataTypes.Float32, 13, 0, 1, shape), q_param, quant_type), deq_param, dequant_type);
if (is_pos)
{
TestMatched<FoldQuantDeQuant>(pre);
Expand All @@ -43,19 +43,6 @@ public void TestFoldQuantDequant(int count, bool is_pos, int[] shape, DataType i
}
}

// todo remove it when resolve the bug
[Theory]
[MemberData(nameof(FoldQuantDequantData))]
public void TestFoldQuantDequant1(int count, bool is_pos, int[] shape, DataType input_dtype, QuantParam q_param, DataType quant_type, QuantParam deq_param, DataType dequant_type)
{
using var dumpScope = new DumpScope($"{count}");
var pre = IR.F.Math.Dequantize(IR.F.Math.Quantize(Random.Normal(input_dtype, 0, 1, 0, shape), q_param, quant_type), deq_param, dequant_type);
if (!is_pos)
{
TestNotMatch<FoldQuantDeQuant>(pre);
}
}

[Theory]
[MemberData(nameof(FoldDequantQuantData))]
public void TestFoldDequantQuant(int count, bool is_pos, int[] shape, DataType input_dtype, QuantParam deq_param, DataType dequant_type, QuantParam q_param, DataType quant_type)
Expand Down
2 changes: 1 addition & 1 deletion tests/kernels/kernel_test.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1185,7 +1185,7 @@ class KernelTest {

std::cout << "cosine_similarity:" << cosine_similarity << std::endl;
return cosine_similarity >
0.999f; // Return true if cosine similarity is close to 1
0.99f; // Return true if cosine similarity is close to 1
}

void print_runtime_tensor(runtime::runtime_tensor lhs) {
Expand Down
36 changes: 21 additions & 15 deletions tests/kernels/test_log_softmax.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,35 +26,47 @@ using namespace nncase;
using namespace nncase::runtime;
using namespace ortki;

class LogSoftmaxTest
: public KernelTest,
public ::testing::TestWithParam<std::tuple<nncase::typecode_t, dims_t>> {
class LogSoftmaxTest : public KernelTest,
public ::testing::TestWithParam<
std::tuple<nncase::typecode_t, dims_t, int64_t>> {
public:
void SetUp() override {
auto &&[typecode, l_shape] = GetParam();
auto &&[typecode, l_shape, value] = GetParam();

input =
hrt::create(typecode, l_shape, host_runtime_tensor::pool_cpu_only)
.expect("create tensor failed");
init_tensor(input);
axis_value = value >= (long)l_shape.size() ? 0 : value;
axis_value = axis_value < -(long)l_shape.size() ? 0 : axis_value;
int64_t axis_ptr[] = {axis_value};
axis = hrt::create(
dt_int64, {1},
{reinterpret_cast<gsl::byte *>(axis_ptr), sizeof(axis_ptr)},
true, host_runtime_tensor::pool_cpu_only)
.expect("create tensor failed");
}

void TearDown() override {}

protected:
runtime_tensor input;
runtime_tensor axis;
int64_t axis_value;
};

INSTANTIATE_TEST_SUITE_P(LogSoftmax, LogSoftmaxTest,
testing::Combine(testing::Values(dt_float32),
testing::Values(dims_t{1, 3, 16,
16})));
INSTANTIATE_TEST_SUITE_P(
LogSoftmax, LogSoftmaxTest,
testing::Combine(testing::Values(dt_float32),
testing::Values(dims_t{1}, dims_t{1, 3},
dims_t{1, 3, 16, 16}, dims_t{1, 3, 16}),
testing::Values(0, 1, 2, 3, -4, -3, -2, -1)));

TEST_P(LogSoftmaxTest, log_softmax) {
auto l_ort = runtime_tensor_2_ort_tensor(input);

// expected
auto output_ort = ortki_LogSoftmax(l_ort, -1);
auto output_ort = ortki_LogSoftmax(l_ort, axis_value);
size_t size = 0;
void *ptr_ort = tensor_buffer(output_ort, &size);
dims_t shape(tensor_rank(output_ort));
Expand All @@ -65,12 +77,6 @@ TEST_P(LogSoftmaxTest, log_softmax) {
.expect("create tensor failed");

// actual
int64_t axis_ptr[] = {-1};
auto axis =
hrt::create(dt_int64, {1},
{reinterpret_cast<gsl::byte *>(axis_ptr), sizeof(axis_ptr)},
true, host_runtime_tensor::pool_cpu_only)
.expect("create tensor failed");
auto output = kernels::stackvm::log_softmax(input.impl(), axis.impl())
.expect("log_softmax failed");
runtime_tensor actual(output.as<tensor>().expect("as tensor failed"));
Expand Down
25 changes: 15 additions & 10 deletions tests/kernels/test_softmax.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,34 +28,45 @@ using namespace ortki;

class SoftmaxTest
: public KernelTest,
public ::testing::TestWithParam<std::tuple<nncase::typecode_t, dims_t>> {
public ::testing::TestWithParam<std::tuple<typecode_t, dims_t, int64_t>> {
public:
void SetUp() override {
auto &&[typecode, l_shape] = GetParam();
auto &&[typecode, l_shape, value] = GetParam();

input =
hrt::create(typecode, l_shape, host_runtime_tensor::pool_cpu_only)
.expect("create tensor failed");
init_tensor(input);
axis_value = value >= (long)l_shape.size() ? 0 : value;
axis_value = axis_value < -(long)l_shape.size() ? 0 : axis_value;
int64_t axis_ptr[] = {axis_value};
axis = hrt::create(
dt_int64, {1},
{reinterpret_cast<gsl::byte *>(axis_ptr), sizeof(axis_ptr)},
true, host_runtime_tensor::pool_cpu_only)
.expect("create tensor failed");
}

void TearDown() override {}

protected:
runtime_tensor input;
runtime_tensor axis;
int64_t axis_value;
};

INSTANTIATE_TEST_SUITE_P(
Softmax, SoftmaxTest,
testing::Combine(testing::Values(dt_float32),
testing::Values(dims_t{1}, dims_t{1, 3},
dims_t{1, 3, 16, 16}, dims_t{1, 3, 16})));
dims_t{1, 3, 16, 16}, dims_t{1, 3, 16}),
testing::Values(0, 1, 2, 3, -4, -3, -2, -1)));

TEST_P(SoftmaxTest, Softmax) {
auto l_ort = runtime_tensor_2_ort_tensor(input);

// expected
auto output_ort = ortki_Softmax(l_ort, -1);
auto output_ort = ortki_Softmax(l_ort, axis_value);
size_t size = 0;
void *ptr_ort = tensor_buffer(output_ort, &size);
dims_t shape(tensor_rank(output_ort));
Expand All @@ -66,12 +77,6 @@ TEST_P(SoftmaxTest, Softmax) {
.expect("create tensor failed");

// actual
int32_t axis_array[] = {-1};
auto axis = hrt::create(dt_int32, {1},
{reinterpret_cast<gsl::byte *>(axis_array),
sizeof(axis_array)},
true, host_runtime_tensor::pool_cpu_only)
.expect("create tensor failed");
auto output = kernels::stackvm::softmax(input.impl(), axis.impl())
.expect("softmax failed");
runtime_tensor actual(output.as<tensor>().expect("as tensor failed"));
Expand Down

0 comments on commit fa0adb2

Please sign in to comment.