diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle index 61f164a3ac2e..dedc989e8bdb 100644 --- a/cranelift/codegen/src/isa/aarch64/inst.isle +++ b/cranelift/codegen/src/isa/aarch64/inst.isle @@ -2428,6 +2428,11 @@ (if-let addr_reg (amode_is_reg addr)) addr_reg) +;; Lower a constant f64. +(decl constant_f64 (u64) Reg) +;; TODO: Port lower_constant_f64() to ISLE. +(extern constructor constant_f64 constant_f64) + ;; Lower a constant f128. (decl constant_f128 (u128) Reg) ;; TODO: Port lower_constant_f128() to ISLE. diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index 0cee3acc08a0..b1dd8fcb3b5e 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -1628,6 +1628,15 @@ (rule (lower (resumable_trap trap_code)) (side_effect (udf trap_code))) +;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule (lower (has_type (ty_vec128 _) (vconst (u128_from_constant x)))) + (constant_f128 x)) + +(rule (lower (has_type ty (vconst (u64_from_constant x)))) + (if (ty_vec64 ty)) + (constant_f64 x)) + ;;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule (lower (has_type ty (splat x @ (value_type in_ty)))) diff --git a/cranelift/codegen/src/isa/aarch64/lower.rs b/cranelift/codegen/src/isa/aarch64/lower.rs index 3854b786bd80..5235db60b43d 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.rs +++ b/cranelift/codegen/src/isa/aarch64/lower.rs @@ -8,7 +8,6 @@ //! - Floating-point immediates (FIMM instruction). use super::lower_inst; -use crate::data_value::DataValue; use crate::ir::condcodes::{FloatCC, IntCC}; use crate::ir::types::*; use crate::ir::Inst as IRInst; @@ -94,13 +93,6 @@ pub(crate) fn input_to_shiftimm( input_to_const(ctx, input).and_then(ShiftOpShiftImm::maybe_from_shift) } -pub(crate) fn const_param_to_u128(ctx: &mut Lower, inst: IRInst) -> Option { - match ctx.get_immediate(inst) { - Some(DataValue::V128(bytes)) => Some(u128::from_le_bytes(bytes)), - _ => None, - } -} - /// How to handle narrow values loaded into registers; see note on `narrow_mode` /// parameter to `put_input_in_*` below. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs index a0252e5dbe9c..35bbefec8776 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs @@ -5,9 +5,9 @@ pub mod generated_code; // Types that the generated ISLE code uses via `use super::*`. use super::{ - insn_inputs, lower_constant_f128, writable_zero_reg, zero_reg, AMode, ASIMDFPModImm, - ASIMDMovModImm, BranchTarget, CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, FPUOpRI, - FloatCC, Imm12, ImmLogic, ImmShift, Inst as MInst, IntCC, JTSequenceInfo, MachLabel, + insn_inputs, lower_constant_f128, lower_constant_f64, writable_zero_reg, zero_reg, AMode, + ASIMDFPModImm, ASIMDMovModImm, BranchTarget, CallIndInfo, CallInfo, Cond, CondBrKind, ExtendOp, + FPUOpRI, FloatCC, Imm12, ImmLogic, ImmShift, Inst as MInst, IntCC, JTSequenceInfo, MachLabel, MoveWideConst, MoveWideOp, NarrowValueMode, Opcode, OperandSize, PairAMode, Reg, ScalarSize, ShiftOpAndAmt, UImm5, VecMisc2, VectorSize, NZCV, }; @@ -484,6 +484,14 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, Flags, IsaFlags, 6> address.is_reg() } + fn constant_f64(&mut self, value: u64) -> Reg { + let rd = self.temp_writable_reg(I8X16); + + lower_constant_f64(self.lower_ctx, rd, f64::from_bits(value)); + + rd.to_reg() + } + fn constant_f128(&mut self, value: u128) -> Reg { let rd = self.temp_writable_reg(I8X16); diff --git a/cranelift/codegen/src/isa/aarch64/lower_inst.rs b/cranelift/codegen/src/isa/aarch64/lower_inst.rs index 69e795c01846..3d25fdbe7752 100644 --- a/cranelift/codegen/src/isa/aarch64/lower_inst.rs +++ b/cranelift/codegen/src/isa/aarch64/lower_inst.rs @@ -662,11 +662,7 @@ pub(crate) fn lower_insn_to_regs( panic!("Branch opcode reached non-branch lowering logic!"); } - Opcode::Vconst => { - let value = const_param_to_u128(ctx, insn).expect("Invalid immediate bytes"); - let rd = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - lower_constant_f128(ctx, rd, value); - } + Opcode::Vconst => implemented_in_isle(ctx), Opcode::RawBitcast => { let rm = put_input_in_reg(ctx, inputs[0], NarrowValueMode::None); diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 67e8df831c4b..257df85de0e8 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -683,6 +683,12 @@ macro_rules! isle_prelude_methods { Some(u128::from_le_bytes(bytes.try_into().ok()?)) } + #[inline] + fn u64_from_constant(&mut self, constant: Constant) -> Option { + let bytes = self.lower_ctx.get_constant_data(constant).as_slice(); + Some(u64::from_le_bytes(bytes.try_into().ok()?)) + } + #[inline] fn u128_from_constant(&mut self, constant: Constant) -> Option { let bytes = self.lower_ctx.get_constant_data(constant).as_slice(); diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs index c8b0492ec7c6..96e5a3fe40f6 100644 --- a/cranelift/codegen/src/machinst/lower.rs +++ b/cranelift/codegen/src/machinst/lower.rs @@ -1387,15 +1387,23 @@ impl<'func, I: VCodeInst> Lower<'func, I> { let inst_data = self.data(ir_inst); match inst_data { InstructionData::Shuffle { imm, .. } => { - let buffer = self.f.dfg.immediates.get(imm.clone()).unwrap().as_slice(); - let value = DataValue::V128(buffer.try_into().expect("a 16-byte data buffer")); + let mask = self.f.dfg.immediates.get(imm.clone()).unwrap().as_slice(); + let value = match mask.len() { + 16 => DataValue::V128(mask.try_into().expect("a 16-byte vector mask")), + 8 => DataValue::V64(mask.try_into().expect("an 8-byte vector mask")), + length => panic!("unexpected Shuffle mask length {}", length), + }; Some(value) } InstructionData::UnaryConst { constant_handle, .. } => { let buffer = self.f.dfg.constants.get(constant_handle.clone()).as_slice(); - let value = DataValue::V128(buffer.try_into().expect("a 16-byte data buffer")); + let value = match buffer.len() { + 16 => DataValue::V128(buffer.try_into().expect("a 16-byte data buffer")), + 8 => DataValue::V64(buffer.try_into().expect("an 8-byte data buffer")), + length => panic!("unexpected UnaryConst buffer length {}", length), + }; Some(value) } _ => inst_data.imm_value(), diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index cdb47de4493f..e75287f9a5b0 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -799,6 +799,11 @@ (decl u128_from_constant (u128) Constant) (extern extractor u128_from_constant u128_from_constant) +;; Accessor for `Constant` as u64. + +(decl u64_from_constant (u64) Constant) +(extern extractor u64_from_constant u64_from_constant) + ;;;; Helpers for tail recursion loops ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/cranelift/filetests/filetests/runtests/simd-vconst-64bit.clif b/cranelift/filetests/filetests/runtests/simd-vconst-64bit.clif new file mode 100644 index 000000000000..d81aeaaecd73 --- /dev/null +++ b/cranelift/filetests/filetests/runtests/simd-vconst-64bit.clif @@ -0,0 +1,39 @@ +test interpret +test run +target aarch64 +; x86_64 and s390x do not support 64-bit vectors. + +function %vconst_zeroes() -> i8x8 { +block0: + v0 = vconst.i8x8 0x00 + return v0 +} +; run: %vconst_zeroes() == [0 0 0 0 0 0 0 0] + +function %vconst_ones() -> i8x8 { +block0: + v0 = vconst.i8x8 0xffffffffffffffff + return v0 +} +; run: %vconst_ones() == [255 255 255 255 255 255 255 255] + +function %vconst_i8x8() -> i8x8 { +block0: + v0 = vconst.i8x8 [0 31 63 95 127 159 191 255] + return v0 +} +; run: %vconst_i8x8() == [0 31 63 95 127 159 191 255] + +function %vconst_i16x4() -> i16x4 { +block0: + v0 = vconst.i16x4 [0 255 32767 65535] + return v0 +} +; run: %vconst_i16x4() == [0 255 32767 65535] + +function %vconst_i32x2() -> i32x2 { +block0: + v0 = vconst.i32x2 [0 4294967295] + return v0 +} +; run: %vconst_i32x2() == [0 4294967295] diff --git a/cranelift/interpreter/src/step.rs b/cranelift/interpreter/src/step.rs index 8860c924e25a..8b3860acdce4 100644 --- a/cranelift/interpreter/src/step.rs +++ b/cranelift/interpreter/src/step.rs @@ -75,7 +75,11 @@ where .constants .get(constant_handle.clone()) .as_slice(); - DataValue::V128(buffer.try_into().expect("a 16-byte data buffer")) + match ctrl_ty.bytes() { + 16 => DataValue::V128(buffer.try_into().expect("a 16-byte data buffer")), + 8 => DataValue::V64(buffer.try_into().expect("an 8-byte data buffer")), + length => panic!("unexpected UnaryConst buffer length {}", length), + } } InstructionData::Shuffle { imm, .. } => { let mask = state @@ -85,7 +89,11 @@ where .get(imm) .unwrap() .as_slice(); - DataValue::V128(mask.try_into().expect("a 16-byte vector mask")) + match ctrl_ty.bytes() { + 16 => DataValue::V128(mask.try_into().expect("a 16-byte vector mask")), + 8 => DataValue::V64(mask.try_into().expect("an 8-byte vector mask")), + length => panic!("unexpected Shuffle mask length {}", length), + } } _ => inst.imm_value().unwrap(), })