Adds support for AArch64 SVE to inline assembly

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -443,11 +443,12 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
                 .supported_types(self.arch, true)
                 .iter()
                 .map(|(ty, _)| ty.size())
+                .filter_map(InlineAsmSize::fixed_size_bytes)
                 .max()
-                .unwrap();
-            let align = rustc_abi::Align::from_bytes(reg_size.bytes()).unwrap();
+                .expect("expected fixed-size type");
+            let align = rustc_abi::Align::from_bytes(reg_size).unwrap();
             let offset = slot_size.align_to(align);
-            *slot_size = offset + reg_size;
+            *slot_size = offset + rustc_abi::Size::from_bytes(reg_size);
             offset
         };
         let mut new_slot = |x| new_slot_fn(&mut slot_size, x);

compiler/rustc_codegen_gcc/src/asm.rs

@@ -692,7 +692,9 @@ fn reg_class_to_gcc(reg_class: InlineAsmRegClass) -> &'static str {
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
         InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
+        InlineAsmRegClass::AArch64(
+            AArch64InlineAsmRegClass::preg | AArch64InlineAsmRegClass::ffr,
+        ) => {
             unreachable!("clobber-only")
         }
         InlineAsmRegClass::Amdgpu(AmdgpuInlineAsmRegClass::Sgpr(_)) => "Sg",
@@ -804,7 +806,9 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
         | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
             cx.type_vector(cx.type_i64(), 2)
         }
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
+        InlineAsmRegClass::AArch64(
+            AArch64InlineAsmRegClass::preg | AArch64InlineAsmRegClass::ffr,
+        ) => {
             unreachable!("clobber-only")
         }
         InlineAsmRegClass::Amdgpu(_) => cx.type_i32(),
@@ -1048,7 +1052,9 @@ fn modifier_to_gcc(
         | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
             if modifier == Some('v') { None } else { modifier }
         }
-        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
+        InlineAsmRegClass::AArch64(
+            AArch64InlineAsmRegClass::preg | AArch64InlineAsmRegClass::ffr,
+        ) => {
             unreachable!("clobber-only")
         }
         InlineAsmRegClass::Amdgpu(_) => None,

compiler/rustc_codegen_llvm/src/asm.rs

@@ -711,18 +711,25 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
                     format!("{{{}{}}}", class, idx)
                 }
             } else if let Some(idx) = a64_vreg_index(reg) {
-                let class = if let Some(layout) = layout {
-                    match layout.size.bytes() {
+                let class = match layout {
+                    Some(layout)
+                        if matches!(
+                            layout.backend_repr,
+                            BackendRepr::SimdScalableVector { .. }
+                        ) =>
+                    {
+                        'z'
+                    }
+                    Some(layout) => match layout.size.bytes() {
                         16 => 'q',
                         8 => 'd',
                         4 => 's',
                         2 => 'h',
                         1 => 'd', // We fixup i8 to i8x8
                         _ => unreachable!(),
-                    }
-                } else {
+                    },
                     // We use i64x2 as the type for discarded outputs
-                    'q'
+                    None => 'q',
                 };
                 format!("{{{}{}}}", class, idx)
             } else if let Some(idx) = hexagon_reg_pair_index(reg) {
@@ -744,7 +751,10 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
             AArch64(AArch64InlineAsmRegClass::reg) => "r",
             AArch64(AArch64InlineAsmRegClass::vreg) => "w",
             AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
-            AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
+            // Although the above link suggests its just 'Upa', llvm's own tests seem to suggest its
+            // '@3Upa'. (see "src/llvm-project/clang/test/CodeGen/AArch64/sve-inline-asm-datatypes.c" line 139)
+            AArch64(AArch64InlineAsmRegClass::preg) => "@3Upa",
+            AArch64(AArch64InlineAsmRegClass::ffr) => unreachable!("clobber-only"),
             Arm(ArmInlineAsmRegClass::reg) => "r",
             Arm(ArmInlineAsmRegClass::sreg)
             | Arm(ArmInlineAsmRegClass::dreg_low16)
@@ -852,7 +862,7 @@ fn modifier_to_llvm(
                 modifier
             }
         }
-        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
+        AArch64(AArch64InlineAsmRegClass::preg | AArch64InlineAsmRegClass::ffr) => None,
         Arm(ArmInlineAsmRegClass::reg) => None,
         Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
         Arm(ArmInlineAsmRegClass::dreg)
@@ -955,7 +965,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'
         AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
             cx.type_vector(cx.type_i64(), 2)
         }
-        AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
+        AArch64(AArch64InlineAsmRegClass::preg) => cx.type_scalable_vector(cx.type_i1(), 16),
+        AArch64(AArch64InlineAsmRegClass::ffr) => unreachable!("clobber-only"),
         Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
         Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
         Arm(ArmInlineAsmRegClass::dreg)

compiler/rustc_hir_typeck/src/inline_asm.rs

@@ -13,7 +13,8 @@ use rustc_session::lint;
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{ErrorGuaranteed, Span, Symbol, sym};
 use rustc_target::asm::{
-    InlineAsmReg, InlineAsmRegClass, InlineAsmRegOrRegClass, InlineAsmType, ModifierInfo,
+    InlineAsmReg, InlineAsmRegClass, InlineAsmRegOrRegClass, InlineAsmSize, InlineAsmType,
+    ModifierInfo,
 };
 use rustc_trait_selection::infer::InferCtxtExt;
 
@@ -158,6 +159,28 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                     _ => Err(NonAsmTypeReason::InvalidElement(field.did, ty)),
                 }
             }
+            ty::Adt(adt, _args) if adt.repr().scalable() => {
+                let (_element_count, elem_ty, _number_of_vectors) =
+                    ty.scalable_vector_parts(self.tcx()).unwrap();
+
+                match elem_ty.kind() {
+                    ty::Int(IntTy::I8) | ty::Uint(UintTy::U8) => Ok(InlineAsmType::SveVecI8),
+                    ty::Int(IntTy::I16) | ty::Uint(UintTy::U16) => Ok(InlineAsmType::SveVecI16),
+                    ty::Int(IntTy::I32) | ty::Uint(UintTy::U32) => Ok(InlineAsmType::SveVecI32),
+                    ty::Int(IntTy::I64) | ty::Uint(UintTy::U64) => Ok(InlineAsmType::SveVecI64),
+                    ty::Int(IntTy::I128) | ty::Uint(UintTy::U128) => Ok(InlineAsmType::SveVecI128),
+                    ty::Float(FloatTy::F16) => Ok(InlineAsmType::SveVecF16),
+                    ty::Float(FloatTy::F32) => Ok(InlineAsmType::SveVecF32),
+                    ty::Float(FloatTy::F64) => Ok(InlineAsmType::SveVecF64),
+                    ty::Float(FloatTy::F128) => Ok(InlineAsmType::SveVecF128),
+                    ty::Bool => Ok(InlineAsmType::SveVecBool),
+                    _ => {
+                        let fields = &adt.non_enum_variant().fields;
+                        let field = &fields[FieldIdx::ZERO];
+                        Err(NonAsmTypeReason::InvalidElement(field.did, ty))
+                    }
+                }
+            }
             ty::Infer(_) => bug!("unexpected infer ty in asm operand"),
             _ => Err(NonAsmTypeReason::Invalid(ty)),
         }
@@ -177,10 +200,10 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
             idx: usize,
             suggested_modifier: char,
             suggested_result: &'a str,
-            suggested_size: u16,
+            suggested_size: InlineAsmSize,
             default_modifier: char,
             default_result: &'a str,
-            default_size: u16,
+            default_size: InlineAsmSize,
         }
 
         impl<'a, 'b> Diagnostic<'a, ()> for FormattingSubRegisterArg<'b> {
@@ -195,13 +218,24 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                     default_result,
                     default_size,
                 } = self;
+
+                fn format_size(size: InlineAsmSize) -> String {
+                    match size {
+                        InlineAsmSize::FixedBytes(size) => format!("{size}-byte values"),
+                        InlineAsmSize::Scalable => "scalable values".to_string(),
+                    }
+                }
                 Diag::new(dcx, level, "formatting may not be suitable for sub-register argument")
                     .with_span_label(expr_span, "for this argument")
                     .with_help(format!(
-                        "use `{{{idx}:{suggested_modifier}}}` to have the register formatted as `{suggested_result}` (for {suggested_size}-bit values)",
+                        "use `{{{idx}:{suggested_modifier}}}` to have the register formatted as \
+                        `{suggested_result}` (for {})",
+                        format_size(suggested_size)
                     ))
                     .with_help(format!(
-                        "or use `{{{idx}:{default_modifier}}}` to keep the default formatting of `{default_result}` (for {default_size}-bit values)",
+                        "or use `{{{idx}:{default_modifier}}}` to keep the default formatting of \
+                        `{default_result}` (for {})",
+                        format_size(default_size)
                     ))
             }
         }
@@ -239,8 +273,8 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                     NonAsmTypeReason::Invalid(ty) => {
                         let msg = format!("cannot use value of type `{ty}` for inline assembly");
                         self.fcx.dcx().struct_span_err(expr.span, msg).with_note(
-                            "only integers, floats, SIMD vectors, pointers and function pointers \
-                            can be used as arguments for inline assembly",
+                            "only integers, floats, SIMD vectors, scalable vectors, pointers and function \
+                            pointers can be used as arguments for inline assembly",
                         ).emit();
                     }
                     NonAsmTypeReason::NotSizedPtr(ty) => {

compiler/rustc_span/src/symbol.rs

@@ -965,6 +965,7 @@ symbols! {
         ffi_const,
         ffi_pure,
         ffi_returns_twice,
+        ffr,
         field,
         field_base,
         field_init_shorthand,
@@ -2082,6 +2083,7 @@ symbols! {
         suggestion,
         super_let,
         supertrait_item_shadowing,
+        sve,
         sve_cast,
         sve_tuple_create2,
         sve_tuple_create3,

compiler/rustc_target/src/asm/aarch64.rs

@@ -1,9 +1,10 @@
+use core::convert::Into;
 use std::fmt;
 
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_span::{Symbol, sym};
 
-use super::{InlineAsmArch, InlineAsmType, ModifierInfo};
+use super::{InlineAsmArch, InlineAsmSize, InlineAsmType, ModifierInfo};
 use crate::spec::{Env, Os, RelocModel, Target};
 
 def_reg_class! {
@@ -12,15 +13,16 @@ def_reg_class! {
         vreg,
         vreg_low16,
         preg,
+        ffr,
     }
 }
 
 impl AArch64InlineAsmRegClass {
     pub fn valid_modifiers(self, _arch: super::InlineAsmArch) -> &'static [char] {
         match self {
             Self::reg => &['w', 'x'],
-            Self::vreg | Self::vreg_low16 => &['b', 'h', 's', 'd', 'q', 'v'],
-            Self::preg => &[],
+            Self::vreg | Self::vreg_low16 => &['b', 'h', 's', 'd', 'q', 'v', 'z'],
+            Self::preg | Self::ffr => &[],
         }
     }
 
@@ -30,43 +32,67 @@ impl AArch64InlineAsmRegClass {
 
     pub fn suggest_modifier(self, _arch: InlineAsmArch, ty: InlineAsmType) -> Option<ModifierInfo> {
         match self {
-            Self::reg => match ty.size().bits() {
-                64 => None,
-                _ => Some(('w', "w0", 32).into()),
+            Self::reg => match ty.size() {
+                InlineAsmSize::FixedBytes(8) => None,
+                _ => Some(('w', "w0", InlineAsmSize::FixedBytes(4)).into()),
             },
-            Self::vreg | Self::vreg_low16 => match ty.size().bits() {
-                8 => Some(('b', "b0", 8).into()),
-                16 => Some(('h', "h0", 16).into()),
-                32 => Some(('s', "s0", 32).into()),
-                64 => Some(('d', "d0", 64).into()),
-                128 => Some(('q', "q0", 128).into()),
+            Self::vreg | Self::vreg_low16 => match ty.size() {
+                InlineAsmSize::FixedBytes(1) => Some(('b', "b0", ty.size()).into()),
+                InlineAsmSize::FixedBytes(2) => Some(('h', "h0", ty.size()).into()),
+                InlineAsmSize::FixedBytes(4) => Some(('s', "s0", ty.size()).into()),
+                InlineAsmSize::FixedBytes(8) => Some(('d', "d0", ty.size()).into()),
+                InlineAsmSize::FixedBytes(16) => Some(('q', "q0", ty.size()).into()),
+                InlineAsmSize::Scalable => Some(('z', "z0", InlineAsmSize::Scalable).into()),
                 _ => None,
             },
-            Self::preg => None,
+            Self::preg | Self::ffr => None,
         }
     }
 
     pub fn default_modifier(self, _arch: InlineAsmArch) -> Option<ModifierInfo> {
         match self {
-            Self::reg => Some(('x', "x0", 64).into()),
-            Self::vreg | Self::vreg_low16 => Some(('v', "v0", 128).into()),
-            Self::preg => None,
+            Self::reg => Some(('x', "x0", InlineAsmSize::FixedBytes(8)).into()),
+            Self::vreg | Self::vreg_low16 => {
+                Some(('v', "v0", InlineAsmSize::FixedBytes(16)).into())
+            }
+            Self::preg | Self::ffr => None,
         }
     }
 
     pub fn supported_types(
         self,
         _arch: InlineAsmArch,
+        allow_experimental_reg: bool,
     ) -> &'static [(InlineAsmType, Option<Symbol>)] {
         match self {
             Self::reg => types! { _: I8, I16, I32, I64, F16, F32, F64; },
-            Self::vreg | Self::vreg_low16 => types! {
-                neon: I8, I16, I32, I64, F16, F32, F64, F128,
-                    VecI8(8), VecI16(4), VecI32(2), VecI64(1), VecF16(4), VecF32(2), VecF64(1),
-                    VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF16(8), VecF32(4), VecF64(2);
-                // Note: When adding support for SVE vector types, they must be rejected for Arm64EC.
-            },
-            Self::preg => &[],
+            Self::vreg | Self::vreg_low16 => {
+                if allow_experimental_reg {
+                    types! {
+                        neon: I8, I16, I32, I64, F16, F32, F64, F128,
+                            VecI8(8), VecI16(4), VecI32(2), VecI64(1), VecF16(4), VecF32(2), VecF64(1),
+                            VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF16(8), VecF32(4), VecF64(2);
+                        sve: SveVecI8, SveVecI16, SveVecI32, SveVecI64, SveVecI128, SveVecF16, SveVecF32,
+                            SveVecF64, SveVecI128, SveVecF128;
+                    }
+                } else {
+                    types! {
+                        neon: I8, I16, I32, I64, F16, F32, F64, F128,
+                            VecI8(8), VecI16(4), VecI32(2), VecI64(1), VecF16(4), VecF32(2), VecF64(1),
+                            VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF16(8), VecF32(4), VecF64(2);
+                    }
+                }
+            }
+            Self::preg => {
+                if allow_experimental_reg {
+                    types! {
+                        sve: SveVecBool;
+                    }
+                } else {
+                    &[]
+                }
+            }
+            Self::ffr => &[],
         }
     }
 }
@@ -190,7 +216,7 @@ def_regs! {
         p13: preg = ["p13"] % restricted_for_arm64ec,
         p14: preg = ["p14"] % restricted_for_arm64ec,
         p15: preg = ["p15"] % restricted_for_arm64ec,
-        ffr: preg = ["ffr"] % restricted_for_arm64ec,
+        ffr: ffr = ["ffr"] % restricted_for_arm64ec,
         #error = ["x19", "w19"] =>
             "x19 is used internally by LLVM and cannot be used as an operand for inline asm",
         #error = ["x29", "w29", "fp", "wfp"] =>

compiler/rustc_target/src/asm/amdgpu.rs

@@ -168,7 +168,7 @@ impl AmdgpuInlineAsmRegClass {
             return None;
         }
 
-        Some(Self::Vgpr(ty.size().bits().try_into().ok()?))
+        Some(Self::Vgpr(ty.size().fixed_size_bytes().map(|byte| byte * 8)?.try_into().ok()?))
     }
 
     pub fn suggest_modifier(