attach global target features to module-level assembly

compiler/rustc_codegen_cranelift/src/global_asm.rs

@@ -30,6 +30,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for GlobalAsmContext<'_, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         _line_spans: &[Span],
+        _target_features: &[String],
     ) {
         codegen_global_asm_inner(self.tcx, self.global_asm, template, operands, options);
     }

compiler/rustc_codegen_gcc/src/asm.rs

@@ -928,6 +928,7 @@ impl<'gcc, 'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         line_spans: &[Span],
+        _target_features: &[String],
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 

compiler/rustc_codegen_llvm/src/asm.rs

@@ -16,12 +16,12 @@ use rustc_target::spec::HasTargetSpec;
 use smallvec::SmallVec;
 use tracing::debug;
 
-use crate::attributes;
 use crate::builder::Builder;
 use crate::common::Funclet;
 use crate::context::CodegenCx;
 use crate::llvm::{self, ToLlvmBool, Type, Value};
 use crate::type_of::LayoutLlvmExt;
+use crate::{attributes, llvm_util};
 
 impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     fn codegen_inline_asm(
@@ -414,6 +414,7 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         _line_spans: &[Span],
+        target_features: &[String],
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -499,7 +500,12 @@ impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
             template_str.push_str("\n.att_syntax\n");
         }
 
-        llvm::append_module_inline_asm(self.llmod, template_str.as_bytes());
+        llvm::append_module_inline_asm(
+            self.llmod,
+            template_str.as_bytes(),
+            &target_features.join(","),
+            llvm_util::target_cpu(self.tcx.sess),
+        );
     }
 
     fn mangled_name(&self, instance: Instance<'tcx>) -> String {

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -1303,9 +1303,9 @@ fn embed_bitcode(
         // We need custom section flags, so emit module-level inline assembly.
         let section_flags = if cgcx.is_pe_coff { "n" } else { "e" };
         let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode);
-        llvm::append_module_inline_asm(llmod, &asm);
+        llvm::append_module_inline_asm(llmod, &asm, "", "");
         let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]);
-        llvm::append_module_inline_asm(llmod, &asm);
+        llvm::append_module_inline_asm(llmod, &asm, "", "");
     }
 }
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -907,13 +907,6 @@ unsafe extern "C" {
     pub(crate) fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char;
     pub(crate) fn LLVMSetDataLayout(M: &Module, Triple: *const c_char);
 
-    /// Append inline assembly to a module. See `Module::appendModuleInlineAsm`.
-    pub(crate) fn LLVMAppendModuleInlineAsm(
-        M: &Module,
-        Asm: *const c_uchar, // See "PTR_LEN_STR".
-        Len: size_t,
-    );
-
     /// Create the specified uniqued inline asm string. See `InlineAsm::get()`.
     pub(crate) fn LLVMGetInlineAsm<'ll>(
         Ty: &'ll Type,
@@ -2119,6 +2112,17 @@ unsafe extern "C" {
         ConstraintsLen: size_t,
     ) -> bool;
 
+    /// Append inline assembly to a module. See `Module::appendModuleInlineAsm`.
+    pub(crate) fn LLVMRustAppendModuleInlineAsm(
+        M: &Module,
+        Asm: *const c_uchar, // See "PTR_LEN_STR".
+        AsmLen: size_t,
+        TargetFeatures: *const c_uchar, // See "PTR_LEN_STR".
+        TargetFeaturesLen: size_t,
+        TargetCpu: *const c_uchar, // See "PTR_LEN_STR".
+        TargetCpuLen: size_t,
+    );
+
     /// A list of pointer-length strings is passed as two pointer-length slices,
     /// one slice containing pointers and one slice containing their corresponding
     /// lengths. The implementation will check that both slices have the same length.

compiler/rustc_codegen_llvm/src/llvm/mod.rs

@@ -474,11 +474,24 @@ pub(crate) fn set_dso_local<'ll>(v: &'ll Value) {
     }
 }
 
-/// Safe wrapper for `LLVMAppendModuleInlineAsm`, which delegates to
+/// Safe wrapper for `LLVMRustAppendModuleInlineAsm`, which delegates to
 /// `Module::appendModuleInlineAsm`.
-pub(crate) fn append_module_inline_asm<'ll>(llmod: &'ll Module, asm: &[u8]) {
+pub(crate) fn append_module_inline_asm<'ll>(
+    llmod: &'ll Module,
+    asm: &[u8],
+    target_features: &str,
+    target_cpu: &str,
+) {
     unsafe {
-        LLVMAppendModuleInlineAsm(llmod, asm.as_ptr(), asm.len());
+        LLVMRustAppendModuleInlineAsm(
+            llmod,
+            asm.as_ptr(),
+            asm.len(),
+            target_features.as_ptr(),
+            target_features.len(),
+            target_cpu.as_ptr(),
+            target_cpu.len(),
+        );
     }
 }
 

compiler/rustc_codegen_ssa/src/base.rs

@@ -490,7 +490,14 @@ where
             })
             .collect();
 
-        cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans);
+        let target_features = cx.tcx().global_backend_features(());
+        cx.codegen_global_asm(
+            asm.template,
+            &operands,
+            asm.options,
+            asm.line_spans,
+            &target_features,
+        );
     } else {
         span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type")
     }

compiler/rustc_codegen_ssa/src/mir/naked_asm.rs

@@ -54,7 +54,9 @@ pub fn codegen_naked_asm<
     template_vec.extend(template.iter().cloned());
     template_vec.push(rustc_ast::ast::InlineAsmTemplatePiece::String(end.into()));
 
-    cx.codegen_global_asm(&template_vec, &operands, options, line_spans);
+    let target_features: Vec<_> =
+        cx.tcx().asm_target_features(instance.def_id()).iter().map(|s| format!("+{s}")).collect();
+    cx.codegen_global_asm(&template_vec, &operands, options, line_spans, &target_features);
 }
 
 fn inline_to_global_operand<'a, 'tcx, Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>>(

compiler/rustc_codegen_ssa/src/traits/asm.rs

@@ -72,6 +72,7 @@ pub trait AsmCodegenMethods<'tcx> {
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,
         line_spans: &[Span],
+        target_features: &[String],
     );
 
     /// The mangled name of this instance

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -661,6 +661,20 @@ extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints,
       unwrap<FunctionType>(Ty), StringRef(Constraints, ConstraintsLen)));
 }
 
+extern "C" void LLVMRustAppendModuleInlineAsm(
+    LLVMModuleRef M, const char *Asm, size_t AsmLen, const char *TargetFeatures,
+    size_t TargetFeaturesLen, const char *TargetCPU, size_t TargetCPULen) {
+#if LLVM_VERSION_GE(23, 0)
+  Module::GlobalAsmProperties Props;
+  Props.TargetFeatures = std::string(TargetFeatures, TargetFeaturesLen);
+  Props.TargetCPU = std::string(TargetCPU, TargetCPULen);
+  unwrap(M)->appendModuleInlineAsm(
+      Module::GlobalAsmFragment(std::string(Asm, AsmLen), Props));
+#else
+  unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen));
+#endif
+}
+
 template <typename DIT> DIT *unwrapDIPtr(LLVMMetadataRef Ref) {
   return (DIT *)(Ref ? unwrap<Metadata>(Ref) : nullptr);
 }

tests/assembly-llvm/naked-functions/target-feature.rs

@@ -0,0 +1,169 @@
+//@ revisions: aarch64-elf aarch64-macho aarch64-coff x86_64 s390x riscv64 powerpc64 loongarch64
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ [x86_64] compile-flags: --target x86_64-unknown-linux-gnu
+//@ [x86_64] needs-llvm-components: x86
+//
+//@ [aarch64-elf] compile-flags: --target aarch64-unknown-linux-gnu
+//@ [aarch64-elf] needs-llvm-components: aarch64
+//@ [aarch64-macho] compile-flags: --target aarch64-apple-darwin
+//@ [aarch64-macho] needs-llvm-components: aarch64
+//@ [aarch64-coff] compile-flags: --target aarch64-pc-windows-gnullvm
+//@ [aarch64-coff] needs-llvm-components: aarch64
+//
+//@ [s390x] compile-flags: --target s390x-unknown-linux-gnu
+//@ [s390x] needs-llvm-components: systemz
+//
+//@ [powerpc64] compile-flags: --target powerpc64-unknown-linux-gnu
+//@ [powerpc64] needs-llvm-components: powerpc
+//
+//@ [riscv64] compile-flags: --target riscv64gc-unknown-linux-gnu
+//@ [riscv64] needs-llvm-components: riscv
+//
+// NOTE: loongarch64 does not error when using an instruction without enabling the corresponding
+// target feature.
+//@ [loongarch64] compile-flags: --target loongarch64-unknown-linux-gnu
+//@ [loongarch64] needs-llvm-components: loongarch
+
+// Test that the #[target_feature(enable = ...)]` works on naked functions.
+
+#![crate_type = "lib"]
+#![feature(no_core, naked_functions_target_feature, asm_experimental_arch)]
+#![feature(
+    s390x_target_feature,
+    powerpc_target_feature,
+    loongarch_target_feature,
+    m68k_target_feature
+)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// x86_64-LABEL: vpclmulqdq:
+// x86_64: vpclmulqdq
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "vpclmulqdq")]
+unsafe extern "C" fn vpclmulqdq() {
+    naked_asm!("vpclmulqdq zmm1, zmm2, zmm3, 4")
+}
+
+// i8mm is not enabled by default
+//
+// note that aarch64-apple-darwin enables more features than aarch64-unknown-linux-gnu
+//
+// aarch64-elf-LABEL: i8mm:
+// aarch64-elf: usdot
+// aarch64-macho-LABEL: i8mm:
+// aarch64-macho: usdot
+// aarch64-coff-LABEL: i8mm:
+// aarch64-coff: usdot
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "i8mm")]
+unsafe extern "C" fn i8mm() {
+    naked_asm!("usdot   v0.4s, v1.16b, v2.4b[3]")
+}
+
+// riscv64: sh1add:
+// riscv64: sh1add
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "riscv64")]
+#[target_feature(enable = "zba")]
+unsafe extern "C" fn sh1add() {
+    naked_asm!("sh1add a0, a1, a2", "ret");
+}
+
+#[cfg(target_arch = "s390x")]
+mod s390x {
+    use super::*;
+
+    // s390x: vector:
+    // s390x: vavglg
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector")]
+    unsafe extern "C" fn vector() {
+        naked_asm!("vavglg  %v0, %v0, %v0")
+    }
+
+    // s390x: vector_enhancements_1:
+    // s390x: vfcesbs
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-enhancements-1")]
+    unsafe extern "C" fn vector_enhancements_1() {
+        naked_asm!("vfcesbs %v0, %v0, %v0")
+    }
+
+    // s390x: vector_enhancements_2:
+    // s390x: vclfp
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-enhancements-2")]
+    unsafe extern "C" fn vector_enhancements_2() {
+        naked_asm!("vclfp   %v0, %v0, 0, 0, 0")
+    }
+
+    // s390x: vector_packed_decimal:
+    // s390x: vlrlr
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal")]
+    unsafe extern "C" fn vector_packed_decimal() {
+        naked_asm!("vlrlr   %v24, %r3, 0(%r2)", "br      %r14")
+    }
+
+    // s390x: vector_packed_decimal_enhancement:
+    // s390x: vcvbg
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal-enhancement")]
+    unsafe extern "C" fn vector_packed_decimal_enhancement() {
+        naked_asm!("vcvbg   %r0, %v0, 0, 1")
+    }
+
+    // s390x: vector_packed_decimal_enhancement_2:
+    // s390x: vupkzl
+    #[no_mangle]
+    #[unsafe(naked)]
+    #[target_feature(enable = "vector-packed-decimal-enhancement-2")]
+    unsafe extern "C" fn vector_packed_decimal_enhancement_2() {
+        naked_asm!("vupkzl  %v0, %v0, 0")
+    }
+}
+
+// powerpc64: power10_vector:
+// powerpc64: xxpermx
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "powerpc64")]
+#[target_feature(enable = "power10-vector")]
+unsafe extern "C" fn power10_vector() {
+    naked_asm!("xxpermx 34, 0, 1, 2, 0", "blr")
+}
+
+// loongarch64: lasx:
+// loongarch64: xvadd.b
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "loongarch64")]
+#[target_feature(enable = "lasx")]
+unsafe extern "C" fn lasx() {
+    naked_asm!("xvadd.b  $xr0, $xr0, $xr1", "ret")
+}
+
+// wasm32: simd128:
+// wasm32: i8x16.shuffle
+#[no_mangle]
+#[unsafe(naked)]
+#[cfg(target_arch = "wasm32")]
+#[target_feature(enable = "simd128")]
+unsafe extern "C" fn simd128() {
+    naked_asm!("i8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15", "return");
+}

tests/ui/asm/global-target-feature.rs

@@ -0,0 +1,49 @@
+//@ build-pass
+//@ add-minicore
+//@ min-llvm-version: 23
+//@ ignore-backends: gcc
+//
+//@ revisions: riscv opt-0-bitcode-no opt-0 opt-s-bitcode-no
+//
+//@[riscv] compile-flags: --target riscv64gc-unknown-linux-gnu -Clto=thin
+//@[riscv] needs-llvm-components: riscv
+//
+//@[opt-0-bitcode-no] compile-flags: --target armv7r-none-eabihf -Copt-level=0 -Cembed-bitcode=no
+//@[opt-0-bitcode-no] needs-llvm-components: arm
+//
+//@[opt-0] compile-flags: --target armv7r-none-eabihf -Copt-level=0
+//@[opt-0] needs-llvm-components: arm
+//
+//@[opt-s-bitcode-no] compile-flags: --target armv7r-none-eabihf -Copt-level=s -Cembed-bitcode=no
+//@[opt-s-bitcode-no] needs-llvm-components: arm
+
+// Regression test for
+//
+// - https://github.com/llvm/llvm-project/issues/61991
+// - https://github.com/rust-lang/rust/issues/80608
+// - https://github.com/rust-lang/rust/issues/127269
+//
+// Since LLVM 23 target features are taken into account for module-level assembly.
+
+#![feature(no_core)]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+#[cfg(target_arch = "riscv64")]
+global_asm!("fld f0, 0(sp)");
+
+#[cfg(target_arch = "arm")]
+global_asm!(
+    r#"
+.section .text.startup
+.global _start
+.code 32
+.align 0
+
+_start:
+    vmsr fpexc, r0
+"#
+);