From efcae7d31d30ba8d8c806fbb8dea634e78d7b969 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 19 Jul 2025 22:38:57 +0200 Subject: [PATCH 1/2] properly use caller-side panic location for some GenericArgs methods --- compiler/rustc_middle/src/ty/generic_args.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs index 7d34d8df3f3b9..a5fd4d7a262c5 100644 --- a/compiler/rustc_middle/src/ty/generic_args.rs +++ b/compiler/rustc_middle/src/ty/generic_args.rs @@ -536,21 +536,28 @@ impl<'tcx> GenericArgs<'tcx> { #[inline] #[track_caller] pub fn type_at(&self, i: usize) -> Ty<'tcx> { - self[i].as_type().unwrap_or_else(|| bug!("expected type for param #{} in {:?}", i, self)) + self[i].as_type().unwrap_or_else( + #[track_caller] + || bug!("expected type for param #{} in {:?}", i, self), + ) } #[inline] #[track_caller] pub fn region_at(&self, i: usize) -> ty::Region<'tcx> { - self[i] - .as_region() - .unwrap_or_else(|| bug!("expected region for param #{} in {:?}", i, self)) + self[i].as_region().unwrap_or_else( + #[track_caller] + || bug!("expected region for param #{} in {:?}", i, self), + ) } #[inline] #[track_caller] pub fn const_at(&self, i: usize) -> ty::Const<'tcx> { - self[i].as_const().unwrap_or_else(|| bug!("expected const for param #{} in {:?}", i, self)) + self[i].as_const().unwrap_or_else( + #[track_caller] + || bug!("expected const for param #{} in {:?}", i, self), + ) } #[inline] From de1b999ff6c981475e4491ea2fff1851655587e5 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sat, 19 Jul 2025 23:23:40 +0200 Subject: [PATCH 2/2] atomicrmw on pointers: move integer-pointer cast hacks into backend --- .../src/intrinsics/mod.rs | 24 ++--- compiler/rustc_codegen_gcc/src/builder.rs | 7 +- compiler/rustc_codegen_llvm/src/builder.rs | 14 ++- compiler/rustc_codegen_ssa/messages.ftl | 2 + compiler/rustc_codegen_ssa/src/errors.rs | 8 ++ .../rustc_codegen_ssa/src/mir/intrinsic.rs | 82 +++++++++++++---- .../rustc_codegen_ssa/src/traits/builder.rs | 3 + .../rustc_hir_analysis/src/check/intrinsic.rs | 12 +-- library/core/src/intrinsics/mod.rs | 30 +++---- library/core/src/sync/atomic.rs | 89 ++++++++++--------- src/tools/miri/src/intrinsics/atomic.rs | 19 ++-- src/tools/miri/src/operator.rs | 8 +- tests/codegen-llvm/atomicptr.rs | 6 +- .../atomic-lock-free/atomic_lock_free.rs | 39 +++++--- tests/ui/intrinsics/intrinsic-atomics.rs | 12 +-- tests/ui/intrinsics/non-integer-atomic.rs | 32 +++---- tests/ui/intrinsics/non-integer-atomic.stderr | 32 +++---- 17 files changed, 243 insertions(+), 176 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 4ff5773a06cb2..ed40901ac9b8b 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -969,7 +969,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = amount.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -982,7 +982,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_xsub => { @@ -991,7 +991,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = amount.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1004,7 +1004,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_and => { @@ -1013,7 +1013,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = src.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1025,7 +1025,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_or => { @@ -1034,7 +1034,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = src.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1046,7 +1046,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_xor => { @@ -1055,7 +1055,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = src.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1067,7 +1067,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_nand => { @@ -1076,7 +1076,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = src.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1088,7 +1088,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_max => { diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index a4ec4bf8deac4..032f5d0a622d8 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -1656,6 +1656,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { dst: RValue<'gcc>, src: RValue<'gcc>, order: AtomicOrdering, + ret_ptr: bool, ) -> RValue<'gcc> { let size = get_maybe_pointer_size(src); let name = match op { @@ -1683,6 +1684,9 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let atomic_function = self.context.get_builtin_function(name); let order = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc()); + // FIXME: If `ret_ptr` is true and `src` is an integer, we should really tell GCC + // that this is a pointer operation that needs to preserve provenance -- but like LLVM, + // GCC does not currently seems to support that. let void_ptr_type = self.context.new_type::<*mut ()>(); let volatile_void_ptr_type = void_ptr_type.make_volatile(); let dst = self.context.new_cast(self.location, dst, volatile_void_ptr_type); @@ -1690,7 +1694,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let new_src_type = atomic_function.get_param(1).to_rvalue().get_type(); let src = self.context.new_bitcast(self.location, src, new_src_type); let res = self.context.new_call(self.location, atomic_function, &[dst, src, order]); - self.context.new_cast(self.location, res, src.get_type()) + let res_type = if ret_ptr { void_ptr_type } else { src.get_type() }; + self.context.new_cast(self.location, res, res_type) } fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope) { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 0ade9edb0d2ea..51593f3f065f4 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1326,15 +1326,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { &mut self, op: rustc_codegen_ssa::common::AtomicRmwBinOp, dst: &'ll Value, - mut src: &'ll Value, + src: &'ll Value, order: rustc_middle::ty::AtomicOrdering, + ret_ptr: bool, ) -> &'ll Value { - // The only RMW operation that LLVM supports on pointers is compare-exchange. - let requires_cast_to_int = self.val_ty(src) == self.type_ptr() - && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg; - if requires_cast_to_int { - src = self.ptrtoint(src, self.type_isize()); - } + // FIXME: If `ret_ptr` is true and `src` is not a pointer, we *should* tell LLVM that the + // LHS is a pointer and the operation should be provenance-preserving, but LLVM does not + // currently support that (https://github.com/llvm/llvm-project/issues/120837). let mut res = unsafe { llvm::LLVMBuildAtomicRMW( self.llbuilder, @@ -1345,7 +1343,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { llvm::False, // SingleThreaded ) }; - if requires_cast_to_int { + if ret_ptr && self.val_ty(res) != self.type_ptr() { res = self.inttoptr(res, self.type_ptr()); } res diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl index c7bd6ffd1f27c..42310a8b8c77b 100644 --- a/compiler/rustc_codegen_ssa/messages.ftl +++ b/compiler/rustc_codegen_ssa/messages.ftl @@ -97,6 +97,8 @@ codegen_ssa_invalid_monomorphization_basic_float_type = invalid monomorphization codegen_ssa_invalid_monomorphization_basic_integer_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer type, found `{$ty}` +codegen_ssa_invalid_monomorphization_basic_integer_or_ptr_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer or pointer type, found `{$ty}` + codegen_ssa_invalid_monomorphization_cannot_return = invalid monomorphization of `{$name}` intrinsic: cannot return `{$ret_ty}`, expected `u{$expected_int_bits}` or `[u8; {$expected_bytes}]` codegen_ssa_invalid_monomorphization_cast_wide_pointer = invalid monomorphization of `{$name}` intrinsic: cannot cast wide pointer `{$ty}` diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index 9040915b6af34..751e7c8a6cb1e 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -764,6 +764,14 @@ pub enum InvalidMonomorphization<'tcx> { ty: Ty<'tcx>, }, + #[diag(codegen_ssa_invalid_monomorphization_basic_integer_or_ptr_type, code = E0511)] + BasicIntegerOrPtrType { + #[primary_span] + span: Span, + name: Symbol, + ty: Ty<'tcx>, + }, + #[diag(codegen_ssa_invalid_monomorphization_basic_float_type, code = E0511)] BasicFloatType { #[primary_span] diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index fc95f62b4a43d..3c667b8e88203 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -92,6 +92,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let invalid_monomorphization_int_type = |ty| { bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty }); }; + let invalid_monomorphization_int_or_ptr_type = |ty| { + bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerOrPtrType { + span, + name, + ty, + }); + }; let parse_atomic_ordering = |ord: ty::Value<'tcx>| { let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf(); @@ -351,7 +358,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { sym::atomic_load => { let ty = fn_args.type_at(0); if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) { - invalid_monomorphization_int_type(ty); + invalid_monomorphization_int_or_ptr_type(ty); return Ok(()); } let ordering = fn_args.const_at(1).to_value(); @@ -367,7 +374,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { sym::atomic_store => { let ty = fn_args.type_at(0); if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) { - invalid_monomorphization_int_type(ty); + invalid_monomorphization_int_or_ptr_type(ty); return Ok(()); } let ordering = fn_args.const_at(1).to_value(); @@ -377,10 +384,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.atomic_store(val, ptr, parse_atomic_ordering(ordering), size); return Ok(()); } + // These are all AtomicRMW ops sym::atomic_cxchg | sym::atomic_cxchgweak => { let ty = fn_args.type_at(0); if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) { - invalid_monomorphization_int_type(ty); + invalid_monomorphization_int_or_ptr_type(ty); return Ok(()); } let succ_ordering = fn_args.const_at(1).to_value(); @@ -407,7 +415,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return Ok(()); } - // These are all AtomicRMW ops sym::atomic_max | sym::atomic_min => { let atom_op = if name == sym::atomic_max { AtomicRmwBinOp::AtomicMax @@ -420,7 +427,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let ordering = fn_args.const_at(1).to_value(); let ptr = args[0].immediate(); let val = args[1].immediate(); - bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering)) + bx.atomic_rmw( + atom_op, + ptr, + val, + parse_atomic_ordering(ordering), + /* ret_ptr */ false, + ) } else { invalid_monomorphization_int_type(ty); return Ok(()); @@ -438,21 +451,44 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let ordering = fn_args.const_at(1).to_value(); let ptr = args[0].immediate(); let val = args[1].immediate(); - bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering)) + bx.atomic_rmw( + atom_op, + ptr, + val, + parse_atomic_ordering(ordering), + /* ret_ptr */ false, + ) } else { invalid_monomorphization_int_type(ty); return Ok(()); } } - sym::atomic_xchg - | sym::atomic_xadd + sym::atomic_xchg => { + let ty = fn_args.type_at(0); + let ordering = fn_args.const_at(1).to_value(); + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() { + let ptr = args[0].immediate(); + let val = args[1].immediate(); + let atomic_op = AtomicRmwBinOp::AtomicXchg; + bx.atomic_rmw( + atomic_op, + ptr, + val, + parse_atomic_ordering(ordering), + /* ret_ptr */ ty.is_raw_ptr(), + ) + } else { + invalid_monomorphization_int_or_ptr_type(ty); + return Ok(()); + } + } + sym::atomic_xadd | sym::atomic_xsub | sym::atomic_and | sym::atomic_nand | sym::atomic_or | sym::atomic_xor => { let atom_op = match name { - sym::atomic_xchg => AtomicRmwBinOp::AtomicXchg, sym::atomic_xadd => AtomicRmwBinOp::AtomicAdd, sym::atomic_xsub => AtomicRmwBinOp::AtomicSub, sym::atomic_and => AtomicRmwBinOp::AtomicAnd, @@ -462,14 +498,28 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => unreachable!(), }; - let ty = fn_args.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() { - let ordering = fn_args.const_at(1).to_value(); - let ptr = args[0].immediate(); - let val = args[1].immediate(); - bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering)) + // The type of the in-memory data. + let ty_mem = fn_args.type_at(0); + // The type of the 2nd operand, given by-value. + let ty_op = fn_args.type_at(1); + + let ordering = fn_args.const_at(2).to_value(); + // We require either both arguments to have the same integer type, or the first to + // be a pointer and the second to be `usize`. + if (int_type_width_signed(ty_mem, bx.tcx()).is_some() && ty_op == ty_mem) + || (ty_mem.is_raw_ptr() && ty_op == bx.tcx().types.usize) + { + let ptr = args[0].immediate(); // of type "pointer to `ty_mem`" + let val = args[1].immediate(); // of type `ty_op` + bx.atomic_rmw( + atom_op, + ptr, + val, + parse_atomic_ordering(ordering), + /* ret_ptr */ ty_mem.is_raw_ptr(), + ) } else { - invalid_monomorphization_int_type(ty); + invalid_monomorphization_int_or_ptr_type(ty_mem); return Ok(()); } } diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 979456a6ba70f..fc17b30d65723 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -548,12 +548,15 @@ pub trait BuilderMethods<'a, 'tcx>: failure_order: AtomicOrdering, weak: bool, ) -> (Self::Value, Self::Value); + /// `ret_ptr` indicates whether the return type (which is also the type `dst` points to) + /// is a pointer or the same type as `src`. fn atomic_rmw( &mut self, op: AtomicRmwBinOp, dst: Self::Value, src: Self::Value, order: AtomicOrdering, + ret_ptr: bool, ) -> Self::Value; fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope); fn set_invariant_load(&mut self, load: Self::Value); diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 6e5fe3823ab51..4441dd6ebd66a 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -652,16 +652,16 @@ pub(crate) fn check_intrinsic_type( sym::atomic_store => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit), sym::atomic_xchg - | sym::atomic_xadd - | sym::atomic_xsub - | sym::atomic_and - | sym::atomic_nand - | sym::atomic_or - | sym::atomic_xor | sym::atomic_max | sym::atomic_min | sym::atomic_umax | sym::atomic_umin => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)), + sym::atomic_xadd + | sym::atomic_xsub + | sym::atomic_and + | sym::atomic_nand + | sym::atomic_or + | sym::atomic_xor => (2, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(1)], param(0)), sym::atomic_fence | sym::atomic_singlethreadfence => (0, 1, Vec::new(), tcx.types.unit), other => { diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 106cc725fee2c..9bf695e30258a 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -150,69 +150,63 @@ pub unsafe fn atomic_xchg(dst: *mut T, src: /// Adds to the current value, returning the previous value. /// `T` must be an integer or pointer type. -/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new -/// value stored at `*dst` will have the provenance of the old value stored there. +/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type. /// /// The stabilized version of this intrinsic is available on the /// [`atomic`] types via the `fetch_add` method. For example, [`AtomicIsize::fetch_add`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_xadd(dst: *mut T, src: T) -> T; +pub unsafe fn atomic_xadd(dst: *mut T, src: U) -> T; /// Subtract from the current value, returning the previous value. /// `T` must be an integer or pointer type. -/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new -/// value stored at `*dst` will have the provenance of the old value stored there. +/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type. /// /// The stabilized version of this intrinsic is available on the /// [`atomic`] types via the `fetch_sub` method. For example, [`AtomicIsize::fetch_sub`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_xsub(dst: *mut T, src: T) -> T; +pub unsafe fn atomic_xsub(dst: *mut T, src: U) -> T; /// Bitwise and with the current value, returning the previous value. /// `T` must be an integer or pointer type. -/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new -/// value stored at `*dst` will have the provenance of the old value stored there. +/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type. /// /// The stabilized version of this intrinsic is available on the /// [`atomic`] types via the `fetch_and` method. For example, [`AtomicBool::fetch_and`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_and(dst: *mut T, src: T) -> T; +pub unsafe fn atomic_and(dst: *mut T, src: U) -> T; /// Bitwise nand with the current value, returning the previous value. /// `T` must be an integer or pointer type. -/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new -/// value stored at `*dst` will have the provenance of the old value stored there. +/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type. /// /// The stabilized version of this intrinsic is available on the /// [`AtomicBool`] type via the `fetch_nand` method. For example, [`AtomicBool::fetch_nand`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_nand(dst: *mut T, src: T) -> T; +pub unsafe fn atomic_nand(dst: *mut T, src: U) -> T; /// Bitwise or with the current value, returning the previous value. /// `T` must be an integer or pointer type. -/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new -/// value stored at `*dst` will have the provenance of the old value stored there. +/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type. /// /// The stabilized version of this intrinsic is available on the /// [`atomic`] types via the `fetch_or` method. For example, [`AtomicBool::fetch_or`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_or(dst: *mut T, src: T) -> T; +pub unsafe fn atomic_or(dst: *mut T, src: U) -> T; /// Bitwise xor with the current value, returning the previous value. /// `T` must be an integer or pointer type. -/// If `T` is a pointer type, the provenance of `src` is ignored: both the return value and the new -/// value stored at `*dst` will have the provenance of the old value stored there. +/// `U` must be the same as `T` if that is an integer type, or `usize` if `T` is a pointer type. /// /// The stabilized version of this intrinsic is available on the /// [`atomic`] types via the `fetch_xor` method. For example, [`AtomicBool::fetch_xor`]. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn atomic_xor(dst: *mut T, src: T) -> T; +pub unsafe fn atomic_xor(dst: *mut T, src: U) -> T; /// Maximum with the current value using a signed comparison. /// `T` must be a signed integer type. diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs index 546f3d91a8099..bf07b18f3e739 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs @@ -2291,7 +2291,7 @@ impl AtomicPtr { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_byte_add(&self, val: usize, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_add(self.p.get(), core::ptr::without_provenance_mut(val), order).cast() } + unsafe { atomic_add(self.p.get(), val, order).cast() } } /// Offsets the pointer's address by subtracting `val` *bytes*, returning the @@ -2316,9 +2316,10 @@ impl AtomicPtr { /// #![feature(strict_provenance_atomic_ptr)] /// use core::sync::atomic::{AtomicPtr, Ordering}; /// - /// let atom = AtomicPtr::::new(core::ptr::without_provenance_mut(1)); - /// assert_eq!(atom.fetch_byte_sub(1, Ordering::Relaxed).addr(), 1); - /// assert_eq!(atom.load(Ordering::Relaxed).addr(), 0); + /// let mut arr = [0i64, 1]; + /// let atom = AtomicPtr::::new(&raw mut arr[1]); + /// assert_eq!(atom.fetch_byte_sub(8, Ordering::Relaxed).addr(), (&raw const arr[1]).addr()); + /// assert_eq!(atom.load(Ordering::Relaxed).addr(), (&raw const arr[0]).addr()); /// ``` #[inline] #[cfg(target_has_atomic = "ptr")] @@ -2326,7 +2327,7 @@ impl AtomicPtr { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_byte_sub(&self, val: usize, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_sub(self.p.get(), core::ptr::without_provenance_mut(val), order).cast() } + unsafe { atomic_sub(self.p.get(), val, order).cast() } } /// Performs a bitwise "or" operation on the address of the current pointer, @@ -2377,7 +2378,7 @@ impl AtomicPtr { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_or(&self, val: usize, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_or(self.p.get(), core::ptr::without_provenance_mut(val), order).cast() } + unsafe { atomic_or(self.p.get(), val, order).cast() } } /// Performs a bitwise "and" operation on the address of the current @@ -2427,7 +2428,7 @@ impl AtomicPtr { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_and(&self, val: usize, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_and(self.p.get(), core::ptr::without_provenance_mut(val), order).cast() } + unsafe { atomic_and(self.p.get(), val, order).cast() } } /// Performs a bitwise "xor" operation on the address of the current @@ -2475,7 +2476,7 @@ impl AtomicPtr { #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub fn fetch_xor(&self, val: usize, order: Ordering) -> *mut T { // SAFETY: data races are prevented by atomic intrinsics. - unsafe { atomic_xor(self.p.get(), core::ptr::without_provenance_mut(val), order).cast() } + unsafe { atomic_xor(self.p.get(), val, order).cast() } } /// Returns a mutable pointer to the underlying pointer. @@ -3975,15 +3976,15 @@ unsafe fn atomic_swap(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_add(dst: *mut T, val: T, order: Ordering) -> T { +unsafe fn atomic_add(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_add`. unsafe { match order { - Relaxed => intrinsics::atomic_xadd::(dst, val), - Acquire => intrinsics::atomic_xadd::(dst, val), - Release => intrinsics::atomic_xadd::(dst, val), - AcqRel => intrinsics::atomic_xadd::(dst, val), - SeqCst => intrinsics::atomic_xadd::(dst, val), + Relaxed => intrinsics::atomic_xadd::(dst, val), + Acquire => intrinsics::atomic_xadd::(dst, val), + Release => intrinsics::atomic_xadd::(dst, val), + AcqRel => intrinsics::atomic_xadd::(dst, val), + SeqCst => intrinsics::atomic_xadd::(dst, val), } } } @@ -3992,15 +3993,15 @@ unsafe fn atomic_add(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_sub(dst: *mut T, val: T, order: Ordering) -> T { +unsafe fn atomic_sub(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_sub`. unsafe { match order { - Relaxed => intrinsics::atomic_xsub::(dst, val), - Acquire => intrinsics::atomic_xsub::(dst, val), - Release => intrinsics::atomic_xsub::(dst, val), - AcqRel => intrinsics::atomic_xsub::(dst, val), - SeqCst => intrinsics::atomic_xsub::(dst, val), + Relaxed => intrinsics::atomic_xsub::(dst, val), + Acquire => intrinsics::atomic_xsub::(dst, val), + Release => intrinsics::atomic_xsub::(dst, val), + AcqRel => intrinsics::atomic_xsub::(dst, val), + SeqCst => intrinsics::atomic_xsub::(dst, val), } } } @@ -4141,15 +4142,15 @@ unsafe fn atomic_compare_exchange_weak( #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_and(dst: *mut T, val: T, order: Ordering) -> T { +unsafe fn atomic_and(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_and` unsafe { match order { - Relaxed => intrinsics::atomic_and::(dst, val), - Acquire => intrinsics::atomic_and::(dst, val), - Release => intrinsics::atomic_and::(dst, val), - AcqRel => intrinsics::atomic_and::(dst, val), - SeqCst => intrinsics::atomic_and::(dst, val), + Relaxed => intrinsics::atomic_and::(dst, val), + Acquire => intrinsics::atomic_and::(dst, val), + Release => intrinsics::atomic_and::(dst, val), + AcqRel => intrinsics::atomic_and::(dst, val), + SeqCst => intrinsics::atomic_and::(dst, val), } } } @@ -4157,15 +4158,15 @@ unsafe fn atomic_and(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_nand(dst: *mut T, val: T, order: Ordering) -> T { +unsafe fn atomic_nand(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_nand` unsafe { match order { - Relaxed => intrinsics::atomic_nand::(dst, val), - Acquire => intrinsics::atomic_nand::(dst, val), - Release => intrinsics::atomic_nand::(dst, val), - AcqRel => intrinsics::atomic_nand::(dst, val), - SeqCst => intrinsics::atomic_nand::(dst, val), + Relaxed => intrinsics::atomic_nand::(dst, val), + Acquire => intrinsics::atomic_nand::(dst, val), + Release => intrinsics::atomic_nand::(dst, val), + AcqRel => intrinsics::atomic_nand::(dst, val), + SeqCst => intrinsics::atomic_nand::(dst, val), } } } @@ -4173,15 +4174,15 @@ unsafe fn atomic_nand(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_or(dst: *mut T, val: T, order: Ordering) -> T { +unsafe fn atomic_or(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_or` unsafe { match order { - SeqCst => intrinsics::atomic_or::(dst, val), - Acquire => intrinsics::atomic_or::(dst, val), - Release => intrinsics::atomic_or::(dst, val), - AcqRel => intrinsics::atomic_or::(dst, val), - Relaxed => intrinsics::atomic_or::(dst, val), + SeqCst => intrinsics::atomic_or::(dst, val), + Acquire => intrinsics::atomic_or::(dst, val), + Release => intrinsics::atomic_or::(dst, val), + AcqRel => intrinsics::atomic_or::(dst, val), + Relaxed => intrinsics::atomic_or::(dst, val), } } } @@ -4189,15 +4190,15 @@ unsafe fn atomic_or(dst: *mut T, val: T, order: Ordering) -> T { #[inline] #[cfg(target_has_atomic)] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces -unsafe fn atomic_xor(dst: *mut T, val: T, order: Ordering) -> T { +unsafe fn atomic_xor(dst: *mut T, val: U, order: Ordering) -> T { // SAFETY: the caller must uphold the safety contract for `atomic_xor` unsafe { match order { - SeqCst => intrinsics::atomic_xor::(dst, val), - Acquire => intrinsics::atomic_xor::(dst, val), - Release => intrinsics::atomic_xor::(dst, val), - AcqRel => intrinsics::atomic_xor::(dst, val), - Relaxed => intrinsics::atomic_xor::(dst, val), + SeqCst => intrinsics::atomic_xor::(dst, val), + Acquire => intrinsics::atomic_xor::(dst, val), + Release => intrinsics::atomic_xor::(dst, val), + AcqRel => intrinsics::atomic_xor::(dst, val), + Relaxed => intrinsics::atomic_xor::(dst, val), } } } diff --git a/src/tools/miri/src/intrinsics/atomic.rs b/src/tools/miri/src/intrinsics/atomic.rs index 0a59a707a101c..341a00a30aec3 100644 --- a/src/tools/miri/src/intrinsics/atomic.rs +++ b/src/tools/miri/src/intrinsics/atomic.rs @@ -105,27 +105,27 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "or" => { - let ord = get_ord_at(1); + let ord = get_ord_at(2); this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord))?; } "xor" => { - let ord = get_ord_at(1); + let ord = get_ord_at(2); this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord))?; } "and" => { - let ord = get_ord_at(1); + let ord = get_ord_at(2); this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord))?; } "nand" => { - let ord = get_ord_at(1); + let ord = get_ord_at(2); this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord))?; } "xadd" => { - let ord = get_ord_at(1); + let ord = get_ord_at(2); this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord))?; } "xsub" => { - let ord = get_ord_at(1); + let ord = get_ord_at(2); this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord))?; } "min" => { @@ -231,15 +231,14 @@ trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> { let place = this.deref_pointer(place)?; let rhs = this.read_immediate(rhs)?; - if !place.layout.ty.is_integral() && !place.layout.ty.is_raw_ptr() { + if !(place.layout.ty.is_integral() || place.layout.ty.is_raw_ptr()) + || !(rhs.layout.ty.is_integral() || rhs.layout.ty.is_raw_ptr()) + { span_bug!( this.cur_span(), "atomic arithmetic operations only work on integer and raw pointer types", ); } - if rhs.layout.ty != place.layout.ty { - span_bug!(this.cur_span(), "atomic arithmetic operation type mismatch"); - } let old = match atomic_op { AtomicOp::Min => diff --git a/src/tools/miri/src/operator.rs b/src/tools/miri/src/operator.rs index 73d671121f653..3c3f2c2853588 100644 --- a/src/tools/miri/src/operator.rs +++ b/src/tools/miri/src/operator.rs @@ -50,17 +50,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } // Some more operations are possible with atomics. - // The return value always has the provenance of the *left* operand. + // The RHS must be `usize`. Add | Sub | BitOr | BitAnd | BitXor => { assert!(left.layout.ty.is_raw_ptr()); - assert!(right.layout.ty.is_raw_ptr()); + assert_eq!(right.layout.ty, this.tcx.types.usize); let ptr = left.to_scalar().to_pointer(this)?; // We do the actual operation with usize-typed scalars. let left = ImmTy::from_uint(ptr.addr().bytes(), this.machine.layouts.usize); - let right = ImmTy::from_uint( - right.to_scalar().to_target_usize(this)?, - this.machine.layouts.usize, - ); let result = this.binary_op(bin_op, &left, &right)?; // Construct a new pointer with the provenance of `ptr` (the LHS). let result_ptr = Pointer::new( diff --git a/tests/codegen-llvm/atomicptr.rs b/tests/codegen-llvm/atomicptr.rs index 4819af40ca2d2..ce6c4aa0d2b8e 100644 --- a/tests/codegen-llvm/atomicptr.rs +++ b/tests/codegen-llvm/atomicptr.rs @@ -1,5 +1,5 @@ // LLVM does not support some atomic RMW operations on pointers, so inside codegen we lower those -// to integer atomics, surrounded by casts to and from integer type. +// to integer atomics, followed by an inttoptr cast. // This test ensures that we do the round-trip correctly for AtomicPtr::fetch_byte_add, and also // ensures that we do not have such a round-trip for AtomicPtr::swap, because LLVM supports pointer // arguments to `atomicrmw xchg`. @@ -20,8 +20,8 @@ pub fn helper(_: usize) {} // CHECK-LABEL: @atomicptr_fetch_byte_add #[no_mangle] pub fn atomicptr_fetch_byte_add(a: &AtomicPtr, v: usize) -> *mut u8 { - // CHECK: %[[INTPTR:.*]] = ptrtoint ptr %{{.*}} to [[USIZE]] - // CHECK-NEXT: %[[RET:.*]] = atomicrmw add ptr %{{.*}}, [[USIZE]] %[[INTPTR]] + // CHECK: llvm.lifetime.start + // CHECK-NEXT: %[[RET:.*]] = atomicrmw add ptr %{{.*}}, [[USIZE]] %v // CHECK-NEXT: inttoptr [[USIZE]] %[[RET]] to ptr a.fetch_byte_add(v, Relaxed) } diff --git a/tests/run-make/atomic-lock-free/atomic_lock_free.rs b/tests/run-make/atomic-lock-free/atomic_lock_free.rs index f5c3b360ee815..92ffd111ce8b7 100644 --- a/tests/run-make/atomic-lock-free/atomic_lock_free.rs +++ b/tests/run-make/atomic-lock-free/atomic_lock_free.rs @@ -14,7 +14,7 @@ pub enum AtomicOrdering { } #[rustc_intrinsic] -unsafe fn atomic_xadd(dst: *mut T, src: T) -> T; +unsafe fn atomic_xadd(dst: *mut T, src: U) -> T; #[lang = "pointee_sized"] pub trait PointeeSized {} @@ -35,51 +35,62 @@ impl Copy for *mut T {} impl ConstParamTy for AtomicOrdering {} #[cfg(target_has_atomic = "8")] +#[unsafe(no_mangle)] // let's make sure we actually generate a symbol to check pub unsafe fn atomic_u8(x: *mut u8) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u8); } #[cfg(target_has_atomic = "8")] +#[unsafe(no_mangle)] pub unsafe fn atomic_i8(x: *mut i8) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i8); } #[cfg(target_has_atomic = "16")] +#[unsafe(no_mangle)] pub unsafe fn atomic_u16(x: *mut u16) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u16); } #[cfg(target_has_atomic = "16")] +#[unsafe(no_mangle)] pub unsafe fn atomic_i16(x: *mut i16) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i16); } #[cfg(target_has_atomic = "32")] +#[unsafe(no_mangle)] pub unsafe fn atomic_u32(x: *mut u32) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u32); } #[cfg(target_has_atomic = "32")] +#[unsafe(no_mangle)] pub unsafe fn atomic_i32(x: *mut i32) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i32); } #[cfg(target_has_atomic = "64")] +#[unsafe(no_mangle)] pub unsafe fn atomic_u64(x: *mut u64) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u64); } #[cfg(target_has_atomic = "64")] +#[unsafe(no_mangle)] pub unsafe fn atomic_i64(x: *mut i64) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i64); } #[cfg(target_has_atomic = "128")] +#[unsafe(no_mangle)] pub unsafe fn atomic_u128(x: *mut u128) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u128); } #[cfg(target_has_atomic = "128")] +#[unsafe(no_mangle)] pub unsafe fn atomic_i128(x: *mut i128) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i128); } #[cfg(target_has_atomic = "ptr")] +#[unsafe(no_mangle)] pub unsafe fn atomic_usize(x: *mut usize) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1usize); } #[cfg(target_has_atomic = "ptr")] +#[unsafe(no_mangle)] pub unsafe fn atomic_isize(x: *mut isize) { - atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1); + atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1isize); } diff --git a/tests/ui/intrinsics/intrinsic-atomics.rs b/tests/ui/intrinsics/intrinsic-atomics.rs index 2275aafff8330..c19948137db01 100644 --- a/tests/ui/intrinsics/intrinsic-atomics.rs +++ b/tests/ui/intrinsics/intrinsic-atomics.rs @@ -33,14 +33,14 @@ pub fn main() { assert_eq!(rusti::atomic_xchg::<_, { Release }>(&mut *x, 0), 1); assert_eq!(*x, 0); - assert_eq!(rusti::atomic_xadd::<_, { SeqCst }>(&mut *x, 1), 0); - assert_eq!(rusti::atomic_xadd::<_, { Acquire }>(&mut *x, 1), 1); - assert_eq!(rusti::atomic_xadd::<_, { Release }>(&mut *x, 1), 2); + assert_eq!(rusti::atomic_xadd::<_, _, { SeqCst }>(&mut *x, 1), 0); + assert_eq!(rusti::atomic_xadd::<_, _, { Acquire }>(&mut *x, 1), 1); + assert_eq!(rusti::atomic_xadd::<_, _, { Release }>(&mut *x, 1), 2); assert_eq!(*x, 3); - assert_eq!(rusti::atomic_xsub::<_, { SeqCst }>(&mut *x, 1), 3); - assert_eq!(rusti::atomic_xsub::<_, { Acquire }>(&mut *x, 1), 2); - assert_eq!(rusti::atomic_xsub::<_, { Release }>(&mut *x, 1), 1); + assert_eq!(rusti::atomic_xsub::<_, _, { SeqCst }>(&mut *x, 1), 3); + assert_eq!(rusti::atomic_xsub::<_, _, { Acquire }>(&mut *x, 1), 2); + assert_eq!(rusti::atomic_xsub::<_, _, { Release }>(&mut *x, 1), 1); assert_eq!(*x, 0); loop { diff --git a/tests/ui/intrinsics/non-integer-atomic.rs b/tests/ui/intrinsics/non-integer-atomic.rs index 853c163710fcf..30f713f12412d 100644 --- a/tests/ui/intrinsics/non-integer-atomic.rs +++ b/tests/ui/intrinsics/non-integer-atomic.rs @@ -13,80 +13,80 @@ pub type Quux = [u8; 100]; pub unsafe fn test_bool_load(p: &mut bool, v: bool) { intrinsics::atomic_load::<_, { SeqCst }>(p); - //~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `bool` + //~^ ERROR `atomic_load` intrinsic: expected basic integer or pointer type, found `bool` } pub unsafe fn test_bool_store(p: &mut bool, v: bool) { intrinsics::atomic_store::<_, { SeqCst }>(p, v); - //~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `bool` + //~^ ERROR `atomic_store` intrinsic: expected basic integer or pointer type, found `bool` } pub unsafe fn test_bool_xchg(p: &mut bool, v: bool) { intrinsics::atomic_xchg::<_, { SeqCst }>(p, v); - //~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `bool` + //~^ ERROR `atomic_xchg` intrinsic: expected basic integer or pointer type, found `bool` } pub unsafe fn test_bool_cxchg(p: &mut bool, v: bool) { intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v); - //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `bool` + //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer or pointer type, found `bool` } pub unsafe fn test_Foo_load(p: &mut Foo, v: Foo) { intrinsics::atomic_load::<_, { SeqCst }>(p); - //~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `Foo` + //~^ ERROR `atomic_load` intrinsic: expected basic integer or pointer type, found `Foo` } pub unsafe fn test_Foo_store(p: &mut Foo, v: Foo) { intrinsics::atomic_store::<_, { SeqCst }>(p, v); - //~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `Foo` + //~^ ERROR `atomic_store` intrinsic: expected basic integer or pointer type, found `Foo` } pub unsafe fn test_Foo_xchg(p: &mut Foo, v: Foo) { intrinsics::atomic_xchg::<_, { SeqCst }>(p, v); - //~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `Foo` + //~^ ERROR `atomic_xchg` intrinsic: expected basic integer or pointer type, found `Foo` } pub unsafe fn test_Foo_cxchg(p: &mut Foo, v: Foo) { intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v); - //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `Foo` + //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer or pointer type, found `Foo` } pub unsafe fn test_Bar_load(p: &mut Bar, v: Bar) { intrinsics::atomic_load::<_, { SeqCst }>(p); - //~^ ERROR expected basic integer type, found `&dyn Fn()` + //~^ ERROR expected basic integer or pointer type, found `&dyn Fn()` } pub unsafe fn test_Bar_store(p: &mut Bar, v: Bar) { intrinsics::atomic_store::<_, { SeqCst }>(p, v); - //~^ ERROR expected basic integer type, found `&dyn Fn()` + //~^ ERROR expected basic integer or pointer type, found `&dyn Fn()` } pub unsafe fn test_Bar_xchg(p: &mut Bar, v: Bar) { intrinsics::atomic_xchg::<_, { SeqCst }>(p, v); - //~^ ERROR expected basic integer type, found `&dyn Fn()` + //~^ ERROR expected basic integer or pointer type, found `&dyn Fn()` } pub unsafe fn test_Bar_cxchg(p: &mut Bar, v: Bar) { intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v); - //~^ ERROR expected basic integer type, found `&dyn Fn()` + //~^ ERROR expected basic integer or pointer type, found `&dyn Fn()` } pub unsafe fn test_Quux_load(p: &mut Quux, v: Quux) { intrinsics::atomic_load::<_, { SeqCst }>(p); - //~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `[u8; 100]` + //~^ ERROR `atomic_load` intrinsic: expected basic integer or pointer type, found `[u8; 100]` } pub unsafe fn test_Quux_store(p: &mut Quux, v: Quux) { intrinsics::atomic_store::<_, { SeqCst }>(p, v); - //~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `[u8; 100]` + //~^ ERROR `atomic_store` intrinsic: expected basic integer or pointer type, found `[u8; 100]` } pub unsafe fn test_Quux_xchg(p: &mut Quux, v: Quux) { intrinsics::atomic_xchg::<_, { SeqCst }>(p, v); - //~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `[u8; 100]` + //~^ ERROR `atomic_xchg` intrinsic: expected basic integer or pointer type, found `[u8; 100]` } pub unsafe fn test_Quux_cxchg(p: &mut Quux, v: Quux) { intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v); - //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `[u8; 100]` + //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer or pointer type, found `[u8; 100]` } diff --git a/tests/ui/intrinsics/non-integer-atomic.stderr b/tests/ui/intrinsics/non-integer-atomic.stderr index e539d99b8aebd..b96ee7ba84688 100644 --- a/tests/ui/intrinsics/non-integer-atomic.stderr +++ b/tests/ui/intrinsics/non-integer-atomic.stderr @@ -1,94 +1,94 @@ -error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `bool` +error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer or pointer type, found `bool` --> $DIR/non-integer-atomic.rs:15:5 | LL | intrinsics::atomic_load::<_, { SeqCst }>(p); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `bool` +error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer or pointer type, found `bool` --> $DIR/non-integer-atomic.rs:20:5 | LL | intrinsics::atomic_store::<_, { SeqCst }>(p, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `bool` +error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer or pointer type, found `bool` --> $DIR/non-integer-atomic.rs:25:5 | LL | intrinsics::atomic_xchg::<_, { SeqCst }>(p, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `bool` +error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer or pointer type, found `bool` --> $DIR/non-integer-atomic.rs:30:5 | LL | intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `Foo` +error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer or pointer type, found `Foo` --> $DIR/non-integer-atomic.rs:35:5 | LL | intrinsics::atomic_load::<_, { SeqCst }>(p); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `Foo` +error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer or pointer type, found `Foo` --> $DIR/non-integer-atomic.rs:40:5 | LL | intrinsics::atomic_store::<_, { SeqCst }>(p, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `Foo` +error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer or pointer type, found `Foo` --> $DIR/non-integer-atomic.rs:45:5 | LL | intrinsics::atomic_xchg::<_, { SeqCst }>(p, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `Foo` +error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer or pointer type, found `Foo` --> $DIR/non-integer-atomic.rs:50:5 | LL | intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `&dyn Fn()` +error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer or pointer type, found `&dyn Fn()` --> $DIR/non-integer-atomic.rs:55:5 | LL | intrinsics::atomic_load::<_, { SeqCst }>(p); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `&dyn Fn()` +error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer or pointer type, found `&dyn Fn()` --> $DIR/non-integer-atomic.rs:60:5 | LL | intrinsics::atomic_store::<_, { SeqCst }>(p, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `&dyn Fn()` +error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer or pointer type, found `&dyn Fn()` --> $DIR/non-integer-atomic.rs:65:5 | LL | intrinsics::atomic_xchg::<_, { SeqCst }>(p, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `&dyn Fn()` +error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer or pointer type, found `&dyn Fn()` --> $DIR/non-integer-atomic.rs:70:5 | LL | intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `[u8; 100]` +error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer or pointer type, found `[u8; 100]` --> $DIR/non-integer-atomic.rs:75:5 | LL | intrinsics::atomic_load::<_, { SeqCst }>(p); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `[u8; 100]` +error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer or pointer type, found `[u8; 100]` --> $DIR/non-integer-atomic.rs:80:5 | LL | intrinsics::atomic_store::<_, { SeqCst }>(p, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `[u8; 100]` +error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer or pointer type, found `[u8; 100]` --> $DIR/non-integer-atomic.rs:85:5 | LL | intrinsics::atomic_xchg::<_, { SeqCst }>(p, v); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `[u8; 100]` +error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer or pointer type, found `[u8; 100]` --> $DIR/non-integer-atomic.rs:90:5 | LL | intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);