aarch64_atomics.diff

diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 87e0f83f85b7..6d80ed031403 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -12,13 +12,19 @@
 
 let TargetPrefix = "aarch64" in {
 
-def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
+def int_aarch64_ldxr : Intrinsic<[llvm_anyint_ty],
+                                 [LLVMAnyPointerType<LLVMMatchType<0>>],
                                  [IntrNoFree, IntrWillReturn]>;
-def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
+def int_aarch64_ldaxr : Intrinsic<[llvm_anyint_ty],
+                                  [LLVMAnyPointerType<LLVMMatchType<0>>],
                                   [IntrNoFree, IntrWillReturn]>;
-def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
+def int_aarch64_stxr : Intrinsic<[llvm_i32_ty],
+                                 [llvm_anyint_ty,
+                                  LLVMAnyPointerType<LLVMMatchType<0>>],
                                  [IntrNoFree, IntrWillReturn]>;
-def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
+def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty],
+                                  [llvm_anyint_ty,
+                                   LLVMAnyPointerType<LLVMMatchType<0>>],
                                   [IntrNoFree, IntrWillReturn]>;
 
 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 96bafdbab23b..89d66c6390e3 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11369,23 +11369,21 @@ bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   }
   case Intrinsic::aarch64_ldaxr:
   case Intrinsic::aarch64_ldxr: {
-    PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
     Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = MVT::getVT(PtrTy->getElementType());
+    Info.memVT = MVT::getVT(I.getType());
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
-    Info.align = DL.getABITypeAlign(PtrTy->getElementType());
+    Info.align = DL.getABITypeAlign(I.getType());
     Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
     return true;
   }
   case Intrinsic::aarch64_stlxr:
   case Intrinsic::aarch64_stxr: {
-    PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
     Info.opc = ISD::INTRINSIC_W_CHAIN;
-    Info.memVT = MVT::getVT(PtrTy->getElementType());
+    Info.memVT = MVT::getVT(I.getArgOperand(0)->getType());
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
-    Info.align = DL.getABITypeAlign(PtrTy->getElementType());
+    Info.align = DL.getABITypeAlign(I.getArgOperand(0)->getType());
     Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
     return true;
   }
@@ -17705,16 +17703,14 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
         Lo, Builder.CreateShl(Hi, ConstantInt::get(ValueTy, 64)), "val64");
   }
 
-  Type *Tys[] = { Addr->getType() };
+  const DataLayout &DL = M->getDataLayout();
+  IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(ValueTy));
+  Type *Tys[] = { IntEltTy, Addr->getType() };
   Intrinsic::ID Int =
       IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
   Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys);
 
-  const DataLayout &DL = M->getDataLayout();
-  IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(ValueTy));
-  Value *Trunc = Builder.CreateTrunc(Builder.CreateCall(Ldxr, Addr), IntEltTy);
-
-  return Builder.CreateBitCast(Trunc, ValueTy);
+  return Builder.CreateBitCast(Builder.CreateCall(Ldxr, Addr), ValueTy);
 }
 
 void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
@@ -17744,19 +17740,16 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilderBase &Builder,
     return Builder.CreateCall(Stxr, {Lo, Hi, Addr});
   }
 
+  const DataLayout &DL = M->getDataLayout();
   Intrinsic::ID Int =
       IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
-  Type *Tys[] = { Addr->getType() };
+  IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
+  Type *Tys[] = { IntValTy, Addr->getType() };
   Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
 
-  const DataLayout &DL = M->getDataLayout();
-  IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
   Val = Builder.CreateBitCast(Val, IntValTy);
 
-  return Builder.CreateCall(Stxr,
-                            {Builder.CreateZExtOrBitCast(
-                                 Val, Stxr->getFunctionType()->getParamType(0)),
-                             Addr});
+  return Builder.CreateCall(Stxr, {Builder.CreateBitCast(Val, IntValTy), Addr});
 }
 
 bool AArch64TargetLowering::functionArgumentNeedsConsecutiveRegisters(
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 84573dac7e41..e0a572eec7f7 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -226,21 +226,11 @@ def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
 }
 
-def : Pat<(ldxr_1 GPR64sp:$addr),
-          (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
-def : Pat<(ldxr_2 GPR64sp:$addr),
-          (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
-def : Pat<(ldxr_4 GPR64sp:$addr),
-          (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
+def : Pat<(ldxr_1 GPR64sp:$addr), (LDXRB GPR64sp:$addr)>;
+def : Pat<(ldxr_2 GPR64sp:$addr), (LDXRH GPR64sp:$addr)>;
+def : Pat<(ldxr_4 GPR64sp:$addr), (LDXRW GPR64sp:$addr)>;
 def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;
 
-def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),
-          (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
-def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
-          (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
-def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
-          (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
-
 // Load-exclusives.
 
 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
@@ -267,21 +257,11 @@ def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
 }
 
-def : Pat<(ldaxr_1 GPR64sp:$addr),
-          (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
-def : Pat<(ldaxr_2 GPR64sp:$addr),
-          (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
-def : Pat<(ldaxr_4 GPR64sp:$addr),
-          (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
+def : Pat<(ldaxr_1 GPR64sp:$addr), (LDAXRB GPR64sp:$addr)>;
+def : Pat<(ldaxr_2 GPR64sp:$addr), (LDAXRH GPR64sp:$addr)>;
+def : Pat<(ldaxr_4 GPR64sp:$addr), (LDAXRW GPR64sp:$addr)>;
 def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;
 
-def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),
-          (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
-def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),
-          (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
-def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),
-          (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
-
 // Store-exclusives.
 
 def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
@@ -313,28 +293,14 @@ def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
 }
 
 
-def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),
-          (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),
-          (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),
-          (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
-          (STXRX GPR64:$val, GPR64sp:$addr)>;
-
-def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
+def : Pat<(stxr_1 GPR32:$val, GPR64sp:$addr),
           (STXRB GPR32:$val, GPR64sp:$addr)>;
-def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
+def : Pat<(stxr_2 GPR32:$val, GPR64sp:$addr),
           (STXRH GPR32:$val, GPR64sp:$addr)>;
-def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),
+def : Pat<(stxr_4 GPR32:$val, GPR64sp:$addr),
           (STXRW GPR32:$val, GPR64sp:$addr)>;
-
-def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
-          (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
-          (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
-          (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
+def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
+          (STXRX GPR64:$val, GPR64sp:$addr)>;
 
 // Store-release-exclusives.
 
@@ -367,28 +333,14 @@ def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
 }
 
 
-def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),
-          (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),
-          (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),
-          (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
-          (STLXRX GPR64:$val, GPR64sp:$addr)>;
-
-def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
+def : Pat<(stlxr_1 GPR32:$val, GPR64sp:$addr),
           (STLXRB GPR32:$val, GPR64sp:$addr)>;
-def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
+def : Pat<(stlxr_2 GPR32:$val, GPR64sp:$addr),
           (STLXRH GPR32:$val, GPR64sp:$addr)>;
-def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),
+def : Pat<(stlxr_4 GPR32:$val, GPR64sp:$addr),
           (STLXRW GPR32:$val, GPR64sp:$addr)>;
-
-def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
-          (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
-          (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
-          (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
+def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
+          (STLXRX GPR64:$val, GPR64sp:$addr)>;
 
 
 // And clear exclusive.
diff --git a/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll b/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
index f5beaebe50cb..5e3831bb6eda 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
@@ -45,8 +45,7 @@ define dso_local void @test_load_i8(i8* %addr) {
 ; GISEL: ldxrb w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldxr.p0i8(i8* %addr)
-  %shortval = trunc i64 %val to i8
+  %shortval = call i8 @llvm.aarch64.ldxr.i8.p0i8(i8* %addr)
   %extval = zext i8 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
@@ -64,8 +63,7 @@ define dso_local void @test_load_i16(i16* %addr) {
 ; GISEL: ldxrh w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldxr.p0i16(i16* %addr)
-  %shortval = trunc i64 %val to i16
+  %shortval = call i16 @llvm.aarch64.ldxr.i16.p0i16(i16* %addr)
   %extval = zext i16 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
@@ -83,8 +81,7 @@ define dso_local void @test_load_i32(i32* %addr) {
 ; GISEL: ldxr w[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldxr.p0i32(i32* %addr)
-  %shortval = trunc i64 %val to i32
+  %shortval = call i32 @llvm.aarch64.ldxr.i32.p0i32(i32* %addr)
   %extval = zext i32 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
@@ -100,16 +97,16 @@ define dso_local void @test_load_i64(i64* %addr) {
 ; GISEL: ldxr x[[LOADVAL:[0-9]+]], [x0]
 ; GISEL-NOT: uxtb
 ; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
-  %val = call i64 @llvm.aarch64.ldxr.p0i64(i64* %addr)
+  %val = call i64 @llvm.aarch64.ldxr.i64.p0i64(i64* %addr)
   store i64 %val, i64* @var, align 8
   ret void
 }
 
 
-declare i64 @llvm.aarch64.ldxr.p0i8(i8*) nounwind
-declare i64 @llvm.aarch64.ldxr.p0i16(i16*) nounwind
-declare i64 @llvm.aarch64.ldxr.p0i32(i32*) nounwind
-declare i64 @llvm.aarch64.ldxr.p0i64(i64*) nounwind
+declare i8 @llvm.aarch64.ldxr.i8.p0i8(i8*) nounwind
+declare i16 @llvm.aarch64.ldxr.i16.p0i16(i16*) nounwind
+declare i32 @llvm.aarch64.ldxr.i32.p0i32(i32*) nounwind
+declare i64 @llvm.aarch64.ldxr.i64.p0i64(i64*) nounwind
 
 ; FALLBACK-NOT: remark:{{.*}}test_store_i8
 define dso_local i32 @test_store_i8(i32, i8 %val, i8* %addr) {
@@ -121,8 +118,7 @@ define dso_local i32 @test_store_i8(i32, i8 %val, i8* %addr) {
 ; GISEL-NOT: uxtb
 ; GISEL-NOT: and
 ; GISEL: stxrb w0, w1, [x2]
-  %extval = zext i8 %val to i64
-  %res = call i32 @llvm.aarch64.stxr.p0i8(i64 %extval, i8* %addr)
+  %res = call i32 @llvm.aarch64.stxr.i8.p0i8(i8 %val, i8* %addr)
   ret i32 %res
 }
 
@@ -136,8 +132,7 @@ define dso_local i32 @test_store_i16(i32, i16 %val, i16* %addr) {
 ; GISEL-NOT: uxth
 ; GISEL-NOT: and
 ; GISEL: stxrh w0, w1, [x2]
-  %extval = zext i16 %val to i64
-  %res = call i32 @llvm.aarch64.stxr.p0i16(i64 %extval, i16* %addr)
+  %res = call i32 @llvm.aarch64.stxr.i16.p0i16(i16 %val, i16* %addr)
   ret i32 %res
 }
 
@@ -151,8 +146,7 @@ define dso_local i32 @test_store_i32(i32, i32 %val, i32* %addr) {
 ; GISEL-NOT: uxtw
 ; GISEL-NOT: and
 ; GISEL: stxr w0, w1, [x2]
-  %extval = zext i32 %val to i64
-  %res = call i32 @llvm.aarch64.stxr.p0i32(i64 %extval, i32* %addr)
+  %res = call i32 @llvm.aarch64.stxr.i32.p0i32(i32 %val, i32* %addr)
   ret i32 %res
 }
 
@@ -162,14 +156,14 @@ define dso_local i32 @test_store_i64(i32, i64 %val, i64* %addr) {
 ; CHECK: stxr w0, x1, [x2]
 ; GISEL-LABEL: test_store_i64:
 ; GISEL: stxr w0, x1, [x2]
-  %res = call i32 @llvm.aarch64.stxr.p0i64(i64 %val, i64* %addr)
+  %res = call i32 @llvm.aarch64.stxr.i64.p0i64(i64 %val, i64* %addr)
   ret i32 %res
 }
 
-declare i32 @llvm.aarch64.stxr.p0i8(i64, i8*) nounwind
-declare i32 @llvm.aarch64.stxr.p0i16(i64, i16*) nounwind
-declare i32 @llvm.aarch64.stxr.p0i32(i64, i32*) nounwind
-declare i32 @llvm.aarch64.stxr.p0i64(i64, i64*) nounwind
+declare i32 @llvm.aarch64.stxr.i8.p0i8(i8, i8*) nounwind
+declare i32 @llvm.aarch64.stxr.i16.p0i16(i16, i16*) nounwind
+declare i32 @llvm.aarch64.stxr.i32.p0i32(i32, i32*) nounwind
+declare i32 @llvm.aarch64.stxr.i64.p0i64(i64, i64*) nounwind
 
 ; CHECK: test_clear:
 ; CHECK: clrex