Use a compare-and-swap loop for AND.

BUG: This might allocate RAX twice.

lib/singlepass-backend/src/codegen_x64.rs

@@ -1564,10 +1564,18 @@ impl X64FunctionCode {
         if check_alignment && align != 1 {
             let tmp_aligncheck = m.acquire_temp_gpr().unwrap();
             //let tmp_mask = m.acquire_temp_gpr().unwrap();
-            a.emit_mov(Size::S32, Location::GPR(tmp_addr), Location::GPR(tmp_aligncheck));
+            a.emit_mov(
+                Size::S32,
+                Location::GPR(tmp_addr),
+                Location::GPR(tmp_aligncheck),
+            );
             //a.emit_mov(Size::S64, Location::Imm64(align - 1), Location::GPR(tmp_mask));
             //a.emit_and(Size::S64, Location::GPR(tmp_mask), Location::GPR(tmp_aligncheck));
-            a.emit_and(Size::S64, Location::Imm32(align - 1), Location::GPR(tmp_aligncheck));
+            a.emit_and(
+                Size::S64,
+                Location::Imm32(align - 1),
+                Location::GPR(tmp_aligncheck),
+            );
             a.emit_conditional_trap(Condition::NotEqual);
             //m.release_temp_gpr(tmp_mask);
             m.release_temp_gpr(tmp_aligncheck);
@@ -5455,7 +5463,8 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                 );
             }
             Operator::I32AtomicRmwAdd { ref memarg } => {
-                let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
+                let loc =
+                    get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let target =
                     get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let ret = self.machine.acquire_locations(
@@ -5466,10 +5475,7 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                 self.value_stack.push(ret);
 
                 let value = self.machine.acquire_temp_gpr().unwrap();
-                a.emit_mov(
+                a.emit_mov(Size::S32, loc, Location::GPR(value));
-                    Size::S32,
-                    loc,
-                    Location::GPR(value));
                 Self::emit_memory_op(
                     module_info,
                     &self.config,
@@ -5481,16 +5487,14 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                     4,
                     |a, _m, addr| {
                         a.emit_lock_xadd(Size::S32, Location::GPR(value), Location::Memory(addr, 0))
-                    }
+                    },
                 );
-                a.emit_mov(
+                a.emit_mov(Size::S32, Location::GPR(value), ret);
-                    Size::S32,
-                    Location::GPR(value),
-                    ret);
                 self.machine.release_temp_gpr(value);
             }
             Operator::I32AtomicRmw8UAdd { ref memarg } => {
-                let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
+                let loc =
+                    get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let target =
                     get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let ret = self.machine.acquire_locations(
@@ -5501,11 +5505,7 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                 self.value_stack.push(ret);
 
                 let value = self.machine.acquire_temp_gpr().unwrap();
-                a.emit_movzx(
+                a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value));
-                    Size::S8,
-                    loc,
-                    Size::S32,
-                    Location::GPR(value));
                 Self::emit_memory_op(
                     module_info,
                     &self.config,
@@ -5517,16 +5517,14 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                     4,
                     |a, _m, addr| {
                         a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0))
-                    }
+                    },
                 );
-                a.emit_mov(
+                a.emit_mov(Size::S32, Location::GPR(value), ret);
-                    Size::S32,
-                    Location::GPR(value),
-                    ret);
                 self.machine.release_temp_gpr(value);
             }
             Operator::I32AtomicRmw16UAdd { ref memarg } => {
-                let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
+                let loc =
+                    get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let target =
                     get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let ret = self.machine.acquire_locations(
@@ -5537,11 +5535,7 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                 self.value_stack.push(ret);
 
                 let value = self.machine.acquire_temp_gpr().unwrap();
-                a.emit_movzx(
+                a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value));
-                    Size::S16,
-                    loc,
-                    Size::S32,
-                    Location::GPR(value));
                 Self::emit_memory_op(
                     module_info,
                     &self.config,
@@ -5553,16 +5547,14 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                     4,
                     |a, _m, addr| {
                         a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0))
-                    }
+                    },
                 );
-                a.emit_mov(
+                a.emit_mov(Size::S32, Location::GPR(value), ret);
-                    Size::S32,
-                    Location::GPR(value),
-                    ret);
                 self.machine.release_temp_gpr(value);
             }
             Operator::I32AtomicRmwSub { ref memarg } => {
-                let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
+                let loc =
+                    get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let target =
                     get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let ret = self.machine.acquire_locations(
@@ -5573,10 +5565,7 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                 self.value_stack.push(ret);
 
                 let value = self.machine.acquire_temp_gpr().unwrap();
-                a.emit_mov(
+                a.emit_mov(Size::S32, loc, Location::GPR(value));
-                    Size::S32,
-                    loc,
-                    Location::GPR(value));
                 a.emit_neg(Size::S32, Location::GPR(value));
                 Self::emit_memory_op(
                     module_info,
@@ -5589,16 +5578,14 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                     4,
                     |a, _m, addr| {
                         a.emit_lock_xadd(Size::S32, Location::GPR(value), Location::Memory(addr, 0))
-                    }
+                    },
                 );
-                a.emit_mov(
+                a.emit_mov(Size::S32, Location::GPR(value), ret);
-                    Size::S32,
-                    Location::GPR(value),
-                    ret);
                 self.machine.release_temp_gpr(value);
             }
             Operator::I32AtomicRmw8USub { ref memarg } => {
-                let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
+                let loc =
+                    get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let target =
                     get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let ret = self.machine.acquire_locations(
@@ -5626,16 +5613,14 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                     4,
                     |a, _m, addr| {
                         a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0))
-                    }
+                    },
                 );
-                a.emit_mov(
+                a.emit_mov(Size::S32, Location::GPR(value), ret);
-                    Size::S32,
-                    Location::GPR(value),
-                    ret);
                 self.machine.release_temp_gpr(value);
             }
             Operator::I32AtomicRmw16USub { ref memarg } => {
-                let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
+                let loc =
+                    get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let target =
                     get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
                 let ret = self.machine.acquire_locations(
@@ -5646,11 +5631,7 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                 self.value_stack.push(ret);
 
                 let value = self.machine.acquire_temp_gpr().unwrap();
-                a.emit_movzx(
+                a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value));
-                    Size::S16,
-                    loc,
-                    Size::S32,
-                    Location::GPR(value));
                 a.emit_neg(Size::S16, Location::GPR(value));
                 Self::emit_memory_op(
                     module_info,
@@ -5663,12 +5644,66 @@ impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
                     4,
                     |a, _m, addr| {
                         a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0))
-                    }
+                    },
                 );
-                a.emit_mov(
+                a.emit_mov(Size::S32, Location::GPR(value), ret);
+                self.machine.release_temp_gpr(value);
+            }
+            Operator::I32AtomicRmwAnd { ref memarg } => {
+                let loc =
+                    get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
+                let target =
+                    get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap());
+                let ret = self.machine.acquire_locations(
+                    a,
+                    &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))],
+                    false,
+                )[0];
+                self.value_stack.push(ret);
+
+                let retry = a.get_label();
+
+                let value = self.machine.acquire_temp_gpr().unwrap();
+                let compare = GPR::RAX;
+
+                a.emit_label(retry);
+
+                a.emit_mov(Size::S32, loc, Location::GPR(value));
+                Self::emit_memory_op(
+                    module_info,
+                    &self.config,
+                    a,
+                    &mut self.machine,
+                    target,
+                    memarg,
+                    true,
+                    4,
+                    |a, _m, addr| {
+                        a.emit_mov(Size::S32, Location::Memory(addr, 0), Location::GPR(compare))
+                    },
+                );
+                a.emit_and(Size::S32, Location::GPR(compare), Location::GPR(value));
+                Self::emit_memory_op(
+                    module_info,
+                    &self.config,
+                    a,
+                    &mut self.machine,
+                    target,
+                    memarg,
+                    true,
+                    4,
+                    |a, _m, addr| {
+                        a.emit_lock_cmpxchg(
                             Size::S32,
                             Location::GPR(value),
-                    ret);
+                            Location::Memory(addr, 0),
+                        )
+                    },
+                );
+
+                a.emit_jmp(Condition::NotEqual, retry);
+
+                a.emit_mov(Size::S32, Location::GPR(value), ret);
                 self.machine.release_temp_gpr(value);
             }
             _ => {