/*
 * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

#include "asm/macroAssembler.inline.hpp"
#include "code/codeBlob.hpp"
#include "code/vmreg.inline.hpp"
#include "gc/z/zAddress.hpp"
#include "gc/z/zBarrier.inline.hpp"
#include "gc/z/zBarrierSet.hpp"
#include "gc/z/zBarrierSetAssembler.hpp"
#include "gc/z/zBarrierSetRuntime.hpp"
#include "gc/z/zThreadLocalData.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_aarch64.hpp"
#include "runtime/icache.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/macros.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/z/c1/zBarrierSetC1.hpp"
#endif // COMPILER1
#ifdef COMPILER2
#include "gc/z/c2/zBarrierSetC2.hpp"
#include "opto/output.hpp"
#endif // COMPILER2

#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
#else
#define BLOCK_COMMENT(str) __ block_comment(str)
#endif

#undef __
#define __ masm->

// Helper for saving and restoring registers across a runtime call that does
// not have any live vector registers.
class ZRuntimeCallSpill {
private:
  MacroAssembler* _masm;
  Register _result;

  void save() {
    MacroAssembler* masm = _masm;

    __ enter(true /* strip_ret_addr */);
    if (_result != noreg) {
      __ push_call_clobbered_registers_except(RegSet::of(_result));
    } else {
      __ push_call_clobbered_registers();
    }
  }

  void restore() {
    MacroAssembler* masm = _masm;

    if (_result != noreg) {
      // Make sure _result has the return value.
      if (_result != r0) {
        __ mov(_result, r0);
      }

      __ pop_call_clobbered_registers_except(RegSet::of(_result));
    } else {
      __ pop_call_clobbered_registers();
    }
    __ leave();
  }

public:
  ZRuntimeCallSpill(MacroAssembler* masm, Register result)
    : _masm(masm),
      _result(result) {
    save();
  }

  ~ZRuntimeCallSpill() {
    restore();
  }
};

void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
                                   DecoratorSet decorators,
                                   BasicType type,
                                   Register dst,
                                   Address src,
                                   Register tmp1,
                                   Register tmp2) {
  if (!ZBarrierSet::barrier_needed(decorators, type)) {
    // Barrier not needed
    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
    return;
  }

  assert_different_registers(tmp1, tmp2, src.base(), noreg);
  assert_different_registers(tmp1, tmp2, src.index());
  assert_different_registers(tmp1, tmp2, dst, noreg);
  assert_different_registers(tmp2, rscratch1);

  Label done;
  Label uncolor;

  // Load bad mask into scratch register.
  const bool on_non_strong =
    (decorators & ON_WEAK_OOP_REF) != 0 ||
    (decorators & ON_PHANTOM_OOP_REF) != 0;

  if (on_non_strong) {
    __ ldr(tmp1, mark_bad_mask_from_thread(rthread));
  } else {
    __ ldr(tmp1, load_bad_mask_from_thread(rthread));
  }

  __ lea(tmp2, src);
  __ ldr(dst, tmp2);

  // Test reference against bad mask. If mask bad, then we need to fix it up.
  __ tst(dst, tmp1);
  __ br(Assembler::EQ, uncolor);

  {
    // Call VM
    ZRuntimeCallSpill rcs(masm, dst);

    if (c_rarg0 != dst) {
      __ mov(c_rarg0, dst);
    }
    __ mov(c_rarg1, tmp2);

    __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
  }

  // Slow-path has already uncolored
  __ b(done);

  __ bind(uncolor);

  // Remove the color bits
  __ lsr(dst, dst, ZPointerLoadShift);

  __ bind(done);
}

void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm,
                                              Address ref_addr,
                                              Register rnew_zaddress,
                                              Register rnew_zpointer,
                                              Register rtmp,
                                              bool in_nmethod,
                                              bool is_atomic,
                                              Label& medium_path,
                                              Label& medium_path_continuation) const {
  assert_different_registers(ref_addr.base(), rnew_zpointer, rtmp);
  assert_different_registers(ref_addr.index(), rnew_zpointer, rtmp);
  assert_different_registers(rnew_zaddress, rnew_zpointer, rtmp);

  if (in_nmethod) {
    if (is_atomic) {
      __ ldrh(rtmp, ref_addr);
      // Atomic operations must ensure that the contents of memory are store-good before
      // an atomic operation can execute.
      // A not relocatable object could have spurious raw null pointers in its fields after
      // getting promoted to the old generation.
      __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
      __ movzw(rnew_zpointer, barrier_Relocation::unpatched);
      __ cmpw(rtmp, rnew_zpointer);
    } else {
      __ ldr(rtmp, ref_addr);
      // Stores on relocatable objects never need to deal with raw null pointers in fields.
      // Raw null pointers may only exist in the young generation, as they get pruned when
      // the object is relocated to old. And no pre-write barrier needs to perform any action
      // in the young generation.
      __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreBadBeforeMov);
      __ movzw(rnew_zpointer, barrier_Relocation::unpatched);
      __ tst(rtmp, rnew_zpointer);
    }
    __ br(Assembler::NE, medium_path);
    __ bind(medium_path_continuation);
    assert_different_registers(rnew_zaddress, rnew_zpointer);
    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
    __ movzw(rnew_zpointer, barrier_Relocation::unpatched);
    __ orr(rnew_zpointer, rnew_zpointer, rnew_zaddress, Assembler::LSL, ZPointerLoadShift);
  } else {
    assert(!is_atomic, "atomics outside of nmethods not supported");
    __ lea(rtmp, ref_addr);
    __ ldr(rtmp, rtmp);
    __ ldr(rnew_zpointer, Address(rthread, ZThreadLocalData::store_bad_mask_offset()));
    __ tst(rtmp, rnew_zpointer);
    __ br(Assembler::NE, medium_path);
    __ bind(medium_path_continuation);
    if (rnew_zaddress == noreg) {
      __ eor(rnew_zpointer, rnew_zpointer, rnew_zpointer);
    } else {
      __ mov(rnew_zpointer, rnew_zaddress);
    }

    // Load the current good shift, and add the color bits
    __ lsl(rnew_zpointer, rnew_zpointer, ZPointerLoadShift);
    __ ldr(rtmp, Address(rthread, ZThreadLocalData::store_good_mask_offset()));
    __ orr(rnew_zpointer, rnew_zpointer, rtmp);
  }
}

static void store_barrier_buffer_add(MacroAssembler* masm,
                                     Address ref_addr,
                                     Register tmp1,
                                     Register tmp2,
                                     Label& slow_path) {
  Address buffer(rthread, ZThreadLocalData::store_barrier_buffer_offset());
  assert_different_registers(ref_addr.base(), ref_addr.index(), tmp1, tmp2);

  __ ldr(tmp1, buffer);

  // Combined pointer bump and check if the buffer is disabled or full
  __ ldr(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));
  __ cmp(tmp2, (uint8_t)0);
  __ br(Assembler::EQ, slow_path);

  // Bump the pointer
  __ sub(tmp2, tmp2, sizeof(ZStoreBarrierEntry));
  __ str(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));

  // Compute the buffer entry address
  __ lea(tmp2, Address(tmp2, ZStoreBarrierBuffer::buffer_offset()));
  __ add(tmp2, tmp2, tmp1);

  // Compute and log the store address
  __ lea(tmp1, ref_addr);
  __ str(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::p_offset())));

  // Load and log the prev value
  __ ldr(tmp1, tmp1);
  __ str(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::prev_offset())));
}

void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
                                                Address ref_addr,
                                                Register rtmp1,
                                                Register rtmp2,
                                                Register rtmp3,
                                                bool is_native,
                                                bool is_atomic,
                                                Label& medium_path_continuation,
                                                Label& slow_path,
                                                Label& slow_path_continuation) const {
  assert_different_registers(ref_addr.base(), ref_addr.index(), rtmp1, rtmp2);

  // The reason to end up in the medium path is that the pre-value was not 'good'.

  if (is_native) {
    __ b(slow_path);
    __ bind(slow_path_continuation);
    __ b(medium_path_continuation);
  } else if (is_atomic) {
    // Atomic accesses can get to the medium fast path because the value was a
    // raw null value. If it was not null, then there is no doubt we need to take a slow path.
    __ lea(rtmp2, ref_addr);
    __ ldr(rtmp1, rtmp2);
    __ cbnz(rtmp1, slow_path);

    // If we get this far, we know there is a young raw null value in the field.
    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
    __ movzw(rtmp1, barrier_Relocation::unpatched);
    __ cmpxchg(rtmp2, zr, rtmp1,
               Assembler::xword,
               false /* acquire */, false /* release */, true /* weak */,
               rtmp3);
    __ br(Assembler::NE, slow_path);

    __ bind(slow_path_continuation);
    __ b(medium_path_continuation);
  } else {
    // A non-atomic relocatable object won't get to the medium fast path due to a
    // raw null in the young generation. We only get here because the field is bad.
    // In this path we don't need any self healing, so we can avoid a runtime call
    // most of the time by buffering the store barrier to be applied lazily.
    store_barrier_buffer_add(masm,
                             ref_addr,
                             rtmp1,
                             rtmp2,
                             slow_path);
    __ bind(slow_path_continuation);
    __ b(medium_path_continuation);
  }
}

void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
                                    DecoratorSet decorators,
                                    BasicType type,
                                    Address dst,
                                    Register val,
                                    Register tmp1,
                                    Register tmp2,
                                    Register tmp3) {
  if (!ZBarrierSet::barrier_needed(decorators, type)) {
    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
    return;
  }

  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;

  assert_different_registers(val, tmp1, dst.base(), dst.index());

  if (dest_uninitialized) {
    if (val == noreg) {
      __ eor(tmp1, tmp1, tmp1);
    } else {
      __ mov(tmp1, val);
    }
    // Add the color bits
    __ lsl(tmp1, tmp1, ZPointerLoadShift);
    __ ldr(tmp2, Address(rthread, ZThreadLocalData::store_good_mask_offset()));
    __ orr(tmp1, tmp2, tmp1);
  } else {
    Label done;
    Label medium;
    Label medium_continuation;
    Label slow;
    Label slow_continuation;
    store_barrier_fast(masm, dst, val, tmp1, tmp2, false, false, medium, medium_continuation);
    __ b(done);
    __ bind(medium);
    store_barrier_medium(masm,
                         dst,
                         tmp1,
                         tmp2,
                         noreg /* tmp3 */,
                         false /* is_native */,
                         false /* is_atomic */,
                         medium_continuation,
                         slow,
                         slow_continuation);

    __ bind(slow);
    {
      // Call VM
      ZRuntimeCallSpill rcs(masm, noreg);
      __ lea(c_rarg0, dst);
      __ MacroAssembler::call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
    }

    __ b(slow_continuation);
    __ bind(done);
  }

  // Store value
  BarrierSetAssembler::store_at(masm, decorators, type, dst, tmp1, tmp2, tmp3, noreg);
}

static FloatRegister z_copy_load_bad_vreg = v17;
static FloatRegister z_copy_store_good_vreg = v18;
static FloatRegister z_copy_store_bad_vreg = v19;

static void load_wide_arraycopy_masks(MacroAssembler* masm) {
  __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorLoadBadMask));
  __ ldrq(z_copy_load_bad_vreg, Address(rscratch1, 0));
  __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorStoreBadMask));
  __ ldrq(z_copy_store_bad_vreg, Address(rscratch1, 0));
  __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorStoreGoodMask));
  __ ldrq(z_copy_store_good_vreg, Address(rscratch1, 0));
}

class ZCopyRuntimeCallSpill {
private:
  MacroAssembler* _masm;
  Register _result;

  void save() {
    MacroAssembler* masm = _masm;

    __ enter(true /* strip_ret_addr */);
    if (_result != noreg) {
      __ push(__ call_clobbered_gp_registers() - RegSet::of(_result), sp);
    } else {
      __ push(__ call_clobbered_gp_registers(), sp);
    }
    int neonSize = wordSize * 2;
    __ sub(sp, sp, 4 * neonSize);
    __ st1(v0, v1, v2, v3, Assembler::T16B, Address(sp, 0));
    __ sub(sp, sp, 4 * neonSize);
    __ st1(v4, v5, v6, v7, Assembler::T16B, Address(sp, 0));
    __ sub(sp, sp, 4 * neonSize);
    __ st1(v16, v17, v18, v19, Assembler::T16B, Address(sp, 0));
  }

  void restore() {
    MacroAssembler* masm = _masm;

    int neonSize = wordSize * 2;
    __ ld1(v16, v17, v18, v19, Assembler::T16B, Address(sp, 0));
    __ add(sp, sp, 4 * neonSize);
    __ ld1(v4, v5, v6, v7, Assembler::T16B, Address(sp, 0));
    __ add(sp, sp, 4 * neonSize);
    __ ld1(v0, v1, v2, v3, Assembler::T16B, Address(sp, 0));
    __ add(sp, sp, 4 * neonSize);
    if (_result != noreg) {
      if (_result != r0) {
        __ mov(_result, r0);
      }
      __ pop(__ call_clobbered_gp_registers() - RegSet::of(_result), sp);
    } else {
      __ pop(__ call_clobbered_gp_registers(), sp);
    }
    __ leave();
  }

public:
  ZCopyRuntimeCallSpill(MacroAssembler* masm, Register result)
    : _masm(masm),
      _result(result) {
    save();
  }

  ~ZCopyRuntimeCallSpill() {
    restore();
  }
};

void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
                                              DecoratorSet decorators,
                                              bool is_oop,
                                              Register src,
                                              Register dst,
                                              Register count,
                                              RegSet saved_regs) {
  if (!is_oop) {
    // Barrier not needed
    return;
  }

  BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");

  load_wide_arraycopy_masks(masm);

  BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
}

static void copy_load_barrier(MacroAssembler* masm,
                              Register ref,
                              Address src,
                              Register tmp) {
  Label done;

  __ ldr(tmp, Address(rthread, ZThreadLocalData::load_bad_mask_offset()));

  // Test reference against bad mask. If mask bad, then we need to fix it up.
  __ tst(ref, tmp);
  __ br(Assembler::EQ, done);

  {
    // Call VM
    ZCopyRuntimeCallSpill rcs(masm, ref);

    __ lea(c_rarg1, src);

    if (c_rarg0 != ref) {
      __ mov(c_rarg0, ref);
    }

    __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(IN_HEAP | ON_STRONG_OOP_REF), 2);
  }

  // Slow-path has uncolored; revert
  __ lsl(ref, ref, ZPointerLoadShift);

  __ bind(done);
}

static void copy_load_barrier(MacroAssembler* masm,
                              FloatRegister ref,
                              Address src,
                              Register tmp1,
                              Register tmp2,
                              FloatRegister vec_tmp) {
  Label done;

  // Test reference against bad mask. If mask bad, then we need to fix it up.
  __ andr(vec_tmp, Assembler::T16B, ref, z_copy_load_bad_vreg);
  __ umaxv(vec_tmp, Assembler::T16B, vec_tmp);
  __ fcmpd(vec_tmp, 0.0);
  __ br(Assembler::EQ, done);

  __ umov(tmp2, ref, Assembler::D, 0);
  copy_load_barrier(masm, tmp2, Address(src.base(), src.offset() + 0), tmp1);
  __ mov(ref, __ D, 0, tmp2);

  __ umov(tmp2, ref, Assembler::D, 1);
  copy_load_barrier(masm, tmp2, Address(src.base(), src.offset() + 8), tmp1);
  __ mov(ref, __ D, 1, tmp2);

  __ bind(done);
}

static void copy_store_barrier(MacroAssembler* masm,
                               Register pre_ref,
                               Register new_ref,
                               Address src,
                               Register tmp1,
                               Register tmp2) {
  Label done;
  Label slow;

  // Test reference against bad mask. If mask bad, then we need to fix it up.
  __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_bad_mask_offset()));
  __ tst(pre_ref, tmp1);
  __ br(Assembler::EQ, done);

  store_barrier_buffer_add(masm, src, tmp1, tmp2, slow);
  __ b(done);

  __ bind(slow);
  {
    // Call VM
    ZCopyRuntimeCallSpill rcs(masm, noreg);

    __ lea(c_rarg0, src);

    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
  }

  __ bind(done);

  if (new_ref != noreg) {
    // Set store-good color, replacing whatever color was there before
    __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_good_mask_offset()));
    __ bfi(new_ref, tmp1, 0, 16);
  }
}

static void copy_store_barrier(MacroAssembler* masm,
                               FloatRegister pre_ref,
                               FloatRegister new_ref,
                               Address src,
                               Register tmp1,
                               Register tmp2,
                               Register tmp3,
                               FloatRegister vec_tmp) {
  Label done;

  // Test reference against bad mask. If mask bad, then we need to fix it up.
  __ andr(vec_tmp, Assembler::T16B, pre_ref, z_copy_store_bad_vreg);
  __ umaxv(vec_tmp, Assembler::T16B, vec_tmp);
  __ fcmpd(vec_tmp, 0.0);
  __ br(Assembler::EQ, done);

  // Extract the 2 oops from the pre_ref vector register
  __ umov(tmp2, pre_ref, Assembler::D, 0);
  copy_store_barrier(masm, tmp2, noreg, Address(src.base(), src.offset() + 0), tmp1, tmp3);

  __ umov(tmp2, pre_ref, Assembler::D, 1);
  copy_store_barrier(masm, tmp2, noreg, Address(src.base(), src.offset() + 8), tmp1, tmp3);

  __ bind(done);

  // Remove any bad colors
  __ bic(new_ref, Assembler::T16B, new_ref, z_copy_store_bad_vreg);
  // Add good colors
  __ orr(new_ref, Assembler::T16B, new_ref, z_copy_store_good_vreg);
}

class ZAdjustAddress {
private:
  MacroAssembler* _masm;
  Address _addr;
  int _pre_adjustment;
  int _post_adjustment;

  void pre() {
    if (_pre_adjustment != 0) {
      _masm->add(_addr.base(), _addr.base(), _addr.offset());
    }
  }

  void post() {
    if (_post_adjustment != 0) {
      _masm->add(_addr.base(), _addr.base(), _addr.offset());
    }
  }

public:
  ZAdjustAddress(MacroAssembler* masm, Address addr)
    : _masm(masm),
      _addr(addr),
      _pre_adjustment(addr.getMode() == Address::pre ? addr.offset() : 0),
      _post_adjustment(addr.getMode() == Address::post ? addr.offset() : 0) {
    pre();
  }

  ~ZAdjustAddress() {
    post();
  }

  Address address() {
    if (_pre_adjustment != 0 || _post_adjustment != 0) {
      return Address(_addr.base(), 0);
    } else {
      return Address(_addr.base(), _addr.offset());
    }
  }
};

void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm,
                                        DecoratorSet decorators,
                                        BasicType type,
                                        size_t bytes,
                                        Register dst1,
                                        Register dst2,
                                        Address src,
                                        Register tmp) {
  if (!is_reference_type(type)) {
    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg);
    return;
  }

  ZAdjustAddress adjust(masm, src);
  src = adjust.address();

  BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg);

  if (bytes == 8) {
    copy_load_barrier(masm, dst1, src, tmp);
  } else if (bytes == 16) {
    copy_load_barrier(masm, dst1, Address(src.base(), src.offset() + 0), tmp);
    copy_load_barrier(masm, dst2, Address(src.base(), src.offset() + 8), tmp);
  } else {
    ShouldNotReachHere();
  }
  if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
    __ lsr(dst1, dst1, ZPointerLoadShift);
  }
}

void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm,
                                         DecoratorSet decorators,
                                         BasicType type,
                                         size_t bytes,
                                         Address dst,
                                         Register src1,
                                         Register src2,
                                         Register tmp1,
                                         Register tmp2,
                                         Register tmp3) {
  if (!is_reference_type(type)) {
    BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg);
    return;
  }

  ZAdjustAddress adjust(masm, dst);
  dst = adjust.address();

  if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
    __ lsl(src1, src1, ZPointerLoadShift);
  }

  bool is_dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;

  if (is_dest_uninitialized) {
    __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_good_mask_offset()));
    if (bytes == 8) {
      __ bfi(src1, tmp1, 0, 16);
    } else if (bytes == 16) {
      __ bfi(src1, tmp1, 0, 16);
      __ bfi(src2, tmp1, 0, 16);
    } else {
      ShouldNotReachHere();
    }
  } else {
    // Store barrier pre values and color new values
    if (bytes == 8) {
      __ ldr(tmp1, dst);
      copy_store_barrier(masm, tmp1, src1, dst, tmp2, tmp3);
    } else if (bytes == 16) {
      Address dst1(dst.base(), dst.offset() + 0);
      Address dst2(dst.base(), dst.offset() + 8);

      __ ldr(tmp1, dst1);
      copy_store_barrier(masm, tmp1, src1, dst1, tmp2, tmp3);

      __ ldr(tmp1, dst2);
      copy_store_barrier(masm, tmp1, src2, dst2, tmp2, tmp3);
    } else {
      ShouldNotReachHere();
    }
  }

  // Store new values
  BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg);
}

void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm,
                                        DecoratorSet decorators,
                                        BasicType type,
                                        size_t bytes,
                                        FloatRegister dst1,
                                        FloatRegister dst2,
                                        Address src,
                                        Register tmp1,
                                        Register tmp2,
                                        FloatRegister vec_tmp) {
  if (!is_reference_type(type)) {
    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg, noreg, fnoreg);
    return;
  }

  ZAdjustAddress adjust(masm, src);
  src = adjust.address();

  BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg, noreg, fnoreg);

  if (bytes == 32) {
    copy_load_barrier(masm, dst1, Address(src.base(), src.offset() + 0), tmp1, tmp2, vec_tmp);
    copy_load_barrier(masm, dst2, Address(src.base(), src.offset() + 16), tmp1, tmp2, vec_tmp);
  } else {
    ShouldNotReachHere();
  }
}

void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm,
                                         DecoratorSet decorators,
                                         BasicType type,
                                         size_t bytes,
                                         Address dst,
                                         FloatRegister src1,
                                         FloatRegister src2,
                                         Register tmp1,
                                         Register tmp2,
                                         Register tmp3,
                                         FloatRegister vec_tmp1,
                                         FloatRegister vec_tmp2,
                                         FloatRegister vec_tmp3) {
  if (!is_reference_type(type)) {
    BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg, fnoreg, fnoreg, fnoreg);
    return;
  }

  bool is_dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;

  ZAdjustAddress adjust(masm, dst);
  dst = adjust.address();

  if (is_dest_uninitialized) {
    if (bytes == 32) {
      __ bic(src1, Assembler::T16B, src1, z_copy_store_bad_vreg);
      __ orr(src1, Assembler::T16B, src1, z_copy_store_good_vreg);
      __ bic(src2, Assembler::T16B, src2, z_copy_store_bad_vreg);
      __ orr(src2, Assembler::T16B, src2, z_copy_store_good_vreg);
    } else {
      ShouldNotReachHere();
    }
  } else {
    // Load pre values
    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, vec_tmp1, vec_tmp2, dst, noreg, noreg, fnoreg);

    // Store barrier pre values and color new values
    if (bytes == 32) {
      copy_store_barrier(masm, vec_tmp1, src1, Address(dst.base(), dst.offset() + 0), tmp1, tmp2, tmp3, vec_tmp3);
      copy_store_barrier(masm, vec_tmp2, src2, Address(dst.base(), dst.offset() + 16), tmp1, tmp2, tmp3, vec_tmp3);
    } else {
      ShouldNotReachHere();
    }
  }

  // Store new values
  BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg, fnoreg, fnoreg, fnoreg);
}

void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
                                                         Register jni_env,
                                                         Register robj,
                                                         Register tmp,
                                                         Label& slowpath) {
  BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");

  Label done, tagged, weak_tagged, uncolor;

  // Test for tag
  __ tst(robj, JNIHandles::tag_mask);
  __ br(Assembler::NE, tagged);

  // Resolve local handle
  __ ldr(robj, robj);
  __ b(done);

  __ bind(tagged);

  // Test for weak tag
  __ tst(robj, JNIHandles::TypeTag::weak_global);
  __ br(Assembler::NE, weak_tagged);

  // Resolve global handle
  __ ldr(robj, Address(robj, -JNIHandles::TypeTag::global));
  __ lea(tmp, load_bad_mask_from_jni_env(jni_env));
  __ ldr(tmp, tmp);
  __ tst(robj, tmp);
  __ br(Assembler::NE, slowpath);
  __ b(uncolor);

  __ bind(weak_tagged);

  // Resolve weak handle
  __ ldr(robj, Address(robj, -JNIHandles::TypeTag::weak_global));
  __ lea(tmp, mark_bad_mask_from_jni_env(jni_env));
  __ ldr(tmp, tmp);
  __ tst(robj, tmp);
  __ br(Assembler::NE, slowpath);

  __ bind(uncolor);

  // Uncolor
  __ lsr(robj, robj, ZPointerLoadShift);

  __ bind(done);

  BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
}

static uint16_t patch_barrier_relocation_value(int format) {
  switch (format) {
  case ZBarrierRelocationFormatLoadGoodBeforeTbX:
    return (uint16_t)exact_log2(ZPointerRemapped);

  case ZBarrierRelocationFormatMarkBadBeforeMov:
    return (uint16_t)ZPointerMarkBadMask;

  case ZBarrierRelocationFormatStoreGoodBeforeMov:
    return (uint16_t)ZPointerStoreGoodMask;

  case ZBarrierRelocationFormatStoreBadBeforeMov:
    return (uint16_t)ZPointerStoreBadMask;

  default:
    ShouldNotReachHere();
    return 0;
  }
}

static void change_immediate(uint32_t& instr, uint32_t imm, uint32_t start, uint32_t end) {
  uint32_t imm_mask = ((1u << start) - 1u) ^ ((1u << (end + 1)) - 1u);
  instr &= ~imm_mask;
  instr |= imm << start;
}

void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
  const uint16_t value = patch_barrier_relocation_value(format);
  uint32_t* const patch_addr = (uint32_t*)addr;

  switch (format) {
  case ZBarrierRelocationFormatLoadGoodBeforeTbX:
    change_immediate(*patch_addr, value, 19, 23);
    break;
  case ZBarrierRelocationFormatStoreGoodBeforeMov:
  case ZBarrierRelocationFormatMarkBadBeforeMov:
  case ZBarrierRelocationFormatStoreBadBeforeMov:
    change_immediate(*patch_addr, value, 5, 20);
    break;
  default:
    ShouldNotReachHere();
  }

  OrderAccess::fence();
  ICache::invalidate_word((address)patch_addr);
}

#ifdef COMPILER1

#undef __
#define __ ce->masm()->

static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) {
  __ lsr(ref->as_register(), ref->as_register(), ZPointerLoadShift);
}

static void z_color(LIR_Assembler* ce, LIR_Opr ref) {
  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
  __ movzw(rscratch2, barrier_Relocation::unpatched);
  __ orr(ref->as_register(), rscratch2, ref->as_register(), Assembler::LSL, ZPointerLoadShift);
}

void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const {
  z_uncolor(ce, ref);
}

void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const {
  z_color(ce, ref);
}

void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce,
                                                    LIR_Opr ref,
                                                    ZLoadBarrierStubC1* stub,
                                                    bool on_non_strong) const {

  if (on_non_strong) {
    // Test against MarkBad mask
    assert_different_registers(rscratch1, rthread, ref->as_register());
    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov);
    __ movzw(rscratch1, barrier_Relocation::unpatched);
    __ tst(ref->as_register(), rscratch1);
    __ br(Assembler::NE, *stub->entry());
    z_uncolor(ce, ref);
  } else {
    Label good;
    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX);
    __ tbz(ref->as_register(), barrier_Relocation::unpatched, good);
    __ b(*stub->entry());
    __ bind(good);
    z_uncolor(ce, ref);
  }
  __ bind(*stub->continuation());
}

void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
                                                         ZLoadBarrierStubC1* stub) const {
  // Stub entry
  __ bind(*stub->entry());

  Register ref = stub->ref()->as_register();
  Register ref_addr = noreg;
  Register tmp = noreg;

  if (stub->tmp()->is_valid()) {
    // Load address into tmp register
    ce->leal(stub->ref_addr(), stub->tmp());
    ref_addr = tmp = stub->tmp()->as_pointer_register();
  } else {
    // Address already in register
    ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
  }

  assert_different_registers(ref, ref_addr, noreg);

  // Save r0 unless it is the result or tmp register
  // Set up SP to accommodate parameters and maybe r0..
  if (ref != r0 && tmp != r0) {
    __ sub(sp, sp, 32);
    __ str(r0, Address(sp, 16));
  } else {
    __ sub(sp, sp, 16);
  }

  // Setup arguments and call runtime stub
  ce->store_parameter(ref_addr, 1);
  ce->store_parameter(ref, 0);

  __ far_call(stub->runtime_stub());

  // Verify result
  __ verify_oop(r0);

  // Move result into place
  if (ref != r0) {
    __ mov(ref, r0);
  }

  // Restore r0 unless it is the result or tmp register
  if (ref != r0 && tmp != r0) {
    __ ldr(r0, Address(sp, 16));
    __ add(sp, sp, 32);
  } else {
    __ add(sp, sp, 16);
  }

  // Stub exit
  __ b(*stub->continuation());
}

void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce,
                                                     LIR_Address* addr,
                                                     LIR_Opr new_zaddress,
                                                     LIR_Opr new_zpointer,
                                                     ZStoreBarrierStubC1* stub) const {
  Register rnew_zaddress = new_zaddress->as_register();
  Register rnew_zpointer = new_zpointer->as_register();

  store_barrier_fast(ce->masm(),
                     ce->as_Address(addr),
                     rnew_zaddress,
                     rnew_zpointer,
                     rscratch2,
                     true,
                     stub->is_atomic(),
                     *stub->entry(),
                     *stub->continuation());
}

void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce,
                                                          ZStoreBarrierStubC1* stub) const {
  // Stub entry
  __ bind(*stub->entry());
  Label slow;
  Label slow_continuation;
  store_barrier_medium(ce->masm(),
                       ce->as_Address(stub->ref_addr()->as_address_ptr()),
                       rscratch2,
                       stub->new_zpointer()->as_register(),
                       rscratch1,
                       false /* is_native */,
                       stub->is_atomic(),
                       *stub->continuation(),
                       slow,
                       slow_continuation);

  __ bind(slow);

  __ lea(stub->new_zpointer()->as_register(), ce->as_Address(stub->ref_addr()->as_address_ptr()));

  __ sub(sp, sp, 16);
  // Setup arguments and call runtime stub
  assert(stub->new_zpointer()->is_valid(), "invariant");
  ce->store_parameter(stub->new_zpointer()->as_register(), 0);
  __ far_call(stub->runtime_stub());
  __ add(sp, sp, 16);

  // Stub exit
  __ b(slow_continuation);
}

#undef __
#define __ sasm->

void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
                                                                 DecoratorSet decorators) const {
  __ prologue("zgc_load_barrier stub", false);

  __ push_call_clobbered_registers_except(RegSet::of(r0));

  // Setup arguments
  __ load_parameter(0, c_rarg0);
  __ load_parameter(1, c_rarg1);

  __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);

  __ pop_call_clobbered_registers_except(RegSet::of(r0));

  __ epilogue();
}

void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
                                                                  bool self_healing) const {
  __ prologue("zgc_store_barrier stub", false);

  __ push_call_clobbered_registers();

  // Setup arguments
  __ load_parameter(0, c_rarg0);

  if (self_healing) {
    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), 1);
  } else {
    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
  }

  __ pop_call_clobbered_registers();

  __ epilogue();
}

#endif // COMPILER1

#ifdef COMPILER2

#undef __
#define __ _masm->

class ZSetupArguments {
private:
  MacroAssembler* const _masm;
  const Register        _ref;
  const Address         _ref_addr;

public:
  ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub)
    : _masm(masm),
      _ref(stub->ref()),
      _ref_addr(stub->ref_addr()) {

    // Setup arguments
    if (_ref_addr.base() == noreg) {
      // No self healing
      if (_ref != c_rarg0) {
        __ mov(c_rarg0, _ref);
      }
      __ mov(c_rarg1, 0);
    } else {
      // Self healing
      if (_ref == c_rarg0) {
        // _ref is already at correct place
        __ lea(c_rarg1, _ref_addr);
      } else if (_ref != c_rarg1) {
        // _ref is in wrong place, but not in c_rarg1, so fix it first
        __ lea(c_rarg1, _ref_addr);
        __ mov(c_rarg0, _ref);
      } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
        assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
        __ mov(c_rarg0, _ref);
        __ lea(c_rarg1, _ref_addr);
      } else {
        assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
        if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) {
          __ mov(rscratch2, c_rarg1);
          __ lea(c_rarg1, _ref_addr);
          __ mov(c_rarg0, rscratch2);
        } else {
          ShouldNotReachHere();
        }
      }
    }
  }

  ~ZSetupArguments() {
    // Transfer result
    if (_ref != r0) {
      __ mov(_ref, r0);
    }
  }
};

#undef __
#define __ masm->

void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
  BLOCK_COMMENT("ZLoadBarrierStubC2");

  // Stub entry
  if (!Compile::current()->output()->in_scratch_emit_size()) {
    __ bind(*stub->entry());
  }

  {
    SaveLiveRegisters save_live_registers(masm, stub);
    ZSetupArguments setup_arguments(masm, stub);
    __ mov(rscratch1, stub->slow_path());
    __ blr(rscratch1);
  }
  // Stub exit
  __ b(*stub->continuation());
}

void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const {
  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
  BLOCK_COMMENT("ZStoreBarrierStubC2");

  // Stub entry
  __ bind(*stub->entry());

  Label slow;
  Label slow_continuation;
  store_barrier_medium(masm,
                       stub->ref_addr(),
                       stub->new_zpointer(),
                       rscratch1,
                       rscratch2,
                       stub->is_native(),
                       stub->is_atomic(),
                       *stub->continuation(),
                       slow,
                       slow_continuation);

  __ bind(slow);

  {
    SaveLiveRegisters save_live_registers(masm, stub);
    __ lea(c_rarg0, stub->ref_addr());

    if (stub->is_native()) {
      __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr()));
    } else if (stub->is_atomic()) {
      __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()));
    } else if (stub->is_nokeepalive()) {
      __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr()));
    } else {
      __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()));
    }
    __ blr(rscratch1);
  }

  // Stub exit
  __ b(slow_continuation);
}

// Only handles forward branch jumps, target_offset >= branch_offset
static bool aarch64_test_and_branch_reachable(int branch_offset, int target_offset) {
  assert(branch_offset >= 0, "branch to stub offsets must be positive");
  assert(target_offset >= 0, "offset in stubs section must be positive");
  assert(target_offset >= branch_offset, "forward branches only, branch_offset -> target_offset");

  const int test_and_branch_delta_limit = 32 * K;

  const int test_and_branch_to_trampoline_delta = target_offset - branch_offset;

  return test_and_branch_to_trampoline_delta < test_and_branch_delta_limit;
}

ZLoadBarrierStubC2Aarch64::ZLoadBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register ref)
  : ZLoadBarrierStubC2(node, ref_addr, ref), _test_and_branch_reachable_entry(), _offset(), _deferred_emit(false), _test_and_branch_reachable(false) {}

ZLoadBarrierStubC2Aarch64::ZLoadBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register ref, int offset)
  : ZLoadBarrierStubC2(node, ref_addr, ref), _test_and_branch_reachable_entry(), _offset(offset), _deferred_emit(false), _test_and_branch_reachable(false) {
  PhaseOutput* const output = Compile::current()->output();
  if (output->in_scratch_emit_size()) {
    return;
  }
  const int code_size = output->buffer_sizing_data()->_code;
  const int offset_code = _offset;
  // Assumption that the stub can always be reached from a branch immediate. (128 M Product, 2 M Debug)
  // Same assumption is made in z_aarch64.ad
  const int trampoline_offset = trampoline_stubs_count() * NativeInstruction::instruction_size;
  _test_and_branch_reachable = aarch64_test_and_branch_reachable(offset_code, code_size + trampoline_offset);
  if (_test_and_branch_reachable) {
    inc_trampoline_stubs_count();
  }
}

int ZLoadBarrierStubC2Aarch64::get_stub_size() {
  PhaseOutput* const output = Compile::current()->output();
  assert(!output->in_scratch_emit_size(), "only used when emitting stubs");
  BufferBlob* const blob = output->scratch_buffer_blob();
  CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin());
  MacroAssembler masm(&cb);
  output->set_in_scratch_emit_size(true);
  ZLoadBarrierStubC2::emit_code(masm);
  output->set_in_scratch_emit_size(false);
  return cb.insts_size();
}

ZLoadBarrierStubC2Aarch64* ZLoadBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register ref) {
  ZLoadBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2Aarch64(node, ref_addr, ref);
  register_stub(stub);
  return stub;
}

ZLoadBarrierStubC2Aarch64* ZLoadBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register ref, int offset) {
  ZLoadBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2Aarch64(node, ref_addr, ref, offset);
  register_stub(stub);
  return stub;
}

#undef __
#define __ masm.

void ZLoadBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) {
  PhaseOutput* const output = Compile::current()->output();
  const int branch_offset = _offset;
  const int target_offset = __ offset();

  // Deferred emission, emit actual stub
  if (_deferred_emit) {
    ZLoadBarrierStubC2::emit_code(masm);
    return;
  }
  _deferred_emit = true;

  // No trampoline used, defer emission to after trampolines
  if (!_test_and_branch_reachable) {
    register_stub(this);
    return;
  }

  // Current assumption is that the barrier stubs are the first stubs emitted after the actual code
  assert(stubs_start_offset() <= output->buffer_sizing_data()->_code, "stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");

  __ bind(_test_and_branch_reachable_entry);

  // Next branch's offset is unknown, but is > branch_offset
  const int next_branch_offset = branch_offset + NativeInstruction::instruction_size;
  // If emitting the stub directly does not interfere with emission of the next trampoline then do it to avoid a double jump.
  if (aarch64_test_and_branch_reachable(next_branch_offset, target_offset + get_stub_size())) {
    // The next potential trampoline will still be reachable even if we emit the whole stub
    ZLoadBarrierStubC2::emit_code(masm);
  } else {
    // Emit trampoline and defer actual stub to the end
    assert(aarch64_test_and_branch_reachable(branch_offset, target_offset), "trampoline should be reachable");
    __ b(*ZLoadBarrierStubC2::entry());
    register_stub(this);
  }
}

bool ZLoadBarrierStubC2Aarch64::is_test_and_branch_reachable() {
  return _test_and_branch_reachable;
}

Label* ZLoadBarrierStubC2Aarch64::entry() {
  if (_test_and_branch_reachable) {
    return &_test_and_branch_reachable_entry;
  }
  return ZBarrierStubC2::entry();
}

ZStoreBarrierStubC2Aarch64::ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive)
  : ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic, is_nokeepalive), _deferred_emit(false) {}

ZStoreBarrierStubC2Aarch64* ZStoreBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive) {
  ZStoreBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2Aarch64(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic, is_nokeepalive);
  register_stub(stub);
  return stub;
}

void ZStoreBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) {
  if (_deferred_emit) {
    ZStoreBarrierStubC2::emit_code(masm);
    return;
  }
  // Defer emission of store barriers so that trampolines are emitted first
  _deferred_emit = true;
  register_stub(this);
}

#undef __

#endif // COMPILER2

#undef __
#define __ masm->

void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
  // C1 calls verfy_oop in the middle of barriers, before they have been uncolored
  // and after being colored. Therefore, we must deal with colored oops as well.
  Label done;
  Label check_oop;
  Label check_zaddress;
  int color_bits = ZPointerRemappedShift + ZPointerRemappedBits;

  uintptr_t shifted_base_start_mask = (UCONST64(1) << (ZAddressHeapBaseShift + color_bits + 1)) - 1;
  uintptr_t shifted_base_end_mask = (UCONST64(1) << (ZAddressHeapBaseShift + 1)) - 1;
  uintptr_t shifted_base_mask = shifted_base_start_mask ^ shifted_base_end_mask;

  uintptr_t shifted_address_end_mask = (UCONST64(1) << (color_bits + 1)) - 1;
  uintptr_t shifted_address_mask = shifted_address_end_mask ^ (uintptr_t)CONST64(-1);

  __ get_nzcv(tmp2);

  // Check colored null
  __ mov(tmp1, shifted_address_mask);
  __ tst(tmp1, obj);
  __ br(Assembler::EQ, done);

  // Check for zpointer
  __ mov(tmp1, shifted_base_mask);
  __ tst(tmp1, obj);
  __ br(Assembler::EQ, check_oop);

  // Uncolor presumed zpointer
  __ lsr(obj, obj, ZPointerLoadShift);

  __ b(check_zaddress);

  __ bind(check_oop);

  // make sure klass is 'reasonable', which is not zero.
  __ load_klass(tmp1, obj);  // get klass
  __ tst(tmp1, tmp1);
  __ br(Assembler::EQ, error); // if klass is null it is broken

  __ bind(check_zaddress);
  // Check if the oop is in the right area of memory
  __ mov(tmp1, (intptr_t) Universe::verify_oop_mask());
  __ andr(tmp1, tmp1, obj);
  __ mov(obj, (intptr_t) Universe::verify_oop_bits());
  __ cmp(tmp1, obj);
  __ br(Assembler::NE, error);

  __ bind(done);

  __ set_nzcv(tmp2);
}

#undef __