/*
 * Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 *
 */

#include "classfile/classLoaderData.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "interpreter/interp_masm.hpp"
#include "memory/universe.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#ifdef COMPILER2
#include "code/vmreg.inline.hpp"
#include "gc/shared/c2/barrierSetC2.hpp"
#endif // COMPILER2


#define __ masm->

void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                  Register dst, Address src, Register tmp1, Register tmp2) {

  // LR is live.  It must be saved around calls.

  bool in_heap = (decorators & IN_HEAP) != 0;
  bool in_native = (decorators & IN_NATIVE) != 0;
  bool is_not_null = (decorators & IS_NOT_NULL) != 0;
  switch (type) {
  case T_OBJECT:
  case T_ARRAY: {
    if (in_heap) {
      if (UseCompressedOops) {
        __ ldrw(dst, src);
        if (is_not_null) {
          __ decode_heap_oop_not_null(dst);
        } else {
          __ decode_heap_oop(dst);
        }
      } else {
        __ ldr(dst, src);
      }
    } else {
      assert(in_native, "why else?");
      __ ldr(dst, src);
    }
    break;
  }
  case T_BOOLEAN: __ load_unsigned_byte (dst, src); break;
  case T_BYTE:    __ load_signed_byte   (dst, src); break;
  case T_CHAR:    __ load_unsigned_short(dst, src); break;
  case T_SHORT:   __ load_signed_short  (dst, src); break;
  case T_INT:     __ ldrw               (dst, src); break;
  case T_LONG:    __ ldr                (dst, src); break;
  case T_ADDRESS: __ ldr                (dst, src); break;
  case T_FLOAT:   __ ldrs               (v0, src);  break;
  case T_DOUBLE:  __ ldrd               (v0, src);  break;
  default: Unimplemented();
  }
}

void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                   Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
  bool in_heap = (decorators & IN_HEAP) != 0;
  bool in_native = (decorators & IN_NATIVE) != 0;
  switch (type) {
  case T_OBJECT:
  case T_ARRAY: {
    val = val == noreg ? zr : val;
    if (in_heap) {
      if (UseCompressedOops) {
        assert(!dst.uses(val), "not enough registers");
        if (val != zr) {
          __ encode_heap_oop(val);
        }
        __ strw(val, dst);
      } else {
        __ str(val, dst);
      }
    } else {
      assert(in_native, "why else?");
      __ str(val, dst);
    }
    break;
  }
  case T_BOOLEAN:
    __ andw(val, val, 0x1);  // boolean is true if LSB is 1
    __ strb(val, dst);
    break;
  case T_BYTE:    __ strb(val, dst); break;
  case T_CHAR:    __ strh(val, dst); break;
  case T_SHORT:   __ strh(val, dst); break;
  case T_INT:     __ strw(val, dst); break;
  case T_LONG:    __ str (val, dst); break;
  case T_ADDRESS: __ str (val, dst); break;
  case T_FLOAT:   __ strs(v0,  dst); break;
  case T_DOUBLE:  __ strd(v0,  dst); break;
  default: Unimplemented();
  }
}

void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
                                       DecoratorSet decorators,
                                       BasicType type,
                                       size_t bytes,
                                       Register dst1,
                                       Register dst2,
                                       Address src,
                                       Register tmp) {
  if (bytes == 1) {
    assert(dst2 == noreg, "invariant");
    __ ldrb(dst1, src);
  } else if (bytes == 2) {
    assert(dst2 == noreg, "invariant");
    __ ldrh(dst1, src);
  } else if (bytes == 4) {
    assert(dst2 == noreg, "invariant");
    __ ldrw(dst1, src);
  } else if (bytes == 8) {
    assert(dst2 == noreg, "invariant");
    __ ldr(dst1, src);
  } else if (bytes == 16) {
    assert(dst2 != noreg, "invariant");
    assert(dst2 != dst1, "invariant");
    __ ldp(dst1, dst2, src);
  } else {
    // Not the right size
    ShouldNotReachHere();
  }
  if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
    __ decode_heap_oop(dst1);
  }
}

void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
                                        DecoratorSet decorators,
                                        BasicType type,
                                        size_t bytes,
                                        Address dst,
                                        Register src1,
                                        Register src2,
                                        Register tmp1,
                                        Register tmp2,
                                        Register tmp3) {
  if ((decorators & ARRAYCOPY_CHECKCAST) != 0 && UseCompressedOops) {
    __ encode_heap_oop(src1);
  }
  if (bytes == 1) {
    assert(src2 == noreg, "invariant");
    __ strb(src1, dst);
  } else if (bytes == 2) {
    assert(src2 == noreg, "invariant");
    __ strh(src1, dst);
  } else if (bytes == 4) {
    assert(src2 == noreg, "invariant");
    __ strw(src1, dst);
  } else if (bytes == 8) {
    assert(src2 == noreg, "invariant");
    __ str(src1, dst);
  } else if (bytes == 16) {
    assert(src2 != noreg, "invariant");
    assert(src2 != src1, "invariant");
    __ stp(src1, src2, dst);
  } else {
    // Not the right size
    ShouldNotReachHere();
  }
}

void BarrierSetAssembler::copy_load_at(MacroAssembler* masm,
                                       DecoratorSet decorators,
                                       BasicType type,
                                       size_t bytes,
                                       FloatRegister dst1,
                                       FloatRegister dst2,
                                       Address src,
                                       Register tmp1,
                                       Register tmp2,
                                       FloatRegister vec_tmp) {
  if (bytes == 32) {
    __ ldpq(dst1, dst2, src);
  } else {
    ShouldNotReachHere();
  }
}

void BarrierSetAssembler::copy_store_at(MacroAssembler* masm,
                                        DecoratorSet decorators,
                                        BasicType type,
                                        size_t bytes,
                                        Address dst,
                                        FloatRegister src1,
                                        FloatRegister src2,
                                        Register tmp1,
                                        Register tmp2,
                                        Register tmp3,
                                        FloatRegister vec_tmp1,
                                        FloatRegister vec_tmp2,
                                        FloatRegister vec_tmp3) {
  if (bytes == 32) {
    __ stpq(src1, src2, dst);
  } else {
    ShouldNotReachHere();
  }
}

void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
                                                        Register obj, Register tmp, Label& slowpath) {
  // If mask changes we need to ensure that the inverse is still encodable as an immediate
  STATIC_ASSERT(JNIHandles::tag_mask == 0b11);
  __ andr(obj, obj, ~JNIHandles::tag_mask);
  __ ldr(obj, Address(obj, 0));             // *obj
}

// Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
                                        Register var_size_in_bytes,
                                        int con_size_in_bytes,
                                        Register t1,
                                        Register t2,
                                        Label& slow_case) {
  assert_different_registers(obj, t2);
  assert_different_registers(obj, var_size_in_bytes);
  Register end = t2;

  // verify_tlab();

  __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
  if (var_size_in_bytes == noreg) {
    __ lea(end, Address(obj, con_size_in_bytes));
  } else {
    __ lea(end, Address(obj, var_size_in_bytes));
  }
  __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
  __ cmp(end, rscratch1);
  __ br(Assembler::HI, slow_case);

  // update the tlab top pointer
  __ str(end, Address(rthread, JavaThread::tlab_top_offset()));

  // recover var_size_in_bytes if necessary
  if (var_size_in_bytes == end) {
    __ sub(var_size_in_bytes, var_size_in_bytes, obj);
  }
  // verify_tlab();
}

static volatile uint32_t _patching_epoch = 0;

address BarrierSetAssembler::patching_epoch_addr() {
  return (address)&_patching_epoch;
}

void BarrierSetAssembler::increment_patching_epoch() {
  Atomic::inc(&_patching_epoch);
}

void BarrierSetAssembler::clear_patching_epoch() {
  _patching_epoch = 0;
}

void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slow_path, Label* continuation, Label* guard) {
  BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();

  Label local_guard;
  Label skip_barrier;
  NMethodPatchingType patching_type = nmethod_patching_type();

  if (slow_path == nullptr) {
    guard = &local_guard;
  }

  // If the slow path is out of line in a stub, we flip the condition
  Assembler::Condition condition = slow_path == nullptr ? Assembler::EQ : Assembler::NE;
  Label& barrier_target = slow_path == nullptr ? skip_barrier : *slow_path;

  __ ldrw(rscratch1, *guard);

  if (patching_type == NMethodPatchingType::stw_instruction_and_data_patch) {
    // With STW patching, no data or instructions are updated concurrently,
    // which means there isn't really any need for any fencing for neither
    // data nor instruction modifications happening concurrently. The
    // instruction patching is handled with isb fences on the way back
    // from the safepoint to Java. So here we can do a plain conditional
    // branch with no fencing.
    Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
    __ ldrw(rscratch2, thread_disarmed_addr);
    __ cmp(rscratch1, rscratch2);
  } else if (patching_type == NMethodPatchingType::conc_instruction_and_data_patch) {
    // If we patch code we need both a code patching and a loadload
    // fence. It's not super cheap, so we use a global epoch mechanism
    // to hide them in a slow path.
    // The high level idea of the global epoch mechanism is to detect
    // when any thread has performed the required fencing, after the
    // last nmethod was disarmed. This implies that the required
    // fencing has been performed for all preceding nmethod disarms
    // as well. Therefore, we do not need any further fencing.
    __ lea(rscratch2, ExternalAddress((address)&_patching_epoch));
    // Embed an artificial data dependency to order the guard load
    // before the epoch load.
    __ orr(rscratch2, rscratch2, rscratch1, Assembler::LSR, 32);
    // Read the global epoch value.
    __ ldrw(rscratch2, rscratch2);
    // Combine the guard value (low order) with the epoch value (high order).
    __ orr(rscratch1, rscratch1, rscratch2, Assembler::LSL, 32);
    // Compare the global values with the thread-local values.
    Address thread_disarmed_and_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
    __ ldr(rscratch2, thread_disarmed_and_epoch_addr);
    __ cmp(rscratch1, rscratch2);
  } else {
    assert(patching_type == NMethodPatchingType::conc_data_patch, "must be");
    // Subsequent loads of oops must occur after load of guard value.
    // BarrierSetNMethod::disarm sets guard with release semantics.
    __ membar(__ LoadLoad);
    Address thread_disarmed_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()));
    __ ldrw(rscratch2, thread_disarmed_addr);
    __ cmpw(rscratch1, rscratch2);
  }
  __ br(condition, barrier_target);

  if (slow_path == nullptr) {
    __ lea(rscratch1, RuntimeAddress(StubRoutines::method_entry_barrier()));
    __ blr(rscratch1);
    __ b(skip_barrier);

    __ bind(local_guard);

    __ emit_int32(0);   // nmethod guard value. Skipped over in common case.
  } else {
    __ bind(*continuation);
  }

  __ bind(skip_barrier);
}

void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
  Label bad_call;
  __ cbz(rmethod, bad_call);

  // Pointer chase to the method holder to find out if the method is concurrently unloading.
  Label method_live;
  __ load_method_holder_cld(rscratch1, rmethod);

  // Is it a strong CLD?
  __ ldrw(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_ref_count_offset()));
  __ cbnz(rscratch2, method_live);

  // Is it a weak but alive CLD?
  __ push(RegSet::of(r10), sp);
  __ ldr(r10, Address(rscratch1, ClassLoaderData::holder_offset()));

  __ resolve_weak_handle(r10, rscratch1, rscratch2);
  __ mov(rscratch1, r10);
  __ pop(RegSet::of(r10), sp);
  __ cbnz(rscratch1, method_live);

  __ bind(bad_call);

  __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
  __ bind(method_live);
}

void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
  // Check if the oop is in the right area of memory
  __ mov(tmp2, (intptr_t) Universe::verify_oop_mask());
  __ andr(tmp1, obj, tmp2);
  __ mov(tmp2, (intptr_t) Universe::verify_oop_bits());

  // Compare tmp1 and tmp2.  We don't use a compare
  // instruction here because the flags register is live.
  __ eor(tmp1, tmp1, tmp2);
  __ cbnz(tmp1, error);

  // make sure klass is 'reasonable', which is not zero.
  __ load_klass(obj, obj); // get klass
  __ cbz(obj, error);      // if klass is null it is broken
}

#ifdef COMPILER2

OptoReg::Name BarrierSetAssembler::encode_float_vector_register_size(const Node* node, OptoReg::Name opto_reg) {
  switch (node->ideal_reg()) {
    case Op_RegF:
      // No need to refine. The original encoding is already fine to distinguish.
      assert(opto_reg % 4 == 0, "Float register should only occupy a single slot");
      break;
    // Use different encoding values of the same fp/vector register to help distinguish different sizes.
    // Such as V16. The OptoReg::name and its corresponding slot value are
    // "V16": 64, "V16_H": 65, "V16_J": 66, "V16_K": 67.
    case Op_RegD:
    case Op_VecD:
      opto_reg &= ~3;
      opto_reg |= 1;
      break;
    case Op_VecX:
      opto_reg &= ~3;
      opto_reg |= 2;
      break;
    case Op_VecA:
      opto_reg &= ~3;
      opto_reg |= 3;
      break;
    default:
      assert(false, "unexpected ideal register");
      ShouldNotReachHere();
  }
  return opto_reg;
}

OptoReg::Name BarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
  if (!OptoReg::is_reg(opto_reg)) {
    return OptoReg::Bad;
  }

  const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
  if (vm_reg->is_FloatRegister()) {
    opto_reg = encode_float_vector_register_size(node, opto_reg);
  }

  return opto_reg;
}

#undef __
#define __ _masm->

void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
  int index = -1;
  GrowableArray<RegisterData> registers;
  VMReg prev_vm_reg = VMRegImpl::Bad();

  RegMaskIterator rmi(stub->preserve_set());
  while (rmi.has_next()) {
    OptoReg::Name opto_reg = rmi.next();
    VMReg vm_reg = OptoReg::as_VMReg(opto_reg);

    if (vm_reg->is_Register()) {
      // GPR may have one or two slots in regmask
      // Determine whether the current vm_reg is the same physical register as the previous one
      if (is_same_register(vm_reg, prev_vm_reg)) {
        registers.at(index)._slots++;
      } else {
        RegisterData reg_data = { vm_reg, 1 };
        index = registers.append(reg_data);
      }
    } else if (vm_reg->is_FloatRegister()) {
      // We have size encoding in OptoReg of stub->preserve_set()
      // After encoding, float/neon/sve register has only one slot in regmask
      // Decode it to get the actual size
      VMReg vm_reg_base = vm_reg->as_FloatRegister()->as_VMReg();
      int slots = decode_float_vector_register_size(opto_reg);
      RegisterData reg_data = { vm_reg_base, slots };
      index = registers.append(reg_data);
    } else if (vm_reg->is_PRegister()) {
      // PRegister has only one slot in regmask
      RegisterData reg_data = { vm_reg, 1 };
      index = registers.append(reg_data);
    } else {
      assert(false, "Unknown register type");
      ShouldNotReachHere();
    }
    prev_vm_reg = vm_reg;
  }

  // Record registers that needs to be saved/restored
  for (GrowableArrayIterator<RegisterData> it = registers.begin(); it != registers.end(); ++it) {
    RegisterData reg_data = *it;
    VMReg vm_reg = reg_data._reg;
    int slots = reg_data._slots;
    if (vm_reg->is_Register()) {
      assert(slots == 1 || slots == 2, "Unexpected register save size");
      _gp_regs += RegSet::of(vm_reg->as_Register());
    } else if (vm_reg->is_FloatRegister()) {
      if (slots == 1 || slots == 2) {
        _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
      } else if (slots == 4) {
        _neon_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
      } else {
        assert(slots == Matcher::scalable_vector_reg_size(T_FLOAT), "Unexpected register save size");
        _sve_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
      }
    } else {
      assert(vm_reg->is_PRegister() && slots == 1, "Unknown register type");
      _p_regs += PRegSet::of(vm_reg->as_PRegister());
    }
  }

  // Remove C-ABI SOE registers and scratch regs
  _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9);

  // Remove C-ABI SOE fp registers
  _fp_regs -= FloatRegSet::range(v8, v15);
}

enum RC SaveLiveRegisters::rc_class(VMReg reg) {
  if (reg->is_reg()) {
    if (reg->is_Register()) {
      return rc_int;
    } else if (reg->is_FloatRegister()) {
      return rc_float;
    } else if (reg->is_PRegister()) {
      return rc_predicate;
    }
  }
  if (reg->is_stack()) {
    return rc_stack;
  }
  return rc_bad;
}

bool SaveLiveRegisters::is_same_register(VMReg reg1, VMReg reg2) {
  if (reg1 == reg2) {
    return true;
  }
  if (rc_class(reg1) == rc_class(reg2)) {
    if (reg1->is_Register()) {
      return reg1->as_Register() == reg2->as_Register();
    } else if (reg1->is_FloatRegister()) {
      return reg1->as_FloatRegister() == reg2->as_FloatRegister();
    } else if (reg1->is_PRegister()) {
      return reg1->as_PRegister() == reg2->as_PRegister();
    }
  }
  return false;
}

int SaveLiveRegisters::decode_float_vector_register_size(OptoReg::Name opto_reg) {
  switch (opto_reg & 3) {
    case 0:
      return 1;
    case 1:
      return 2;
    case 2:
      return 4;
    case 3:
      return Matcher::scalable_vector_reg_size(T_FLOAT);
    default:
      ShouldNotReachHere();
      return 0;
  }
}

SaveLiveRegisters::SaveLiveRegisters(MacroAssembler* masm, BarrierStubC2* stub)
  : _masm(masm),
    _gp_regs(),
    _fp_regs(),
    _neon_regs(),
    _sve_regs(),
    _p_regs() {

  // Figure out what registers to save/restore
  initialize(stub);

  // Save registers
  __ push(_gp_regs, sp);
  __ push_fp(_fp_regs, sp, MacroAssembler::PushPopFp);
  __ push_fp(_neon_regs, sp, MacroAssembler::PushPopNeon);
  __ push_fp(_sve_regs, sp, MacroAssembler::PushPopSVE);
  __ push_p(_p_regs, sp);
}

SaveLiveRegisters::~SaveLiveRegisters() {
  // Restore registers
  __ pop_p(_p_regs, sp);
  __ pop_fp(_sve_regs, sp, MacroAssembler::PushPopSVE);
  __ pop_fp(_neon_regs, sp, MacroAssembler::PushPopNeon);
  __ pop_fp(_fp_regs, sp, MacroAssembler::PushPopFp);

  // External runtime call may clobber ptrue reg
  __ reinitialize_ptrue();

  __ pop(_gp_regs, sp);
}

#endif // COMPILER2