/* * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2021, 2025 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. */ #include "asm/macroAssembler.inline.hpp" #include "asm/register.hpp" #include "code/codeBlob.hpp" #include "code/vmreg.inline.hpp" #include "gc/z/zAddress.hpp" #include "gc/z/zBarrier.inline.hpp" #include "gc/z/zBarrierSet.hpp" #include "gc/z/zBarrierSetAssembler.hpp" #include "gc/z/zBarrierSetRuntime.hpp" #include "gc/z/zThreadLocalData.hpp" #include "memory/resourceArea.hpp" #include "register_ppc.hpp" #include "runtime/jniHandles.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/macros.hpp" #ifdef COMPILER1 #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" #include "gc/z/c1/zBarrierSetC1.hpp" #endif // COMPILER1 #ifdef COMPILER2 #include "gc/z/c2/zBarrierSetC2.hpp" #include "opto/output.hpp" #endif // COMPILER2 #undef __ #define __ masm-> // Helper for saving and restoring registers across a runtime call that does // not have any live vector registers. class ZRuntimeCallSpill { MacroAssembler* _masm; Register _result; bool _needs_frame, _preserve_gp_registers, _preserve_fp_registers; int _nbytes_save; void save() { MacroAssembler* masm = _masm; if (_needs_frame) { if (_preserve_gp_registers) { bool preserve_R3 = _result != R3_ARG1; _nbytes_save = (MacroAssembler::num_volatile_gp_regs + (_preserve_fp_registers ? MacroAssembler::num_volatile_fp_regs : 0) - (preserve_R3 ? 0 : 1) ) * BytesPerWord; __ save_volatile_gprs(R1_SP, -_nbytes_save, _preserve_fp_registers, preserve_R3); } __ save_LR(R0); __ push_frame_reg_args(_nbytes_save, R0); } } void restore() { MacroAssembler* masm = _masm; Register result = R3_RET; if (_needs_frame) { __ pop_frame(); __ restore_LR(R0); if (_preserve_gp_registers) { bool restore_R3 = _result != R3_ARG1; if (restore_R3 && _result != noreg) { __ mr(R0, R3_RET); result = R0; } __ restore_volatile_gprs(R1_SP, -_nbytes_save, _preserve_fp_registers, restore_R3); } } if (_result != noreg) { __ mr_if_needed(_result, result); } } public: ZRuntimeCallSpill(MacroAssembler* masm, Register result, MacroAssembler::PreservationLevel preservation_level) : _masm(masm), _result(result), _needs_frame(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR), _preserve_gp_registers(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS), _preserve_fp_registers(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS), _nbytes_save(0) { save(); } ~ZRuntimeCallSpill() { restore(); } }; void ZBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register base, RegisterOrConstant ind_or_offs, Register dst, Register tmp1, Register tmp2, MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) { __ block_comment("load_at (zgc) {"); // Check whether a special gc barrier is required for this particular load // (e.g. whether it's a reference load or not) if (!ZBarrierSet::barrier_needed(decorators, type)) { BarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst, tmp1, tmp2, preservation_level, L_handle_null); return; } if (ind_or_offs.is_register()) { assert_different_registers(base, ind_or_offs.as_register(), tmp1, tmp2, R0, noreg); assert_different_registers(dst, ind_or_offs.as_register(), tmp1, tmp2, R0, noreg); } else { assert_different_registers(base, tmp1, tmp2, R0, noreg); assert_different_registers(dst, tmp1, tmp2, R0, noreg); } /* ==== Load the pointer using the standard implementation for the actual heap access and the decompression of compressed pointers ==== */ // Result of 'load_at' (standard implementation) will be written back to 'dst'. // As 'base' is required for the C-call, it must be reserved in case of a register clash. Register saved_base = base; if (base == dst) { __ mr(tmp2, base); saved_base = tmp2; } __ ld(dst, ind_or_offs, base); /* ==== Check whether pointer is dirty ==== */ Label done, uncolor; const bool on_non_strong = (decorators & ON_WEAK_OOP_REF) != 0 || (decorators & ON_PHANTOM_OOP_REF) != 0; // Load bad mask into scratch register. if (on_non_strong) { __ ld(tmp1, in_bytes(ZThreadLocalData::mark_bad_mask_offset()), R16_thread); } else { __ ld(tmp1, in_bytes(ZThreadLocalData::load_bad_mask_offset()), R16_thread); } // The color bits of the to-be-tested pointer do not have to be equivalent to the 'bad_mask' testing bits. // A pointer is classified as dirty if any of the color bits that also match the bad mask is set. // Conversely, it follows that the logical AND of the bad mask and the pointer must be zero // if the pointer is not dirty. // Only dirty pointers must be processed by this barrier, so we can skip it in case the latter condition holds true. __ and_(tmp1, tmp1, dst); __ beq(CR0, uncolor); /* ==== Invoke barrier ==== */ { ZRuntimeCallSpill rcs(masm, dst, preservation_level); // Setup arguments if (saved_base != R3_ARG1 && ind_or_offs.register_or_noreg() != R3_ARG1) { __ mr_if_needed(R3_ARG1, dst); __ add(R4_ARG2, ind_or_offs, saved_base); } else if (dst != R4_ARG2) { __ add(R4_ARG2, ind_or_offs, saved_base); __ mr(R3_ARG1, dst); } else { __ add(R0, ind_or_offs, saved_base); __ mr(R3_ARG1, dst); __ mr(R4_ARG2, R0); } __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators)); } // Slow-path has already uncolored if (L_handle_null != nullptr) { __ cmpdi(CR0, dst, 0); __ beq(CR0, *L_handle_null); } __ b(done); __ bind(uncolor); if (L_handle_null == nullptr) { __ srdi(dst, dst, ZPointerLoadShift); } else { __ srdi_(dst, dst, ZPointerLoadShift); __ beq(CR0, *L_handle_null); } __ bind(done); __ block_comment("} load_at (zgc)"); } static void load_least_significant_16_oop_bits(MacroAssembler* masm, Register dst, RegisterOrConstant ind_or_offs, Register base) { assert_different_registers(dst, base); #ifndef VM_LITTLE_ENDIAN const int BE_offset = 6; if (ind_or_offs.is_register()) { __ addi(dst, ind_or_offs.as_register(), BE_offset); __ lhzx(dst, base, dst); } else { __ lhz(dst, ind_or_offs.as_constant() + BE_offset, base); } #else __ lhz(dst, ind_or_offs, base); #endif } static void emit_store_fast_path_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, bool is_atomic, Label& medium_path) { if (is_atomic) { assert(ZPointerLoadShift + LogMinObjAlignmentInBytes >= 16, "or replace following code"); load_least_significant_16_oop_bits(masm, R0, ind_or_offs, base); // Atomic operations must ensure that the contents of memory are store-good before // an atomic operation can execute. // A not relocatable object could have spurious raw null pointers in its fields after // getting promoted to the old generation. __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits); __ cmplwi(CR0, R0, barrier_Relocation::unpatched); } else { __ ld(R0, ind_or_offs, base); // Stores on relocatable objects never need to deal with raw null pointers in fields. // Raw null pointers may only exist in the young generation, as they get pruned when // the object is relocated to old. And no pre-write barrier needs to perform any action // in the young generation. __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreBadMask); __ andi_(R0, R0, barrier_Relocation::unpatched); } __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), medium_path); } void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm, Register ref_base, RegisterOrConstant ind_or_offset, Register rnew_zaddress, Register rnew_zpointer, bool in_nmethod, bool is_atomic, Label& medium_path, Label& medium_path_continuation) const { assert_different_registers(ref_base, rnew_zpointer); assert_different_registers(ind_or_offset.register_or_noreg(), rnew_zpointer); assert_different_registers(rnew_zaddress, rnew_zpointer); if (in_nmethod) { emit_store_fast_path_check(masm, ref_base, ind_or_offset, is_atomic, medium_path); __ bind(medium_path_continuation); __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits); __ li(rnew_zpointer, barrier_Relocation::unpatched); // Load color bits. if (rnew_zaddress == noreg) { // noreg encodes null. if (ZPointerLoadShift >= 16) { __ rldicl(rnew_zpointer, rnew_zpointer, 0, 64 - ZPointerLoadShift); // Clear sign extension from li. } } } else { __ ld(R0, ind_or_offset, ref_base); __ ld(rnew_zpointer, in_bytes(ZThreadLocalData::store_bad_mask_offset()), R16_thread); __ and_(R0, R0, rnew_zpointer); __ bne(CR0, medium_path); __ bind(medium_path_continuation); __ ld(rnew_zpointer, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); } if (rnew_zaddress != noreg) { // noreg encodes null. __ rldimi(rnew_zpointer, rnew_zaddress, ZPointerLoadShift, 0); // Insert shifted pointer. } } static void store_barrier_buffer_add(MacroAssembler* masm, Register ref_base, RegisterOrConstant ind_or_offs, Register tmp1, Label& slow_path) { __ ld(tmp1, in_bytes(ZThreadLocalData::store_barrier_buffer_offset()), R16_thread); // Combined pointer bump and check if the buffer is disabled or full __ ld(R0, in_bytes(ZStoreBarrierBuffer::current_offset()), tmp1); __ addic_(R0, R0, -(int)sizeof(ZStoreBarrierEntry)); __ blt(CR0, slow_path); __ std(R0, in_bytes(ZStoreBarrierBuffer::current_offset()), tmp1); // Entry is at ZStoreBarrierBuffer (tmp1) + buffer_offset + scaled index (R0) __ add(tmp1, tmp1, R0); // Compute and log the store address Register store_addr = ref_base; if (!ind_or_offs.is_constant() || ind_or_offs.as_constant() != 0) { __ add(R0, ind_or_offs, ref_base); store_addr = R0; } __ std(store_addr, in_bytes(ZStoreBarrierBuffer::buffer_offset()) + in_bytes(ZStoreBarrierEntry::p_offset()), tmp1); // Load and log the prev value __ ld(R0, ind_or_offs, ref_base); __ std(R0, in_bytes(ZStoreBarrierBuffer::buffer_offset()) + in_bytes(ZStoreBarrierEntry::prev_offset()), tmp1); } void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm, Register ref_base, RegisterOrConstant ind_or_offs, Register tmp, bool is_atomic, Label& medium_path_continuation, Label& slow_path) const { assert_different_registers(ref_base, tmp, R0); // The reason to end up in the medium path is that the pre-value was not 'good'. if (is_atomic) { // Atomic accesses can get to the medium fast path because the value was a // raw null value. If it was not null, then there is no doubt we need to take a slow path. __ ld(tmp, ind_or_offs, ref_base); __ cmpdi(CR0, tmp, 0); __ bne(CR0, slow_path); // If we get this far, we know there is a young raw null value in the field. // Try to self-heal null values for atomic accesses bool need_restore = false; if (!ind_or_offs.is_constant() || ind_or_offs.as_constant() != 0) { __ add(ref_base, ref_base, ind_or_offs); need_restore = true; } __ ld(R0, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); __ cmpxchgd(CR0, tmp, (intptr_t)0, R0, ref_base, MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, need_restore ? nullptr : &slow_path); if (need_restore) { __ sub(ref_base, ref_base, ind_or_offs); __ bne(CR0, slow_path); } } else { // A non-atomic relocatable object won't get to the medium fast path due to a // raw null in the young generation. We only get here because the field is bad. // In this path we don't need any self healing, so we can avoid a runtime call // most of the time by buffering the store barrier to be applied lazily. store_barrier_buffer_add(masm, ref_base, ind_or_offs, tmp, slow_path); } __ b(medium_path_continuation); } // The Z store barrier only verifies the pointers it is operating on and is thus a sole debugging measure. void ZBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register base, RegisterOrConstant ind_or_offs, Register val, Register tmp1, Register tmp2, Register tmp3, MacroAssembler::PreservationLevel preservation_level) { __ block_comment("store_at (zgc) {"); bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if (is_reference_type(type)) { assert_different_registers(base, val, tmp1, tmp2, tmp3); if (dest_uninitialized) { // tmp1 = (val << ZPointerLoadShift) | store_good_mask __ ld(tmp1, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); if (val != noreg) { // noreg encodes null. __ rldimi(tmp1, val, ZPointerLoadShift, 0); } } else { Label done; Label medium; Label medium_continuation; // bound in store_barrier_fast Label slow; store_barrier_fast(masm, base, ind_or_offs, val, tmp1, false, false, medium, medium_continuation); __ b(done); __ bind(medium); store_barrier_medium(masm, base, ind_or_offs, tmp1, false, medium_continuation, slow); __ bind(slow); { ZRuntimeCallSpill rcs(masm, noreg, preservation_level); __ add(R3_ARG1, ind_or_offs, base); __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), R3_ARG1); } __ b(medium_continuation); __ bind(done); } BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, tmp1, tmp2, tmp3, noreg, preservation_level); } else { BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, val, tmp1, tmp2, tmp3, preservation_level); } __ block_comment("} store_at (zgc)"); } /* arraycopy */ const Register _load_bad_mask = R6, _store_bad_mask = R7, _store_good_mask = R8; void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register preserve1, Register preserve2) { bool is_checkcast_copy = (decorators & ARRAYCOPY_CHECKCAST) != 0, dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if (!ZBarrierSet::barrier_needed(decorators, type) || is_checkcast_copy) { // Barrier not needed return; } __ block_comment("arraycopy_prologue (zgc) {"); load_copy_masks(masm, _load_bad_mask, _store_bad_mask, _store_good_mask, dest_uninitialized); __ block_comment("} arraycopy_prologue (zgc)"); } void ZBarrierSetAssembler::load_copy_masks(MacroAssembler* masm, Register load_bad_mask, Register store_bad_mask, Register store_good_mask, bool dest_uninitialized) const { __ ld(load_bad_mask, in_bytes(ZThreadLocalData::load_bad_mask_offset()), R16_thread); __ ld(store_good_mask, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); if (dest_uninitialized) { DEBUG_ONLY( __ li(store_bad_mask, -1); ) } else { __ ld(store_bad_mask, in_bytes(ZThreadLocalData::store_bad_mask_offset()), R16_thread); } } void ZBarrierSetAssembler::copy_load_at_fast(MacroAssembler* masm, Register zpointer, Register addr, Register load_bad_mask, Label& slow_path, Label& continuation) const { __ ldx(zpointer, addr); __ and_(R0, zpointer, load_bad_mask); __ bne(CR0, slow_path); __ bind(continuation); } void ZBarrierSetAssembler::copy_load_at_slow(MacroAssembler* masm, Register zpointer, Register addr, Register tmp, Label& slow_path, Label& continuation) const { __ align(32); __ bind(slow_path); __ mfctr(tmp); // preserve loop counter { ZRuntimeCallSpill rcs(masm, R0, MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS); assert(zpointer != R4_ARG2, "or change argument setup"); __ mr_if_needed(R4_ARG2, addr); __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(), zpointer, R4_ARG2); } __ sldi(zpointer, R0, ZPointerLoadShift); // Slow-path has uncolored; revert __ mtctr(tmp); // restore loop counter __ b(continuation); } void ZBarrierSetAssembler::copy_store_at_fast(MacroAssembler* masm, Register zpointer, Register addr, Register store_bad_mask, Register store_good_mask, Label& medium_path, Label& continuation, bool dest_uninitialized) const { if (!dest_uninitialized) { __ ldx(R0, addr); __ and_(R0, R0, store_bad_mask); __ bne(CR0, medium_path); __ bind(continuation); } __ rldimi(zpointer, store_good_mask, 0, 64 - ZPointerLoadShift); // Replace color bits. __ stdx(zpointer, addr); } void ZBarrierSetAssembler::copy_store_at_slow(MacroAssembler* masm, Register addr, Register tmp, Label& medium_path, Label& continuation, bool dest_uninitialized) const { if (!dest_uninitialized) { Label slow_path; __ align(32); __ bind(medium_path); store_barrier_medium(masm, addr, (intptr_t)0, tmp, false, continuation, slow_path); __ bind(slow_path); __ mfctr(tmp); // preserve loop counter { ZRuntimeCallSpill rcs(masm, noreg, MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS); __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), addr); } __ mtctr(tmp); // restore loop counter __ b(continuation); } } // Arguments for generated stub: // from: R3_ARG1 // to: R4_ARG2 // count: R5_ARG3 (int >= 0) void ZBarrierSetAssembler::generate_disjoint_oop_copy(MacroAssembler* masm, bool dest_uninitialized) { const Register zpointer = R2, tmp = R9; Label done, loop, load_bad, load_good, store_bad, store_good; __ cmpdi(CR0, R5_ARG3, 0); __ beq(CR0, done); __ mtctr(R5_ARG3); __ align(32); __ bind(loop); copy_load_at_fast(masm, zpointer, R3_ARG1, _load_bad_mask, load_bad, load_good); copy_store_at_fast(masm, zpointer, R4_ARG2, _store_bad_mask, _store_good_mask, store_bad, store_good, dest_uninitialized); __ addi(R3_ARG1, R3_ARG1, 8); __ addi(R4_ARG2, R4_ARG2, 8); __ bdnz(loop); __ bind(done); __ li(R3_RET, 0); __ blr(); copy_load_at_slow(masm, zpointer, R3_ARG1, tmp, load_bad, load_good); copy_store_at_slow(masm, R4_ARG2, tmp, store_bad, store_good, dest_uninitialized); } void ZBarrierSetAssembler::generate_conjoint_oop_copy(MacroAssembler* masm, bool dest_uninitialized) { const Register zpointer = R2, tmp = R9; Label done, loop, load_bad, load_good, store_bad, store_good; __ sldi_(R0, R5_ARG3, 3); __ beq(CR0, done); __ mtctr(R5_ARG3); // Point behind last elements and copy backwards. __ add(R3_ARG1, R3_ARG1, R0); __ add(R4_ARG2, R4_ARG2, R0); __ align(32); __ bind(loop); __ addi(R3_ARG1, R3_ARG1, -8); __ addi(R4_ARG2, R4_ARG2, -8); copy_load_at_fast(masm, zpointer, R3_ARG1, _load_bad_mask, load_bad, load_good); copy_store_at_fast(masm, zpointer, R4_ARG2, _store_bad_mask, _store_good_mask, store_bad, store_good, dest_uninitialized); __ bdnz(loop); __ bind(done); __ li(R3_RET, 0); __ blr(); copy_load_at_slow(masm, zpointer, R3_ARG1, tmp, load_bad, load_good); copy_store_at_slow(masm, R4_ARG2, tmp, store_bad, store_good, dest_uninitialized); } // Verify a colored pointer. void ZBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const char* msg) { if (!VerifyOops) { return; } Label done, skip_uncolor; // Skip (colored) null. __ srdi_(R0, obj, ZPointerLoadShift); __ beq(CR0, done); // Check if ZAddressHeapBase << ZPointerLoadShift is set. If so, we need to uncolor. __ rldicl_(R0, obj, 64 - ZAddressHeapBaseShift - ZPointerLoadShift, 63); __ mr(R0, obj); __ beq(CR0, skip_uncolor); __ srdi(R0, obj, ZPointerLoadShift); __ bind(skip_uncolor); __ verify_oop(R0, msg); __ bind(done); } void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env, Register obj, Register tmp, Label& slowpath) { __ block_comment("try_resolve_jobject_in_native (zgc) {"); Label done, tagged, weak_tagged, check_color; Address load_bad_mask = load_bad_mask_from_jni_env(jni_env), mark_bad_mask = mark_bad_mask_from_jni_env(jni_env); // Test for tag __ andi_(tmp, obj, JNIHandles::tag_mask); __ bne(CR0, tagged); // Resolve local handle __ ld(dst, 0, obj); __ b(done); __ bind(tagged); // Test for weak tag __ andi_(tmp, obj, JNIHandles::TypeTag::weak_global); __ clrrdi(dst, obj, JNIHandles::tag_size); // Untag. __ bne(CR0, weak_tagged); // Resolve global handle __ ld(dst, 0, dst); __ ld(tmp, load_bad_mask); __ b(check_color); __ bind(weak_tagged); // Resolve weak handle __ ld(dst, 0, dst); __ ld(tmp, mark_bad_mask); __ bind(check_color); __ and_(tmp, tmp, dst); __ bne(CR0, slowpath); // Uncolor __ srdi(dst, dst, ZPointerLoadShift); __ bind(done); __ block_comment("} try_resolve_jobject_in_native (zgc)"); } #undef __ #ifdef COMPILER1 #define __ ce->masm()-> static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) { Register r = ref->as_register(); __ srdi(r, r, ZPointerLoadShift); } static void check_color(LIR_Assembler* ce, LIR_Opr ref, bool on_non_strong) { int relocFormat = on_non_strong ? ZBarrierRelocationFormatMarkBadMask : ZBarrierRelocationFormatLoadBadMask; __ relocate(barrier_Relocation::spec(), relocFormat); __ andi_(R0, ref->as_register(), barrier_Relocation::unpatched); } static void z_color(LIR_Assembler* ce, LIR_Opr ref) { __ sldi(ref->as_register(), ref->as_register(), ZPointerLoadShift); __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits); __ ori(ref->as_register(), ref->as_register(), barrier_Relocation::unpatched); } void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const { z_uncolor(ce, ref); } void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const { z_color(ce, ref); } void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce, LIR_Opr ref, ZLoadBarrierStubC1* stub, bool on_non_strong) const { check_color(ce, ref, on_non_strong); __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *stub->entry()); z_uncolor(ce, ref); __ bind(*stub->continuation()); } // Code emitted by code stub "ZLoadBarrierStubC1" which in turn is emitted by ZBarrierSetC1::load_barrier. // Invokes the runtime stub which is defined just below. void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce, ZLoadBarrierStubC1* stub) const { __ block_comment("c1_load_barrier_stub (zgc) {"); __ bind(*stub->entry()); /* ==== Determine relevant data registers and ensure register sanity ==== */ Register ref = stub->ref()->as_register(); Register ref_addr = noreg; // Determine reference address if (stub->tmp()->is_valid()) { // 'tmp' register is given, so address might have an index or a displacement. ce->leal(stub->ref_addr(), stub->tmp()); ref_addr = stub->tmp()->as_pointer_register(); } else { // 'tmp' register is not given, so address must have neither an index nor a displacement. // The address' base register is thus usable as-is. assert(stub->ref_addr()->as_address_ptr()->disp() == 0, "illegal displacement"); assert(!stub->ref_addr()->as_address_ptr()->index()->is_valid(), "illegal index"); ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register(); } assert_different_registers(ref, ref_addr, R0, noreg); /* ==== Invoke stub ==== */ // Pass arguments via stack. The stack pointer will be bumped by the stub. __ std(ref, -1 * BytesPerWord, R1_SP); __ std(ref_addr, -2 * BytesPerWord, R1_SP); __ load_const_optimized(R0, stub->runtime_stub(), /* temp */ ref); __ call_stub(R0); // The runtime stub passes the result via the R0 register, overriding the previously-loaded stub address. __ mr(ref, R0); __ b(*stub->continuation()); __ block_comment("} c1_load_barrier_stub (zgc)"); } void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce, LIR_Address* addr, LIR_Opr new_zaddress, LIR_Opr new_zpointer, ZStoreBarrierStubC1* stub) const { Register rnew_zaddress = new_zaddress->as_register(); Register rnew_zpointer = new_zpointer->as_register(); Register rbase = addr->base()->as_pointer_register(); RegisterOrConstant ind_or_offs = (addr->index()->is_illegal()) ? (RegisterOrConstant)addr->disp() : (RegisterOrConstant)addr->index()->as_pointer_register(); store_barrier_fast(ce->masm(), rbase, ind_or_offs, rnew_zaddress, rnew_zpointer, true, stub->is_atomic(), *stub->entry(), *stub->continuation()); } void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce, ZStoreBarrierStubC1* stub) const { // Stub entry __ bind(*stub->entry()); Label slow; LIR_Address* addr = stub->ref_addr()->as_address_ptr(); assert(addr->index()->is_illegal() || addr->disp() == 0, "can't have both"); Register rbase = addr->base()->as_pointer_register(); RegisterOrConstant ind_or_offs = (addr->index()->is_illegal()) ? (RegisterOrConstant)addr->disp() : (RegisterOrConstant)addr->index()->as_pointer_register(); Register new_zpointer = stub->new_zpointer()->as_register(); store_barrier_medium(ce->masm(), rbase, ind_or_offs, new_zpointer, // temp stub->is_atomic(), *stub->continuation(), slow); __ bind(slow); __ load_const_optimized(/*stub address*/ new_zpointer, stub->runtime_stub(), R0); __ add(R0, ind_or_offs, rbase); // pass store address in R0 __ mtctr(new_zpointer); __ bctrl(); // Stub exit __ b(*stub->continuation()); } #undef __ #define __ sasm-> // Code emitted by runtime code stub which in turn is emitted by ZBarrierSetC1::generate_c1_runtime_stubs. void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) const { __ block_comment("c1_load_barrier_runtime_stub (zgc) {"); const int stack_parameters = 2; const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_parameters) * BytesPerWord; __ save_volatile_gprs(R1_SP, -nbytes_save); __ save_LR(R0); // Load arguments back again from the stack. __ ld(R3_ARG1, -1 * BytesPerWord, R1_SP); // ref __ ld(R4_ARG2, -2 * BytesPerWord, R1_SP); // ref_addr __ push_frame_reg_args(nbytes_save, R0); __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators)); __ verify_oop(R3_RET, "Bad pointer after barrier invocation"); __ mr(R0, R3_RET); __ pop_frame(); __ restore_LR(R3_RET); __ restore_volatile_gprs(R1_SP, -nbytes_save); __ blr(); __ block_comment("} c1_load_barrier_runtime_stub (zgc)"); } void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm, bool self_healing) const { __ block_comment("c1_store_barrier_runtime_stub (zgc) {"); const int nbytes_save = MacroAssembler::num_volatile_regs * BytesPerWord; __ save_volatile_gprs(R1_SP, -nbytes_save); __ mr(R3_ARG1, R0); // store address __ save_LR(R0); __ push_frame_reg_args(nbytes_save, R0); if (self_healing) { __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()); } else { __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()); } __ pop_frame(); __ restore_LR(R3_RET); __ restore_volatile_gprs(R1_SP, -nbytes_save); __ blr(); __ block_comment("} c1_store_barrier_runtime_stub (zgc)"); } #undef __ #endif // COMPILER1 #ifdef COMPILER2 #undef __ #define __ _masm-> class ZSetupArguments { MacroAssembler* const _masm; const Register _ref; const Address _ref_addr; public: ZSetupArguments(MacroAssembler* masm, ZLoadBarrierStubC2* stub) : _masm(masm), _ref(stub->ref()), _ref_addr(stub->ref_addr()) { // Desired register/argument configuration: // _ref: R3_ARG1 // _ref_addr: R4_ARG2 // '_ref_addr' can be unspecified. In that case, the barrier will not heal the reference. if (_ref_addr.base() == noreg) { assert_different_registers(_ref, R0, noreg); __ mr_if_needed(R3_ARG1, _ref); __ li(R4_ARG2, 0); } else { assert_different_registers(_ref, _ref_addr.base(), R0, noreg); assert(!_ref_addr.index()->is_valid(), "reference addresses must not contain an index component"); if (_ref != R4_ARG2) { // Calculate address first as the address' base register might clash with R4_ARG2 __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp()); __ mr_if_needed(R3_ARG1, _ref); } else if (_ref_addr.base() != R3_ARG1) { __ mr(R3_ARG1, _ref); __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp()); // Clobbering _ref } else { // Arguments are provided in inverse order (i.e. _ref == R4_ARG2, _ref_addr == R3_ARG1) __ mr(R0, _ref); __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp()); __ mr(R3_ARG1, R0); } } } }; #undef __ #define __ masm-> void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const { Assembler::InlineSkippedInstructionsCounter skipped_counter(masm); __ block_comment("generate_c2_load_barrier_stub (zgc) {"); __ bind(*stub->entry()); Register ref = stub->ref(); Address ref_addr = stub->ref_addr(); assert_different_registers(ref, ref_addr.base()); { SaveLiveRegisters save_live_registers(masm, stub); ZSetupArguments setup_arguments(masm, stub); __ call_VM_leaf(stub->slow_path()); __ mr_if_needed(ref, R3_RET); } __ b(*stub->continuation()); __ block_comment("} generate_c2_load_barrier_stub (zgc)"); } void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const { Assembler::InlineSkippedInstructionsCounter skipped_counter(masm); __ block_comment("ZStoreBarrierStubC2"); // Stub entry __ bind(*stub->entry()); Label slow; Address addr = stub->ref_addr(); Register rbase = addr.base(); RegisterOrConstant ind_or_offs = (addr.index() == noreg) ? (RegisterOrConstant)addr.disp() : (RegisterOrConstant)addr.index(); if (!stub->is_native()) { store_barrier_medium(masm, rbase, ind_or_offs, stub->new_zpointer(), stub->is_atomic(), *stub->continuation(), slow); } __ bind(slow); { SaveLiveRegisters save_live_registers(masm, stub); __ add(R3_ARG1, ind_or_offs, rbase); if (stub->is_native()) { __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr(), R3_ARG1); } else if (stub->is_atomic()) { __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), R3_ARG1); } else if (stub->is_nokeepalive()) { __ call_VM_leaf(ZBarrierSetRuntime::no_keepalive_store_barrier_on_oop_field_without_healing_addr(), R3_ARG1); } else { __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), R3_ARG1); } } // Stub exit __ b(*stub->continuation()); } #undef __ #endif // COMPILER2 static uint16_t patch_barrier_relocation_value(int format) { switch (format) { case ZBarrierRelocationFormatLoadBadMask: return (uint16_t)ZPointerLoadBadMask; case ZBarrierRelocationFormatMarkBadMask: return (uint16_t)ZPointerMarkBadMask; case ZBarrierRelocationFormatStoreGoodBits: return (uint16_t)ZPointerStoreGoodMask; case ZBarrierRelocationFormatStoreBadMask: return (uint16_t)ZPointerStoreBadMask; default: ShouldNotReachHere(); return 0; } } void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) { #ifdef ASSERT int inst = *(int*)addr; if (format == ZBarrierRelocationFormatStoreGoodBits) { assert(Assembler::is_li(inst) || Assembler::is_ori(inst) || Assembler::is_cmpli(inst), "unexpected instruction 0x%04x", inst); // Note: li uses sign extend, but these bits will get cleared by rldimi. } else { assert(Assembler::is_andi(inst), "unexpected instruction 0x%04x", inst); } #endif // Patch the signed/unsigned 16 bit immediate field of the instruction. *(uint16_t*)(addr BIG_ENDIAN_ONLY(+2)) = patch_barrier_relocation_value(format); ICache::ppc64_flush_icache_bytes(addr, BytesPerInstWord); }