// // Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it // under the terms of the GNU General Public License version 2 only, as // published by the Free Software Foundation. // // This code is distributed in the hope that it will be useful, but WITHOUT // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License // version 2 for more details (a copy is included in the LICENSE file that // accompanied this code). // // You should have received a copy of the GNU General Public License version // 2 along with this work; if not, write to the Free Software Foundation, // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. // // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA // or visit www.oracle.com if you need additional information or have any // questions. // source_hpp %{ #include "gc/g1/c2/g1BarrierSetC2.hpp" #include "gc/shared/gc_globals.hpp" %} source %{ #include "gc/g1/g1BarrierSetAssembler_aarch64.hpp" #include "gc/g1/g1BarrierSetRuntime.hpp" static void write_barrier_pre(MacroAssembler* masm, const MachNode* node, Register obj, Register pre_val, Register tmp1, Register tmp2, RegSet preserve = RegSet(), RegSet no_preserve = RegSet()) { if (!G1PreBarrierStubC2::needs_barrier(node)) { return; } Assembler::InlineSkippedInstructionsCounter skip_counter(masm); G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node); for (RegSetIterator reg = preserve.begin(); *reg != noreg; ++reg) { stub->preserve(*reg); } for (RegSetIterator reg = no_preserve.begin(); *reg != noreg; ++reg) { stub->dont_preserve(*reg); } g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, rthread, tmp1, tmp2, stub); } static void write_barrier_post(MacroAssembler* masm, const MachNode* node, Register store_addr, Register new_val, Register tmp1, Register tmp2) { if (!G1PostBarrierStubC2::needs_barrier(node)) { return; } Assembler::InlineSkippedInstructionsCounter skip_counter(masm); G1BarrierSetAssembler* g1_asm = static_cast(BarrierSet::barrier_set()->barrier_set_assembler()); G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node); g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, stub); } %} // BEGIN This section of the file is automatically generated. Do not edit -------------- // This section is generated from g1_aarch64.m4 // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0); match(Set mem (StoreP mem src)); effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(INSN_COST); format %{ "str $src, $mem\t# ptr" %} ins_encode %{ write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $src$$Register) /* preserve */); __ str($src$$Register, $mem$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $src$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(istore_reg_mem); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1StorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0); match(Set mem (StoreP mem src)); effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "stlr $src, $mem\t# ptr" %} ins_encode %{ write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $src$$Register) /* preserve */); __ stlr($src$$Register, $mem$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $src$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_class_memory); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0); match(Set mem (StoreN mem src)); effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(INSN_COST); format %{ "strw $src, $mem\t# compressed ptr" %} ins_encode %{ write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $src$$Register) /* preserve */); __ strw($src$$Register, $mem$$Register); if ((barrier_data() & G1C2BarrierPost) != 0) { if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { __ decode_heap_oop($tmp1$$Register, $src$$Register); } else { __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register); } } write_barrier_post(masm, this, $mem$$Register /* store_addr */, $tmp1$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(istore_reg_mem); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1StoreNVolatile(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0); match(Set mem (StoreN mem src)); effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "stlrw $src, $mem\t# compressed ptr" %} ins_encode %{ write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $src$$Register) /* preserve */); __ stlrw($src$$Register, $mem$$Register); if ((barrier_data() & G1C2BarrierPost) != 0) { if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { __ decode_heap_oop($tmp1$$Register, $src$$Register); } else { __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register); } } write_barrier_post(masm, this, $mem$$Register /* store_addr */, $tmp1$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_class_memory); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0); match(Set mem (StoreN mem (EncodeP src))); effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(INSN_COST); format %{ "encode_heap_oop $tmp1, $src\n\t" "strw $tmp1, $mem\t# compressed ptr" %} ins_encode %{ write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $src$$Register) /* preserve */); if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { __ encode_heap_oop($tmp1$$Register, $src$$Register); } else { __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register); } __ strw($tmp1$$Register, $mem$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $src$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(istore_reg_mem); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1EncodePAndStoreNVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0); match(Set mem (StoreN mem (EncodeP src))); effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "encode_heap_oop $tmp1, $src\n\t" "stlrw $tmp1, $mem\t# compressed ptr" %} ins_encode %{ write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $src$$Register) /* preserve */); if ((barrier_data() & G1C2BarrierPostNotNull) == 0) { __ encode_heap_oop($tmp1$$Register, $src$$Register); } else { __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register); } __ stlrw($tmp1$$Register, $mem$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $src$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_class_memory); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set res (CompareAndExchangeP mem (Binary oldval newval))); effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr); ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %} ins_encode %{ assert_different_registers($oldval$$Register, $mem$$Register); assert_different_registers($newval$$Register, $mem$$Register); // Pass $oldval to the pre-barrier (instead of loading from $mem), because // $oldval is the only value that can be overwritten. // The same holds for g1CompareAndSwapP and its Acq variant. write_barrier_pre(masm, this, noreg /* obj */, $oldval$$Register /* pre_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */, RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, RegSet::of($res$$Register) /* no_preserve */); __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, false /* acquire */, true /* release */, false /* weak */, $res$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $newval$$Register /* new_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */); %} ins_pipe(pipe_slow); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set res (CompareAndExchangeP mem (Binary oldval newval))); effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %} ins_encode %{ assert_different_registers($oldval$$Register, $mem$$Register); assert_different_registers($newval$$Register, $mem$$Register); // Pass $oldval to the pre-barrier (instead of loading from $mem), because // $oldval is the only value that can be overwritten. // The same holds for g1CompareAndSwapP and its Acq variant. write_barrier_pre(masm, this, noreg /* obj */, $oldval$$Register /* pre_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */, RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, RegSet::of($res$$Register) /* no_preserve */); __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, true /* acquire */, true /* release */, false /* weak */, $res$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $newval$$Register /* new_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */); %} ins_pipe(pipe_slow); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %} ins_encode %{ assert_different_registers($oldval$$Register, $mem$$Register); assert_different_registers($newval$$Register, $mem$$Register); write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, RegSet::of($res$$Register) /* no_preserve */); __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word, false /* acquire */, true /* release */, false /* weak */, $res$$Register); __ decode_heap_oop($tmp1$$Register, $newval$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $tmp1$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_slow); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set res (CompareAndExchangeN mem (Binary oldval newval))); effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %} ins_encode %{ assert_different_registers($oldval$$Register, $mem$$Register); assert_different_registers($newval$$Register, $mem$$Register); write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, RegSet::of($res$$Register) /* no_preserve */); __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word, true /* acquire */, true /* release */, false /* weak */, $res$$Register); __ decode_heap_oop($tmp1$$Register, $newval$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $tmp1$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_slow); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr) %{ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set res (CompareAndSwapP mem (Binary oldval newval))); match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr); ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t" "cset $res, EQ" %} ins_encode %{ assert_different_registers($oldval$$Register, $mem$$Register); assert_different_registers($newval$$Register, $mem$$Register); write_barrier_pre(masm, this, noreg /* obj */, $oldval$$Register /* pre_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */, RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, RegSet::of($res$$Register) /* no_preserve */); __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, false /* acquire */, true /* release */, false /* weak */, noreg); __ cset($res$$Register, Assembler::EQ); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $newval$$Register /* new_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */); %} ins_pipe(pipe_slow); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr) %{ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set res (CompareAndSwapP mem (Binary oldval newval))); match(Set res (WeakCompareAndSwapP mem (Binary oldval newval))); effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t" "cset $res, EQ" %} ins_encode %{ assert_different_registers($oldval$$Register, $mem$$Register); assert_different_registers($newval$$Register, $mem$$Register); write_barrier_pre(masm, this, noreg /* obj */, $oldval$$Register /* pre_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */, RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, RegSet::of($res$$Register) /* no_preserve */); __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, true /* acquire */, true /* release */, false /* weak */, noreg); __ cset($res$$Register, Assembler::EQ); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $newval$$Register /* new_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */); %} ins_pipe(pipe_slow); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr) %{ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set res (CompareAndSwapN mem (Binary oldval newval))); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(2 * VOLATILE_REF_COST); format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t" "cset $res, EQ" %} ins_encode %{ assert_different_registers($oldval$$Register, $mem$$Register); assert_different_registers($newval$$Register, $mem$$Register); write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, RegSet::of($res$$Register) /* no_preserve */); __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word, false /* acquire */, true /* release */, false /* weak */, noreg); __ cset($res$$Register, Assembler::EQ); __ decode_heap_oop($tmp1$$Register, $newval$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $tmp1$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_slow); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr) %{ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set res (CompareAndSwapN mem (Binary oldval newval))); match(Set res (WeakCompareAndSwapN mem (Binary oldval newval))); effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t" "cset $res, EQ" %} ins_encode %{ assert_different_registers($oldval$$Register, $mem$$Register); assert_different_registers($newval$$Register, $mem$$Register); write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */, RegSet::of($res$$Register) /* no_preserve */); __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word, true /* acquire */, true /* release */, false /* weak */, noreg); __ cset($res$$Register, Assembler::EQ); __ decode_heap_oop($tmp1$$Register, $newval$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $tmp1$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_slow); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr) %{ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set preval (GetAndSetP mem newval)); effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr); ins_cost(2 * VOLATILE_REF_COST); format %{ "atomic_xchg $preval, $newval, [$mem]" %} ins_encode %{ assert_different_registers($mem$$Register, $newval$$Register); write_barrier_pre(masm, this, $mem$$Register /* obj */, $preval$$Register /* pre_val (as a temporary register) */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */, RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $newval$$Register /* new_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */); %} ins_pipe(pipe_serial); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr) %{ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set preval (GetAndSetP mem newval)); effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %} ins_encode %{ assert_different_registers($mem$$Register, $newval$$Register); write_barrier_pre(masm, this, $mem$$Register /* obj */, $preval$$Register /* pre_val (as a temporary register) */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */, RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $newval$$Register /* new_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */); %} ins_pipe(pipe_serial); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr) %{ predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set preval (GetAndSetN mem newval)); effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(2 * VOLATILE_REF_COST); format %{ "atomic_xchgw $preval, $newval, [$mem]" %} ins_encode %{ assert_different_registers($mem$$Register, $newval$$Register); write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register); __ decode_heap_oop($tmp1$$Register, $newval$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $tmp1$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_serial); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr) %{ predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0); match(Set preval (GetAndSetN mem newval)); effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(VOLATILE_REF_COST); format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %} ins_encode %{ assert_different_registers($mem$$Register, $newval$$Register); write_barrier_pre(masm, this, $mem$$Register /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */, RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */); __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register); __ decode_heap_oop($tmp1$$Register, $newval$$Register); write_barrier_post(masm, this, $mem$$Register /* store_addr */, $tmp1$$Register /* new_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); %} ins_pipe(pipe_serial); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr) %{ // This instruction does not need an acquiring counterpart because it is only // used for reference loading (Reference::get()). The same holds for g1LoadN. predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0); match(Set dst (LoadP mem)); effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr); ins_cost(4 * INSN_COST); format %{ "ldr $dst, $mem\t# ptr" %} ins_encode %{ __ ldr($dst$$Register, $mem$$Register); write_barrier_pre(masm, this, noreg /* obj */, $dst$$Register /* pre_val */, $tmp1$$Register /* tmp1 */, $tmp2$$Register /* tmp2 */); %} ins_pipe(iload_reg_mem); %} // This pattern is generated automatically from g1_aarch64.m4. // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr) %{ predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0); match(Set dst (LoadN mem)); effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr); ins_cost(4 * INSN_COST); format %{ "ldrw $dst, $mem\t# compressed ptr" %} ins_encode %{ __ ldrw($dst$$Register, $mem$$Register); if ((barrier_data() & G1C2BarrierPre) != 0) { __ decode_heap_oop($tmp1$$Register, $dst$$Register); write_barrier_pre(masm, this, noreg /* obj */, $tmp1$$Register /* pre_val */, $tmp2$$Register /* tmp1 */, $tmp3$$Register /* tmp2 */); } %} ins_pipe(iload_reg_mem); %} // END This section of the file is automatically generated. Do not edit --------------