/* * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2017, 2024 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. * */ #include "asm/assembler.hpp" #include "asm/assembler.inline.hpp" #include "opto/c2_MacroAssembler.hpp" #include "opto/intrinsicnode.hpp" #include "runtime/stubRoutines.hpp" #define BLOCK_COMMENT(str) block_comment(str) #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2) { compiler_fast_lock_lightweight_object(obj, box, temp1, temp2); } void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2) { compiler_fast_unlock_lightweight_object(obj, box, temp1, temp2); } void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) { // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract // obj-start, so that we can load from the object's mark-word instead. z_lg(dst, src.plus_disp(-oopDesc::klass_offset_in_bytes())); z_srlg(dst, dst, markWord::klass_shift); // TODO: could be z_sra } //------------------------------------------------------ // Special String Intrinsics. Implementation //------------------------------------------------------ // Intrinsics for CompactStrings // Compress char[] to byte[]. // Restores: src, dst // Uses: cnt // Kills: tmp, Z_R0, Z_R1. // Early clobber: result. // Note: // cnt is signed int. Do not rely on high word! // counts # characters, not bytes. // // The result indicates success or failure of the operation. // General compress operation (cut off high order byte which must be all zeroes). // = len - all characters have been successfully compressed. // = 0 - compress failed. At least one character was found with a non-zero high order byte. // This is the failure return value which exactly corresponds to the Java implementation. // 0 <= result < len - compress failed. That many characters were compressed successfully // before the first non-compressable character was found. This is the // current, but not fully compatible, implementation. See below. // Encode to ISO or 7-bit ASCII array. // = len - all characters have been encoded successfully. // < len - encode failed. That many characters were encoded successfully. // When used as an index into the character array, the return value addresses the // first not encodeable character. // // If precise is true, the processing stops exactly at the point where a failure is detected. // More characters than indicated by the return value may have been read from the src array. // Exactly the number of characters indicated by the return value have been written to dst. // If precise is false, a few characters more than indicated by the return value may have been // written to the dst array. In any failure case, The result value indexes the first invalid character. unsigned int C2_MacroAssembler::string_compress(Register result, Register Rsrc, Register Rdst, Register Rcnt, Register tmp, bool precise, bool toASCII, VectorRegister Vtmp1, VectorRegister Vtmp2, VectorRegister Vmask, VectorRegister Vzero, VectorRegister Vsrc_first, VectorRegister v21, VectorRegister v22, VectorRegister Vsrc_last) { assert_different_registers(Z_R0, Z_R1, result, Rsrc, Rdst, Rcnt, tmp); unsigned short char_mask = 0xff00; // all selected bits must be '0' for a char to be valid unsigned int mask_ix_l = 0; // leftmost one bit pos in mask unsigned int mask_ix_r = 7; // rightmost one bit pos in mask if (precise) { if (toASCII) { BLOCK_COMMENT("encode_ascii_array {"); char_mask = 0xff80; mask_ix_r = 8; // rightmost one bit pos in mask. ASCII only uses codes 0..127 } else { BLOCK_COMMENT("encode_iso_array {"); } } else { BLOCK_COMMENT("string_compress {"); assert(!toASCII, "Can't compress strings to 7-bit ASCII"); } int block_start = offset(); Register Rix = tmp; Register Rmask = result; // holds incompatibility check mask until result value is stored. Label ScalarShortcut, AllDone; z_iilf(Rmask, (unsigned int)char_mask<<16 | (unsigned int)char_mask); z_iihf(Rmask, (unsigned int)char_mask<<16 | (unsigned int)char_mask); #if 0 // Sacrifice shortcuts for code compactness { //---< shortcuts for short strings (very frequent) >--- // Strings with 4 and 8 characters were found to occur very frequently. // Therefore, we handle them right away with minimal overhead. Label skipShortcut, skip4Shortcut, skip8Shortcut; Register Rout = Z_R0; z_chi(Rcnt, 4); z_brne(skip4Shortcut); // 4 characters are very frequent z_lg(Z_R0, 0, Rsrc); // Treat exactly 4 characters specially. if (VM_Version::has_DistinctOpnds()) { Rout = Z_R0; z_ngrk(Rix, Z_R0, Rmask); } else { Rout = Rix; z_lgr(Rix, Z_R0); z_ngr(Z_R0, Rmask); } z_brnz(skipShortcut); z_stcmh(Rout, 5, 0, Rdst); z_stcm(Rout, 5, 2, Rdst); z_lgfr(result, Rcnt); z_bru(AllDone); bind(skip4Shortcut); z_chi(Rcnt, 8); z_brne(skip8Shortcut); // There's more to do... z_lmg(Z_R0, Z_R1, 0, Rsrc); // Treat exactly 8 characters specially. if (VM_Version::has_DistinctOpnds()) { Rout = Z_R0; z_ogrk(Rix, Z_R0, Z_R1); z_ngr(Rix, Rmask); } else { Rout = Rix; z_lgr(Rix, Z_R0); z_ogr(Z_R0, Z_R1); z_ngr(Z_R0, Rmask); } z_brnz(skipShortcut); z_stcmh(Rout, 5, 0, Rdst); z_stcm(Rout, 5, 2, Rdst); z_stcmh(Z_R1, 5, 4, Rdst); z_stcm(Z_R1, 5, 6, Rdst); z_lgfr(result, Rcnt); z_bru(AllDone); bind(skip8Shortcut); clear_reg(Z_R0, true, false); // #characters already processed (none). Precond for scalar loop. z_brl(ScalarShortcut); // Just a few characters bind(skipShortcut); } #endif clear_reg(Z_R0); // make sure register is properly initialized. if (VM_Version::has_VectorFacility()) { const int min_vcnt = 32; // Minimum #characters required to use vector instructions. // Otherwise just do nothing in vector mode. // Must correspond to # vector registers used by implementation, // and must be a power of 2. const int log_min_vcnt = exact_log2(min_vcnt); Label VectorLoop, VectorDone, VectorBreak; assert((Vsrc_last->encoding() - Vsrc_first->encoding() + 1) == min_vcnt/8, "logic error"); assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()"); z_srak(Rix, Rcnt, log_min_vcnt); // # vector loop iterations z_brz(VectorDone); // not enough data for vector loop z_vzero(Vzero); // all zeroes z_vgmh(Vmask, mask_ix_l, mask_ix_r); // generate 0xff00/0xff80 mask for all 2-byte elements z_sllg(Z_R0, Rix, log_min_vcnt); // remember #chars that will be processed by vector loop bind(VectorLoop); z_vlm(Vsrc_first, Vsrc_last, 0, Rsrc); add2reg(Rsrc, min_vcnt*2); //---< check for incompatible character >--- z_vo(Vtmp1, Vsrc_first, v21); z_vo(Vtmp2, v22, Vsrc_last); z_vo(Vtmp1, Vtmp1, Vtmp2); z_vn(Vtmp1, Vtmp1, Vmask); z_vceqhs(Vtmp1, Vtmp1, Vzero); // all bits selected by mask must be zero for successful compress. z_bvnt(VectorBreak); // break vector loop if not all vector elements compare eq -> incompatible character found. // re-process data from current iteration in break handler. //---< pack & store characters >--- z_vpkh(Vtmp1, Vsrc_first, v21); // pack (src1, src2) -> tmp1 z_vpkh(Vtmp2, v22, Vsrc_last); // pack (src3, src4) -> tmp2 z_vstm(Vtmp1, Vtmp2, 0, Rdst); // store packed string add2reg(Rdst, min_vcnt); z_brct(Rix, VectorLoop); z_bru(VectorDone); bind(VectorBreak); add2reg(Rsrc, -min_vcnt*2); // Fix Rsrc. Rsrc was already updated, but Rdst and Rix are not. z_sll(Rix, log_min_vcnt); // # chars processed so far in VectorLoop, excl. current iteration. z_sr(Z_R0, Rix); // correct # chars processed in total. bind(VectorDone); } { const int min_cnt = 8; // Minimum #characters required to use unrolled loop. // Otherwise just do nothing in unrolled loop. // Must correspond to # registers used by implementation, // and must be a power of 2. const int log_min_cnt = exact_log2(min_cnt); Label UnrolledLoop, UnrolledDone, UnrolledBreak; if (VM_Version::has_DistinctOpnds()) { z_srk(Rix, Rcnt, Z_R0); // remaining # chars to compress in unrolled loop } else { z_lr(Rix, Rcnt); z_sr(Rix, Z_R0); } z_sra(Rix, log_min_cnt); // unrolled loop count z_brz(UnrolledDone); bind(UnrolledLoop); z_lmg(Z_R0, Z_R1, 0, Rsrc); if (precise) { z_ogr(Z_R1, Z_R0); // check all 8 chars for incompatibility z_ngr(Z_R1, Rmask); z_brnz(UnrolledBreak); z_lg(Z_R1, 8, Rsrc); // reload destroyed register z_stcmh(Z_R0, 5, 0, Rdst); z_stcm(Z_R0, 5, 2, Rdst); } else { z_stcmh(Z_R0, 5, 0, Rdst); z_stcm(Z_R0, 5, 2, Rdst); z_ogr(Z_R0, Z_R1); z_ngr(Z_R0, Rmask); z_brnz(UnrolledBreak); } z_stcmh(Z_R1, 5, 4, Rdst); z_stcm(Z_R1, 5, 6, Rdst); add2reg(Rsrc, min_cnt*2); add2reg(Rdst, min_cnt); z_brct(Rix, UnrolledLoop); z_lgfr(Z_R0, Rcnt); // # chars processed in total after unrolled loop. z_nilf(Z_R0, ~(min_cnt-1)); z_tmll(Rcnt, min_cnt-1); z_brnaz(ScalarShortcut); // if all bits zero, there is nothing left to do for scalar loop. // Rix == 0 in all cases. z_sllg(Z_R1, Rcnt, 1); // # src bytes already processed. Only lower 32 bits are valid! // Z_R1 contents must be treated as unsigned operand! For huge strings, // (Rcnt >= 2**30), the value may spill into the sign bit by sllg. z_lgfr(result, Rcnt); // all characters processed. z_slgfr(Rdst, Rcnt); // restore ptr z_slgfr(Rsrc, Z_R1); // restore ptr, double the element count for Rsrc restore z_bru(AllDone); bind(UnrolledBreak); z_lgfr(Z_R0, Rcnt); // # chars processed in total after unrolled loop z_nilf(Z_R0, ~(min_cnt-1)); z_sll(Rix, log_min_cnt); // # chars not yet processed in UnrolledLoop (due to break), broken iteration not included. z_sr(Z_R0, Rix); // fix # chars processed OK so far. if (!precise) { // Because we don't need to be precise, we just return the # of characters which have been written. // The first illegal character is in the index range [result-min_cnt/2, result+min_cnt/2). z_lgfr(result, Z_R0); z_sllg(Z_R1, Z_R0, 1); // # src bytes already processed. Only lower 32 bits are valid! // Z_R1 contents must be treated as unsigned operand! For huge strings, // (Rcnt >= 2**30), the value may spill into the sign bit by sllg. z_aghi(result, min_cnt/2); // min_cnt/2 characters have already been written // but ptrs were not updated yet. z_slgfr(Rdst, Z_R0); // restore ptr z_slgfr(Rsrc, Z_R1); // restore ptr, double the element count for Rsrc restore z_bru(AllDone); } bind(UnrolledDone); } { Label ScalarLoop, ScalarDone, ScalarBreak; bind(ScalarShortcut); z_ltgfr(result, Rcnt); z_brz(AllDone); #if 0 // Sacrifice shortcuts for code compactness { //---< Special treatment for very short strings (one or two characters) >--- // For these strings, we are sure that the above code was skipped. // Thus, no registers were modified, register restore is not required. Label ScalarDoit, Scalar2Char; z_chi(Rcnt, 2); z_brh(ScalarDoit); z_llh(Z_R1, 0, Z_R0, Rsrc); z_bre(Scalar2Char); z_tmll(Z_R1, char_mask); z_lghi(result, 0); // cnt == 1, first char invalid, no chars successfully processed z_brnaz(AllDone); z_stc(Z_R1, 0, Z_R0, Rdst); z_lghi(result, 1); z_bru(AllDone); bind(Scalar2Char); z_llh(Z_R0, 2, Z_R0, Rsrc); z_tmll(Z_R1, char_mask); z_lghi(result, 0); // cnt == 2, first char invalid, no chars successfully processed z_brnaz(AllDone); z_stc(Z_R1, 0, Z_R0, Rdst); z_tmll(Z_R0, char_mask); z_lghi(result, 1); // cnt == 2, second char invalid, one char successfully processed z_brnaz(AllDone); z_stc(Z_R0, 1, Z_R0, Rdst); z_lghi(result, 2); z_bru(AllDone); bind(ScalarDoit); } #endif if (VM_Version::has_DistinctOpnds()) { z_srk(Rix, Rcnt, Z_R0); // remaining # chars to compress in scalar loop } else { z_lr(Rix, Rcnt); z_sr(Rix, Z_R0); } z_lgfr(result, Rcnt); // # processed characters (if all encodes ok). z_brz(ScalarDone); // anything left to do? (uses CC from Rix calculation) bind(ScalarLoop); z_llh(Z_R1, 0, Z_R0, Rsrc); z_tmll(Z_R1, char_mask); z_brnaz(ScalarBreak); z_stc(Z_R1, 0, Z_R0, Rdst); add2reg(Rsrc, 2); add2reg(Rdst, 1); z_brct(Rix, ScalarLoop); z_bru(ScalarDone); bind(ScalarBreak); z_sr(result, Rix); bind(ScalarDone); z_sgfr(Rdst, result); // restore ptr z_sgfr(Rsrc, result); // restore ptr, double the element count for Rsrc restore z_sgfr(Rsrc, result); } bind(AllDone); if (precise) { if (toASCII) { BLOCK_COMMENT("} encode_ascii_array"); } else { BLOCK_COMMENT("} encode_iso_array"); } } else { BLOCK_COMMENT("} string_compress"); } return offset() - block_start; } // Inflate byte[] to char[]. unsigned int C2_MacroAssembler::string_inflate_trot(Register src, Register dst, Register cnt, Register tmp) { int block_start = offset(); BLOCK_COMMENT("string_inflate {"); Register stop_char = Z_R0; Register table = Z_R1; Register src_addr = tmp; assert_different_registers(Z_R0, Z_R1, tmp, src, dst, cnt); assert(dst->encoding()%2 == 0, "must be even reg"); assert(cnt->encoding()%2 == 1, "must be odd reg"); assert(cnt->encoding() - dst->encoding() == 1, "must be even/odd pair"); StubRoutines::zarch::generate_load_trot_table_addr(this, table); // kills Z_R0 (if ASSERT) clear_reg(stop_char); // Stop character. Not used here, but initialized to have a defined value. lgr_if_needed(src_addr, src); z_llgfr(cnt, cnt); // # src characters, must be a positive simm32. translate_ot(dst, src_addr, /* mask = */ 0x0001); BLOCK_COMMENT("} string_inflate"); return offset() - block_start; } // Inflate byte[] to char[]. // Restores: src, dst // Uses: cnt // Kills: tmp, Z_R0, Z_R1. // Note: // cnt is signed int. Do not rely on high word! // counts # characters, not bytes. unsigned int C2_MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp, VectorRegister v20, VectorRegister v21, VectorRegister v22, VectorRegister v23, VectorRegister v24, VectorRegister v25) { assert_different_registers(Z_R0, Z_R1, src, dst, cnt, tmp); BLOCK_COMMENT("string_inflate {"); int block_start = offset(); Register Rcnt = cnt; // # characters (src: bytes, dst: char (2-byte)), remaining after current loop. Register Rix = tmp; // loop index Register Rsrc = src; // addr(src array) Register Rdst = dst; // addr(dst array) Label ScalarShortcut, AllDone; #if 0 // Sacrifice shortcuts for code compactness { //---< shortcuts for short strings (very frequent) >--- Label skipShortcut, skip4Shortcut; z_ltr(Rcnt, Rcnt); // absolutely nothing to do for strings of len == 0. z_brz(AllDone); clear_reg(Z_R0); // make sure registers are properly initialized. clear_reg(Z_R1); z_chi(Rcnt, 4); z_brne(skip4Shortcut); // 4 characters are very frequent z_icm(Z_R0, 5, 0, Rsrc); // Treat exactly 4 characters specially. z_icm(Z_R1, 5, 2, Rsrc); z_stm(Z_R0, Z_R1, 0, Rdst); z_bru(AllDone); bind(skip4Shortcut); z_chi(Rcnt, 8); z_brh(skipShortcut); // There's a lot to do... z_lgfr(Z_R0, Rcnt); // remaining #characters (<= 8). Precond for scalar loop. // This does not destroy the "register cleared" state of Z_R0. z_brl(ScalarShortcut); // Just a few characters z_icmh(Z_R0, 5, 0, Rsrc); // Treat exactly 8 characters specially. z_icmh(Z_R1, 5, 4, Rsrc); z_icm(Z_R0, 5, 2, Rsrc); z_icm(Z_R1, 5, 6, Rsrc); z_stmg(Z_R0, Z_R1, 0, Rdst); z_bru(AllDone); bind(skipShortcut); } #endif clear_reg(Z_R0); // make sure register is properly initialized. if (VM_Version::has_VectorFacility()) { const int min_vcnt = 32; // Minimum #characters required to use vector instructions. // Otherwise just do nothing in vector mode. // Must be multiple of vector register length (16 bytes = 128 bits). const int log_min_vcnt = exact_log2(min_vcnt); Label VectorLoop, VectorDone; assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()"); z_srak(Rix, Rcnt, log_min_vcnt); // calculate # vector loop iterations z_brz(VectorDone); // skip if none z_sllg(Z_R0, Rix, log_min_vcnt); // remember #chars that will be processed by vector loop bind(VectorLoop); z_vlm(v20, v21, 0, Rsrc); // get next 32 characters (single-byte) add2reg(Rsrc, min_vcnt); z_vuplhb(v22, v20); // V2 <- (expand) V0(high) z_vupllb(v23, v20); // V3 <- (expand) V0(low) z_vuplhb(v24, v21); // V4 <- (expand) V1(high) z_vupllb(v25, v21); // V5 <- (expand) V1(low) z_vstm(v22, v25, 0, Rdst); // store next 32 bytes add2reg(Rdst, min_vcnt*2); z_brct(Rix, VectorLoop); bind(VectorDone); } const int min_cnt = 8; // Minimum #characters required to use unrolled scalar loop. // Otherwise just do nothing in unrolled scalar mode. // Must be multiple of 8. { const int log_min_cnt = exact_log2(min_cnt); Label UnrolledLoop, UnrolledDone; if (VM_Version::has_DistinctOpnds()) { z_srk(Rix, Rcnt, Z_R0); // remaining # chars to process in unrolled loop } else { z_lr(Rix, Rcnt); z_sr(Rix, Z_R0); } z_sra(Rix, log_min_cnt); // unrolled loop count z_brz(UnrolledDone); clear_reg(Z_R0); clear_reg(Z_R1); bind(UnrolledLoop); z_icmh(Z_R0, 5, 0, Rsrc); z_icmh(Z_R1, 5, 4, Rsrc); z_icm(Z_R0, 5, 2, Rsrc); z_icm(Z_R1, 5, 6, Rsrc); add2reg(Rsrc, min_cnt); z_stmg(Z_R0, Z_R1, 0, Rdst); add2reg(Rdst, min_cnt*2); z_brct(Rix, UnrolledLoop); bind(UnrolledDone); z_lgfr(Z_R0, Rcnt); // # chars left over after unrolled loop. z_nilf(Z_R0, min_cnt-1); z_brnz(ScalarShortcut); // if zero, there is nothing left to do for scalar loop. // Rix == 0 in all cases. z_sgfr(Z_R0, Rcnt); // negative # characters the ptrs have been advanced previously. z_agr(Rdst, Z_R0); // restore ptr, double the element count for Rdst restore. z_agr(Rdst, Z_R0); z_agr(Rsrc, Z_R0); // restore ptr. z_bru(AllDone); } { bind(ScalarShortcut); // Z_R0 must contain remaining # characters as 64-bit signed int here. // register contents is preserved over scalar processing (for register fixup). #if 0 // Sacrifice shortcuts for code compactness { Label ScalarDefault; z_chi(Rcnt, 2); z_brh(ScalarDefault); z_llc(Z_R0, 0, Z_R0, Rsrc); // 6 bytes z_sth(Z_R0, 0, Z_R0, Rdst); // 4 bytes z_brl(AllDone); z_llc(Z_R0, 1, Z_R0, Rsrc); // 6 bytes z_sth(Z_R0, 2, Z_R0, Rdst); // 4 bytes z_bru(AllDone); bind(ScalarDefault); } #endif Label CodeTable; // Some comments on Rix calculation: // - Rcnt is small, therefore no bits shifted out of low word (sll(g) instructions). // - high word of both Rix and Rcnt may contain garbage // - the final lngfr takes care of that garbage, extending the sign to high word z_sllg(Rix, Z_R0, 2); // calculate 10*Rix = (4*Rix + Rix)*2 z_ar(Rix, Z_R0); z_larl(Z_R1, CodeTable); z_sll(Rix, 1); z_lngfr(Rix, Rix); // ix range: [0..7], after inversion & mult: [-(7*12)..(0*12)]. z_bc(Assembler::bcondAlways, 0, Rix, Z_R1); z_llc(Z_R1, 6, Z_R0, Rsrc); // 6 bytes z_sth(Z_R1, 12, Z_R0, Rdst); // 4 bytes z_llc(Z_R1, 5, Z_R0, Rsrc); z_sth(Z_R1, 10, Z_R0, Rdst); z_llc(Z_R1, 4, Z_R0, Rsrc); z_sth(Z_R1, 8, Z_R0, Rdst); z_llc(Z_R1, 3, Z_R0, Rsrc); z_sth(Z_R1, 6, Z_R0, Rdst); z_llc(Z_R1, 2, Z_R0, Rsrc); z_sth(Z_R1, 4, Z_R0, Rdst); z_llc(Z_R1, 1, Z_R0, Rsrc); z_sth(Z_R1, 2, Z_R0, Rdst); z_llc(Z_R1, 0, Z_R0, Rsrc); z_sth(Z_R1, 0, Z_R0, Rdst); bind(CodeTable); z_chi(Rcnt, 8); // no fixup for small strings. Rdst, Rsrc were not modified. z_brl(AllDone); z_sgfr(Z_R0, Rcnt); // # characters the ptrs have been advanced previously. z_agr(Rdst, Z_R0); // restore ptr, double the element count for Rdst restore. z_agr(Rdst, Z_R0); z_agr(Rsrc, Z_R0); // restore ptr. } bind(AllDone); BLOCK_COMMENT("} string_inflate"); return offset() - block_start; } // Inflate byte[] to char[], length known at compile time. // Restores: src, dst // Kills: tmp, Z_R0, Z_R1. // Note: // len is signed int. Counts # characters, not bytes. unsigned int C2_MacroAssembler::string_inflate_const(Register src, Register dst, Register tmp, int len , VectorRegister v20, VectorRegister v21, VectorRegister v22, VectorRegister v23, VectorRegister v24, VectorRegister v25) { assert_different_registers(Z_R0, Z_R1, src, dst, tmp); BLOCK_COMMENT("string_inflate_const {"); int block_start = offset(); Register Rix = tmp; // loop index Register Rsrc = src; // addr(src array) Register Rdst = dst; // addr(dst array) Label ScalarShortcut, AllDone; int nprocessed = 0; int src_off = 0; // compensate for saved (optimized away) ptr advancement. int dst_off = 0; // compensate for saved (optimized away) ptr advancement. bool restore_inputs = false; bool workreg_clear = false; if ((len >= 32) && VM_Version::has_VectorFacility()) { const int min_vcnt = 32; // Minimum #characters required to use vector instructions. // Otherwise just do nothing in vector mode. // Must be multiple of vector register length (16 bytes = 128 bits). const int log_min_vcnt = exact_log2(min_vcnt); const int iterations = (len - nprocessed) >> log_min_vcnt; nprocessed += iterations << log_min_vcnt; Label VectorLoop; if (iterations == 1) { z_vlm(v20, v21, 0+src_off, Rsrc); // get next 32 characters (single-byte) z_vuplhb(v22, v20); // V2 <- (expand) V0(high) z_vupllb(v23, v20); // V3 <- (expand) V0(low) z_vuplhb(v24, v21); // V4 <- (expand) V1(high) z_vupllb(v25, v21); // V5 <- (expand) V1(low) z_vstm(v22, v25, 0+dst_off, Rdst); // store next 32 bytes src_off += min_vcnt; dst_off += min_vcnt*2; } else { restore_inputs = true; z_lgfi(Rix, len>>log_min_vcnt); bind(VectorLoop); z_vlm(v20, v21, 0, Rsrc); // get next 32 characters (single-byte) add2reg(Rsrc, min_vcnt); z_vuplhb(v22, v20); // V2 <- (expand) V0(high) z_vupllb(v23, v20); // V3 <- (expand) V0(low) z_vuplhb(v24, v21); // V4 <- (expand) V1(high) z_vupllb(v25, v21); // V5 <- (expand) V1(low) z_vstm(v22, v25, 0, Rdst); // store next 32 bytes add2reg(Rdst, min_vcnt*2); z_brct(Rix, VectorLoop); } } if (((len-nprocessed) >= 16) && VM_Version::has_VectorFacility()) { const int min_vcnt = 16; // Minimum #characters required to use vector instructions. // Otherwise just do nothing in vector mode. // Must be multiple of vector register length (16 bytes = 128 bits). const int log_min_vcnt = exact_log2(min_vcnt); const int iterations = (len - nprocessed) >> log_min_vcnt; nprocessed += iterations << log_min_vcnt; assert(iterations == 1, "must be!"); z_vl(v20, 0+src_off, Z_R0, Rsrc); // get next 16 characters (single-byte) z_vuplhb(v22, v20); // V2 <- (expand) V0(high) z_vupllb(v23, v20); // V3 <- (expand) V0(low) z_vstm(v22, v23, 0+dst_off, Rdst); // store next 32 bytes src_off += min_vcnt; dst_off += min_vcnt*2; } if ((len-nprocessed) > 8) { const int min_cnt = 8; // Minimum #characters required to use unrolled scalar loop. // Otherwise just do nothing in unrolled scalar mode. // Must be multiple of 8. const int log_min_cnt = exact_log2(min_cnt); const int iterations = (len - nprocessed) >> log_min_cnt; nprocessed += iterations << log_min_cnt; //---< avoid loop overhead/ptr increment for small # iterations >--- if (iterations <= 2) { clear_reg(Z_R0); clear_reg(Z_R1); workreg_clear = true; z_icmh(Z_R0, 5, 0+src_off, Rsrc); z_icmh(Z_R1, 5, 4+src_off, Rsrc); z_icm(Z_R0, 5, 2+src_off, Rsrc); z_icm(Z_R1, 5, 6+src_off, Rsrc); z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst); src_off += min_cnt; dst_off += min_cnt*2; } if (iterations == 2) { z_icmh(Z_R0, 5, 0+src_off, Rsrc); z_icmh(Z_R1, 5, 4+src_off, Rsrc); z_icm(Z_R0, 5, 2+src_off, Rsrc); z_icm(Z_R1, 5, 6+src_off, Rsrc); z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst); src_off += min_cnt; dst_off += min_cnt*2; } if (iterations > 2) { Label UnrolledLoop; restore_inputs = true; clear_reg(Z_R0); clear_reg(Z_R1); workreg_clear = true; z_lgfi(Rix, iterations); bind(UnrolledLoop); z_icmh(Z_R0, 5, 0, Rsrc); z_icmh(Z_R1, 5, 4, Rsrc); z_icm(Z_R0, 5, 2, Rsrc); z_icm(Z_R1, 5, 6, Rsrc); add2reg(Rsrc, min_cnt); z_stmg(Z_R0, Z_R1, 0, Rdst); add2reg(Rdst, min_cnt*2); z_brct(Rix, UnrolledLoop); } } if ((len-nprocessed) > 0) { switch (len-nprocessed) { case 8: if (!workreg_clear) { clear_reg(Z_R0); clear_reg(Z_R1); } z_icmh(Z_R0, 5, 0+src_off, Rsrc); z_icmh(Z_R1, 5, 4+src_off, Rsrc); z_icm(Z_R0, 5, 2+src_off, Rsrc); z_icm(Z_R1, 5, 6+src_off, Rsrc); z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst); break; case 7: if (!workreg_clear) { clear_reg(Z_R0); clear_reg(Z_R1); } clear_reg(Rix); z_icm(Z_R0, 5, 0+src_off, Rsrc); z_icm(Z_R1, 5, 2+src_off, Rsrc); z_icm(Rix, 5, 4+src_off, Rsrc); z_stm(Z_R0, Z_R1, 0+dst_off, Rdst); z_llc(Z_R0, 6+src_off, Z_R0, Rsrc); z_st(Rix, 8+dst_off, Z_R0, Rdst); z_sth(Z_R0, 12+dst_off, Z_R0, Rdst); break; case 6: if (!workreg_clear) { clear_reg(Z_R0); clear_reg(Z_R1); } clear_reg(Rix); z_icm(Z_R0, 5, 0+src_off, Rsrc); z_icm(Z_R1, 5, 2+src_off, Rsrc); z_icm(Rix, 5, 4+src_off, Rsrc); z_stm(Z_R0, Z_R1, 0+dst_off, Rdst); z_st(Rix, 8+dst_off, Z_R0, Rdst); break; case 5: if (!workreg_clear) { clear_reg(Z_R0); clear_reg(Z_R1); } z_icm(Z_R0, 5, 0+src_off, Rsrc); z_icm(Z_R1, 5, 2+src_off, Rsrc); z_llc(Rix, 4+src_off, Z_R0, Rsrc); z_stm(Z_R0, Z_R1, 0+dst_off, Rdst); z_sth(Rix, 8+dst_off, Z_R0, Rdst); break; case 4: if (!workreg_clear) { clear_reg(Z_R0); clear_reg(Z_R1); } z_icm(Z_R0, 5, 0+src_off, Rsrc); z_icm(Z_R1, 5, 2+src_off, Rsrc); z_stm(Z_R0, Z_R1, 0+dst_off, Rdst); break; case 3: if (!workreg_clear) { clear_reg(Z_R0); } z_llc(Z_R1, 2+src_off, Z_R0, Rsrc); z_icm(Z_R0, 5, 0+src_off, Rsrc); z_sth(Z_R1, 4+dst_off, Z_R0, Rdst); z_st(Z_R0, 0+dst_off, Rdst); break; case 2: z_llc(Z_R0, 0+src_off, Z_R0, Rsrc); z_llc(Z_R1, 1+src_off, Z_R0, Rsrc); z_sth(Z_R0, 0+dst_off, Z_R0, Rdst); z_sth(Z_R1, 2+dst_off, Z_R0, Rdst); break; case 1: z_llc(Z_R0, 0+src_off, Z_R0, Rsrc); z_sth(Z_R0, 0+dst_off, Z_R0, Rdst); break; default: guarantee(false, "Impossible"); break; } src_off += len-nprocessed; dst_off += (len-nprocessed)*2; nprocessed = len; } //---< restore modified input registers >--- if ((nprocessed > 0) && restore_inputs) { z_agfi(Rsrc, -(nprocessed-src_off)); if (nprocessed < 1000000000) { // avoid int overflow z_agfi(Rdst, -(nprocessed*2-dst_off)); } else { z_agfi(Rdst, -(nprocessed-dst_off)); z_agfi(Rdst, -nprocessed); } } BLOCK_COMMENT("} string_inflate_const"); return offset() - block_start; } // Returns the number of non-negative bytes (aka US-ASCII characters) found // before the first negative byte is encountered. unsigned int C2_MacroAssembler::count_positives(Register result, Register src, Register cnt, Register tmp) { const unsigned int block_start = offset(); const unsigned int byte_mask = 0x80; const unsigned int twobyte_mask = byte_mask<<8 | byte_mask; const unsigned int unroll_factor = 16; const unsigned int log_unroll_factor = exact_log2(unroll_factor); Register pos = src; // current position in src array, restored at end Register ctr = result; // loop counter, result value Register mask = tmp; // holds the sign detection mask Label unrolledLoop, unrolledDone, byteLoop, allDone; assert_different_registers(result, src, cnt, tmp); BLOCK_COMMENT("count_positives {"); lgr_if_needed(pos, src); // current position in src array z_srak(ctr, cnt, log_unroll_factor); // # iterations of unrolled loop z_brnh(unrolledDone); // array too short for unrolled loop z_iilf(mask, twobyte_mask<<16 | twobyte_mask); z_iihf(mask, twobyte_mask<<16 | twobyte_mask); bind(unrolledLoop); z_lmg(Z_R0, Z_R1, 0, pos); z_ogr(Z_R0, Z_R1); z_ngr(Z_R0, mask); z_brne(unrolledDone); // There is a negative byte somewhere. // ctr and pos are not updated yet -> // delegate finding correct pos to byteLoop. add2reg(pos, unroll_factor); z_brct(ctr, unrolledLoop); // Once we arrive here, we have to examine at most (unroll_factor - 1) bytes more. // We then either have reached the end of the array or we hit a negative byte. bind(unrolledDone); z_sll(ctr, log_unroll_factor); // calculate # bytes not processed by unrolled loop // > 0 only if a negative byte was found z_lr(Z_R0, cnt); // calculate remainder bytes z_nilf(Z_R0, unroll_factor - 1); z_ar(ctr, Z_R0); // remaining bytes z_brnh(allDone); // shortcut if nothing left to do bind(byteLoop); z_cli(0, pos, byte_mask); // unsigned comparison! byte@pos must be smaller that byte_mask z_brnl(allDone); // negative byte found. add2reg(pos, 1); z_brct(ctr, byteLoop); bind(allDone); z_srk(ctr, cnt, ctr); // # bytes actually processed (= cnt or index of first negative byte) z_sgfr(pos, ctr); // restore src z_lgfr(result, ctr); // unnecessary. Only there to be sure the high word has a defined state. BLOCK_COMMENT("} count_positives"); return offset() - block_start; } // kill: cnt1, cnt2, odd_reg, even_reg; early clobber: result unsigned int C2_MacroAssembler::string_compare(Register str1, Register str2, Register cnt1, Register cnt2, Register odd_reg, Register even_reg, Register result, int ae) { int block_start = offset(); assert_different_registers(str1, cnt1, cnt2, odd_reg, even_reg, result); assert_different_registers(str2, cnt1, cnt2, odd_reg, even_reg, result); // If strings are equal up to min length, return the length difference. const Register diff = result, // Pre-set result with length difference. min = cnt1, // min number of bytes tmp = cnt2; // Note: Making use of the fact that compareTo(a, b) == -compareTo(b, a) // we interchange str1 and str2 in the UL case and negate the result. // Like this, str1 is always latin1 encoded, except for the UU case. // In addition, we need 0 (or sign which is 0) extend when using 64 bit register. const bool used_as_LU = (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL); BLOCK_COMMENT("string_compare {"); if (used_as_LU) { z_srl(cnt2, 1); } // See if the lengths are different, and calculate min in cnt1. // Save diff in case we need it for a tie-breaker. // diff = cnt1 - cnt2 if (VM_Version::has_DistinctOpnds()) { z_srk(diff, cnt1, cnt2); } else { z_lr(diff, cnt1); z_sr(diff, cnt2); } if (str1 != str2) { if (VM_Version::has_LoadStoreConditional()) { z_locr(min, cnt2, Assembler::bcondHigh); } else { Label Lskip; z_brl(Lskip); // min ok if cnt1 < cnt2 z_lr(min, cnt2); // min = cnt2 bind(Lskip); } } if (ae == StrIntrinsicNode::UU) { z_sra(diff, 1); } if (str1 != str2) { Label Ldone; if (used_as_LU) { // Loop which searches the first difference character by character. Label Lloop; const Register ind1 = Z_R1, ind2 = min; int stride1 = 1, stride2 = 2; // See comment above. // ind1: index, even_reg: index increment, odd_reg: index limit z_llilf(ind1, (unsigned int)(-stride1)); z_lhi(even_reg, stride1); add2reg(odd_reg, -stride1, min); clear_reg(ind2); // kills min bind(Lloop); z_brxh(ind1, even_reg, Ldone); z_llc(tmp, Address(str1, ind1)); z_llh(Z_R0, Address(str2, ind2)); z_ahi(ind2, stride2); z_sr(tmp, Z_R0); z_bre(Lloop); z_lr(result, tmp); } else { // Use clcle in fast loop (only for same encoding). z_lgr(Z_R0, str1); z_lgr(even_reg, str2); z_llgfr(Z_R1, min); z_llgfr(odd_reg, min); if (ae == StrIntrinsicNode::LL) { compare_long_ext(Z_R0, even_reg, 0); } else { compare_long_uni(Z_R0, even_reg, 0); } z_bre(Ldone); z_lgr(Z_R1, Z_R0); if (ae == StrIntrinsicNode::LL) { z_llc(Z_R0, Address(even_reg)); z_llc(result, Address(Z_R1)); } else { z_llh(Z_R0, Address(even_reg)); z_llh(result, Address(Z_R1)); } z_sr(result, Z_R0); } // Otherwise, return the difference between the first mismatched chars. bind(Ldone); } if (ae == StrIntrinsicNode::UL) { z_lcr(result, result); // Negate result (see note above). } BLOCK_COMMENT("} string_compare"); return offset() - block_start; } unsigned int C2_MacroAssembler::array_equals(bool is_array_equ, Register ary1, Register ary2, Register limit, Register odd_reg, Register even_reg, Register result, bool is_byte) { int block_start = offset(); BLOCK_COMMENT("array_equals {"); assert_different_registers(ary1, limit, odd_reg, even_reg); assert_different_registers(ary2, limit, odd_reg, even_reg); Label Ldone, Ldone_true, Ldone_false, Lclcle, CLC_template; int base_offset = 0; if (ary1 != ary2) { if (is_array_equ) { base_offset = arrayOopDesc::base_offset_in_bytes(is_byte ? T_BYTE : T_CHAR); // Return true if the same array. compareU64_and_branch(ary1, ary2, Assembler::bcondEqual, Ldone_true); // Return false if one of them is null. compareU64_and_branch(ary1, (intptr_t)0, Assembler::bcondEqual, Ldone_false); compareU64_and_branch(ary2, (intptr_t)0, Assembler::bcondEqual, Ldone_false); // Load the lengths of arrays. z_llgf(odd_reg, Address(ary1, arrayOopDesc::length_offset_in_bytes())); // Return false if the two arrays are not equal length. z_c(odd_reg, Address(ary2, arrayOopDesc::length_offset_in_bytes())); z_brne(Ldone_false); // string len in bytes (right operand) if (!is_byte) { z_chi(odd_reg, 128); z_sll(odd_reg, 1); // preserves flags z_brh(Lclcle); } else { compareU32_and_branch(odd_reg, (intptr_t)256, Assembler::bcondHigh, Lclcle); } } else { z_llgfr(odd_reg, limit); // Need to zero-extend prior to using the value. compareU32_and_branch(limit, (intptr_t)256, Assembler::bcondHigh, Lclcle); } // Use clc instruction for up to 256 bytes. { Register str1_reg = ary1, str2_reg = ary2; if (is_array_equ) { str1_reg = Z_R1; str2_reg = even_reg; add2reg(str1_reg, base_offset, ary1); // string addr (left operand) add2reg(str2_reg, base_offset, ary2); // string addr (right operand) } z_ahi(odd_reg, -1); // Clc uses decremented limit. Also compare result to 0. z_brl(Ldone_true); // Note: We could jump to the template if equal. assert(VM_Version::has_ExecuteExtensions(), "unsupported hardware"); z_exrl(odd_reg, CLC_template); z_bre(Ldone_true); // fall through bind(Ldone_false); clear_reg(result); z_bru(Ldone); bind(CLC_template); z_clc(0, 0, str1_reg, 0, str2_reg); } // Use clcle instruction. { bind(Lclcle); add2reg(even_reg, base_offset, ary2); // string addr (right operand) add2reg(Z_R0, base_offset, ary1); // string addr (left operand) z_lgr(Z_R1, odd_reg); // string len in bytes (left operand) if (is_byte) { compare_long_ext(Z_R0, even_reg, 0); } else { compare_long_uni(Z_R0, even_reg, 0); } z_lghi(result, 0); // Preserve flags. z_brne(Ldone); } } // fall through bind(Ldone_true); z_lghi(result, 1); // All characters are equal. bind(Ldone); BLOCK_COMMENT("} array_equals"); return offset() - block_start; } // kill: haycnt, needlecnt, odd_reg, even_reg; early clobber: result unsigned int C2_MacroAssembler::string_indexof(Register result, Register haystack, Register haycnt, Register needle, Register needlecnt, int needlecntval, Register odd_reg, Register even_reg, int ae) { int block_start = offset(); // Ensure 0 256) { bind(L_clcle); // Main Loop: clcle version (now we have at least 256 bytes). Label L_OuterLoop, CLC_template; bind(L_OuterLoop); // Search for 1st 2 characters. z_lgr(Z_R1, haycnt); if (h_csize == 1) { MacroAssembler::search_string(Z_R1, result); } else { MacroAssembler::search_string_uni(Z_R1, result); } z_brc(Assembler::bcondNotFound, L_NotFound); add2reg(Z_R0, n_csize, needle); add2reg(even_reg, h_csize, Z_R1); z_lgr(result, Z_R1); if (needlecnt != noreg) { z_llgfr(Z_R1, needlecnt); // needle len in bytes (left operand) z_llgfr(odd_reg, needlecnt); } else { load_const_optimized(Z_R1, needle_bytes); if (Immediate::is_simm16(needle_bytes)) { z_lghi(odd_reg, needle_bytes); } else { z_lgr(odd_reg, Z_R1); } } if (h_csize == 1) { compare_long_ext(Z_R0, even_reg, 0); } else { compare_long_uni(Z_R0, even_reg, 0); } z_bre(L_Found); if (n_csize == 2) { z_llgh(Z_R0, Address(needle)); } else { z_llgc(Z_R0, Address(needle)); } // Reload. z_aghi(result, h_csize); // This is the new address we want to use for comparing. z_bru(L_OuterLoop); } } if (needlecnt != noreg || needlecntval == 1) { bind(L_needle1); // Single needle character version. if (h_csize == 1) { MacroAssembler::search_string(haycnt, result); } else { MacroAssembler::search_string_uni(haycnt, result); } z_lgr(result, haycnt); z_brc(Assembler::bcondFound, L_Found); } bind(L_NotFound); add2reg(result, -1, haystack); // Return -1. bind(L_Found); // Return index (or -1 in fallthrough case). z_sgr(result, haystack); if (h_csize == 2) { z_srag(result, result, exact_log2(sizeof(jchar))); } } BLOCK_COMMENT("} string_indexof"); return offset() - block_start; } // early clobber: result unsigned int C2_MacroAssembler::string_indexof_char(Register result, Register haystack, Register haycnt, Register needle, jchar needleChar, Register odd_reg, Register even_reg, bool is_byte) { int block_start = offset(); BLOCK_COMMENT("string_indexof_char {"); if (needle == haystack) { z_lhi(result, 0); } else { Label Ldone; z_llgfr(odd_reg, haycnt); // Preset loop ctr/searchrange end. if (needle == noreg) { load_const_optimized(Z_R0, (unsigned long)needleChar); } else { if (is_byte) { z_llgcr(Z_R0, needle); // First (and only) needle char. } else { z_llghr(Z_R0, needle); // First (and only) needle char. } } if (!is_byte) { z_agr(odd_reg, odd_reg); // Calc #bytes to be processed with SRSTU. } z_lgr(even_reg, haystack); // haystack addr z_agr(odd_reg, haystack); // First char after range end. z_lghi(result, -1); if (is_byte) { MacroAssembler::search_string(odd_reg, even_reg); } else { MacroAssembler::search_string_uni(odd_reg, even_reg); } z_brc(Assembler::bcondNotFound, Ldone); if (is_byte) { if (VM_Version::has_DistinctOpnds()) { z_sgrk(result, odd_reg, haystack); } else { z_sgr(odd_reg, haystack); z_lgr(result, odd_reg); } } else { z_slgr(odd_reg, haystack); z_srlg(result, odd_reg, exact_log2(sizeof(jchar))); } bind(Ldone); } BLOCK_COMMENT("} string_indexof_char"); return offset() - block_start; }