diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 57f81f7fd51e..6151bef13d9c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -60,7 +60,7 @@ jobs:
runs-on: ubuntu-24.04
env:
# List of platforms to exclude by default
- EXCLUDED_PLATFORMS: 'alpine-linux-x64'
+ EXCLUDED_PLATFORMS: 'alpine-linux-x64,macos-x64'
outputs:
linux-x64: ${{ steps.include.outputs.linux-x64 }}
linux-x64-variants: ${{ steps.include.outputs.linux-x64-variants }}
diff --git a/doc/testing.html b/doc/testing.html
index 648bc8baa6ae..f52f72ac2f7d 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -424,11 +424,29 @@
JCOV
For more fine-grained control, you can pass arbitrary filters to JCov
using --with-jcov-filters, and you can specify a specific
JDK to instrument using --with-jcov-input-jdk.
+The resulting coverage is written into
+build/$BUILD/test-results/jcov-output/result.xml.
The JCov report is stored in
build/$BUILD/test-results/jcov-output/report.
Please note that running with JCov reporting can be very memory
intensive.
-JCOV_DIFF_CHANGESET
+JCov scales
+JCov scales make it possible to record which tests cover each part of
+the instrumented code. To collect coverage with scales, set
+JCOV_SCALES=true, for example:
+$ make jcov-test TEST=jdk_lang TEST_OPTS="JCOV_SCALES=true"
+The resulting coverage data contains the association between covered
+code and the tests that covered it. A corresponding
+testlist.txt file, which contains the test names, is
+generated in the same directory.
+The JCov report displays the names of the tests that cover each
+class.
+Collecting coverage scales forces jtreg tests to be run in
+othervm mode, which takes longer than ordinary JCov
+collection. The coverage data is also larger because it includes scale
+information, and the generated report is larger because it includes test
+names.
+JCOV_DIFF_CHANGESET
While collecting code coverage with JCov, it is also possible to find
coverage for only recently changed code. JCOV_DIFF_CHANGESET specifies a
source revision. A textual report will be generated showing coverage of
diff --git a/doc/testing.md b/doc/testing.md
index 1de6c94b679f..e02a9e09d4e7 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -353,11 +353,33 @@ For more fine-grained control, you can pass arbitrary filters to JCov using
`--with-jcov-filters`, and you can specify a specific JDK to instrument
using `--with-jcov-input-jdk`.
+The resulting coverage is written into
+`build/$BUILD/test-results/jcov-output/result.xml`.
+
The JCov report is stored in `build/$BUILD/test-results/jcov-output/report`.
Please note that running with JCov reporting can be very memory intensive.
-#### JCOV_DIFF_CHANGESET
+##### JCov scales
+
+JCov scales make it possible to record which tests cover each part of the
+instrumented code. To collect coverage with scales, set `JCOV_SCALES=true`,
+for example:
+
+ $ make jcov-test TEST=jdk_lang TEST_OPTS="JCOV_SCALES=true"
+
+The resulting coverage data contains the association between covered code and
+the tests that covered it. A corresponding `testlist.txt` file, which contains
+the test names, is generated in the same directory.
+
+The JCov report displays the names of the tests that cover each class.
+
+Collecting coverage scales forces jtreg tests to be run in `othervm` mode,
+which takes longer than ordinary JCov collection. The coverage data is also
+larger because it includes scale information, and the generated report is
+larger because it includes test names.
+
+##### JCOV_DIFF_CHANGESET
While collecting code coverage with JCov, it is also possible to find coverage
for only recently changed code. JCOV_DIFF_CHANGESET specifies a source
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index 1433ab32e5ce..1013f38baa77 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -45,7 +45,7 @@ ifneq ($(TEST_VM_OPTS), )
endif
$(eval $(call ParseKeywordVariable, TEST_OPTS, \
- SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR JCOV JCOV_DIFF_CHANGESET AOT_JDK, \
+ SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR JCOV JCOV_DIFF_CHANGESET JCOV_SCALES AOT_JDK, \
STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS, \
))
@@ -121,9 +121,21 @@ ifeq ($(TEST_OPTS_JCOV), true)
JCOV_SUPPORT_DIR := $(TEST_SUPPORT_DIR)/jcov-support
JCOV_GRABBER_LOG := $(JCOV_OUTPUT_DIR)/grabber.log
JCOV_RESULT_FILE := $(JCOV_OUTPUT_DIR)/result.xml
+ JCOV_TESTLIST := $(JCOV_OUTPUT_DIR)/testlist.txt
JCOV_REPORT := $(JCOV_OUTPUT_DIR)/report
+ JCOV_GRABBER_OPTIONS ?=
+ JCOV_REPGEN_OPTIONS ?=
+ TEST_OPTS_JCOV_SCALES ?= false
JCOV_MEM_OPTIONS := -Xms64m -Xmx4g
+ ifeq ($(TEST_OPTS_JCOV_SCALES), true)
+ JCOV_GRABBER_OPTIONS += -scale -mergebyname -outTestList $(JCOV_TESTLIST)
+ TEST_JOBS := 1
+ JTREG_TEST_MODE := othervm
+ JTREG_VM_OPTIONS += -Djcov.extension=com.sun.tdk.jcov.runtime.TestNameDecorator
+ JCOV_REPGEN_OPTIONS += -tests $(JCOV_TESTLIST)
+ endif
+
# Replace our normal test JDK with the JCov image.
JDK_UNDER_TEST := $(JCOV_IMAGE_DIR)
@@ -1414,6 +1426,7 @@ ifeq ($(TEST_OPTS_JCOV), true)
fi
$(JAVA) $(JCOV_VM_OPTS) -jar $(JCOV_HOME)/lib/jcov.jar Grabber -v -t \
$(JCOV_IMAGE_DIR)/template.xml -o $(JCOV_RESULT_FILE) \
+ $(JCOV_GRABBER_OPTIONS) \
1>$(JCOV_GRABBER_LOG) 2>&1 &
jcov-start-grabber: jcov-do-start-grabber
@@ -1441,6 +1454,7 @@ ifeq ($(TEST_OPTS_JCOV), true)
`$(ECHO) $(TOPDIR)/src/*/share/classes/ | $(TR) ' ' ':'` -fmt html \
$(JCOV_MODULES_FILTER) $(JCOV_FILTERS) \
-mainReportTitle "$(JCOV_REPORT_TITLE)" \
+ $(JCOV_REPGEN_OPTIONS) \
-o $(JCOV_REPORT) $(JCOV_RESULT_FILE))
TARGETS += jcov-do-start-grabber jcov-start-grabber jcov-stop-grabber \
diff --git a/make/autoconf/flags-cflags.m4 b/make/autoconf/flags-cflags.m4
index 8f8a7af47fca..04a475c65b63 100644
--- a/make/autoconf/flags-cflags.m4
+++ b/make/autoconf/flags-cflags.m4
@@ -213,6 +213,7 @@ AC_DEFUN([FLAGS_SETUP_WARNINGS],
WARNINGS_ENABLE_ADDITIONAL=""
WARNINGS_ENABLE_ADDITIONAL_CXX=""
WARNINGS_ENABLE_ADDITIONAL_JVM=""
+ WARNINGS_ENABLE_ADDITIONAL_JDK="-w34189"
DISABLED_WARNINGS="4800 5105"
CFLAGS_CONVERSION_WARNINGS=
;;
@@ -623,8 +624,8 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_HELPER],
ADLC_LANGSTD_CXXFLAGS="$LANGSTD_CXXFLAGS"
# CFLAGS WARNINGS STUFF
- WARNING_CFLAGS_JDK_CONLY="$WARNINGS_ENABLE_ALL"
- WARNING_CFLAGS_JDK_CXXONLY="$WARNINGS_ENABLE_ALL_CXX"
+ WARNING_CFLAGS_JDK_CONLY="$WARNINGS_ENABLE_ALL $WARNINGS_ENABLE_ADDITIONAL_JDK"
+ WARNING_CFLAGS_JDK_CXXONLY="$WARNINGS_ENABLE_ALL_CXX $WARNINGS_ENABLE_ADDITIONAL_JDK"
WARNING_CFLAGS_JVM="$WARNINGS_ENABLE_ALL_JVM"
# Set some additional per-OS defines.
diff --git a/make/autoconf/jdk-options.m4 b/make/autoconf/jdk-options.m4
index 9abc849f0080..aaac824ecddb 100644
--- a/make/autoconf/jdk-options.m4
+++ b/make/autoconf/jdk-options.m4
@@ -829,9 +829,18 @@ AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_NOCOOPS],
AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_PREVIEW],
[
- UTIL_ARG_ENABLE(NAME: cds-archive-preview, DEFAULT: true, RESULT: BUILD_CDS_ARCHIVE_PREVIEW,
+ UTIL_ARG_ENABLE(NAME: cds-archive-preview, DEFAULT: auto, RESULT: BUILD_CDS_ARCHIVE_PREVIEW,
DESC: [enable generation of preview CDS archives (requires --enable-cds-archive)],
- CHECKING_MSG: [if default CDS archives for preview should be generated])
+ CHECKING_MSG: [if default CDS archives for preview should be generated],
+ CHECK_AVAILABLE: [
+ AC_MSG_CHECKING([if value objects are supported])
+ if test "x$OPENJDK_TARGET_CPU_BITS" = "x64"; then
+ AC_MSG_RESULT([yes])
+ else
+ AC_MSG_RESULT([no (64-bit only)])
+ AVAILABLE=false
+ fi
+ ])
AC_SUBST(BUILD_CDS_ARCHIVE_PREVIEW)
])
diff --git a/make/autoconf/platform.m4 b/make/autoconf/platform.m4
index 28aea489f7ec..cdeae97c8ac8 100644
--- a/make/autoconf/platform.m4
+++ b/make/autoconf/platform.m4
@@ -660,7 +660,19 @@ AC_DEFUN([PLATFORM_CHECK_DEPRECATION],
[
AC_ARG_ENABLE(deprecated-ports, [AS_HELP_STRING([--enable-deprecated-ports@<:@=yes/no@:>@],
[Suppress the error when configuring for a deprecated port @<:@no@:>@])])
- # There are no deprecated ports. Implement the deprecation warnings here.
+ if test "x$OPENJDK_TARGET_OS" = xmacosx && test "x$OPENJDK_TARGET_CPU" = xx86_64; then
+ # Unfortunately, variants have not been parsed yet, so we have to check the configure option
+ # directly. Allow only the directly specified Zero variant, treat any other mix as containing
+ # something non-Zero.
+ if test "x$with_jvm_variants" != xzero; then
+ if test "x$enable_deprecated_ports" = "xyes"; then
+ AC_MSG_WARN([The macOS/x64 port is deprecated and may be removed in a future release.])
+ else
+ AC_MSG_ERROR(m4_normalize([The macOS/x64 port is deprecated and may be removed in a future release.
+ Use --enable-deprecated-ports to suppress this error.]))
+ fi
+ fi
+ fi
])
AC_DEFUN_ONCE([PLATFORM_SETUP_OPENJDK_BUILD_OS_VERSION],
diff --git a/make/common/TestFilesCompilation.gmk b/make/common/TestFilesCompilation.gmk
index fd1c54eaf484..dd02d90c537e 100644
--- a/make/common/TestFilesCompilation.gmk
+++ b/make/common/TestFilesCompilation.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -118,6 +118,7 @@ define SetupTestFilesCompilationBody
DISABLED_WARNINGS_clang := format-nonliteral \
missing-field-initializers sometimes-uninitialized undef \
unused-but-set-variable unused-function unused-variable, \
+ DISABLED_WARNINGS_microsoft := 4189, \
DEFAULT_LIBCXX := false, \
JDK_LIBS := $$($1_JDK_LIBS_$$(name)), \
LIBS := $$($1_LIBS) $$($1_LIBS_$$(name)), \
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index 32f07325c058..b83ca709a888 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -416,6 +416,7 @@ var getJibProfilesProfiles = function (input, common, data) {
"--with-zlib=system",
"--with-macosx-version-max=11.00.00",
"--enable-compatible-cds-alignment",
+ "--enable-deprecated-ports",
// Use system SetFile instead of the one in the devkit as the
// devkit one may not work on Catalina.
"SETFILE=/usr/bin/SetFile"
@@ -1192,8 +1193,8 @@ var getJibProfilesDependencies = function (input, common) {
server: "jpg",
product: "jcov",
version: "3.0",
- build_number: "6",
- file: "bundles/jcov-3.0+6.zip",
+ build_number: "9",
+ file: "bundles/jcov-3.0+9.zip",
environment_name: "JCOV_HOME",
},
diff --git a/make/hotspot/lib/CompileJvm.gmk b/make/hotspot/lib/CompileJvm.gmk
index bb67474b3a07..50b632a49f91 100644
--- a/make/hotspot/lib/CompileJvm.gmk
+++ b/make/hotspot/lib/CompileJvm.gmk
@@ -285,13 +285,6 @@ ifeq ($(call isTargetOs, windows), true)
$(BUILD_LIBJVM_TARGET): $(WIN_EXPORT_FILE)
endif
-# Always recompile abstract_vm_version.cpp if libjvm needs to be relinked. This ensures
-# that the internal vm version is updated as it relies on __DATE__ and __TIME__
-# macros.
-ABSTRACT_VM_VERSION_OBJ := $(JVM_OUTPUTDIR)/objs/abstract_vm_version$(OBJ_SUFFIX)
-$(ABSTRACT_VM_VERSION_OBJ): $(filter-out $(ABSTRACT_VM_VERSION_OBJ), \
- $(BUILD_LIBJVM_TARGET_DEPS))
-
ifneq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
ifeq ($(call isTargetOs, windows), true)
# It doesn't matter which jvm.lib file gets exported, but we need
diff --git a/make/modules/java.base/gensrc/GensrcMisc.gmk b/make/modules/java.base/gensrc/GensrcMisc.gmk
index edb5e8bc58ef..2c563f4bfd4f 100644
--- a/make/modules/java.base/gensrc/GensrcMisc.gmk
+++ b/make/modules/java.base/gensrc/GensrcMisc.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2026, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -72,7 +72,7 @@ else
OPENJDK_TARGET_OS_CANONICAL := $(OPENJDK_TARGET_OS)
endif
-$(eval $(call SetupTextFileProcessing, BUILD_PLATFORMPROPERTIES_JAVA, \
+$(eval $(call SetupTextFileProcessing, BUILD_PLATFORMPROPERTIES_JAVA_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU), \
SOURCE_FILES := $(TOPDIR)/src/java.base/share/classes/jdk/internal/util/PlatformProps.java.template, \
OUTPUT_FILE := $(SUPPORT_OUTPUTDIR)/gensrc/java.base/jdk/internal/util/PlatformProps.java, \
REPLACEMENTS := \
@@ -82,7 +82,8 @@ $(eval $(call SetupTextFileProcessing, BUILD_PLATFORMPROPERTIES_JAVA, \
@@OPENJDK_TARGET_CPU_BITS@@ => $(OPENJDK_TARGET_CPU_BITS), \
))
-TARGETS += $(BUILD_VERSION_JAVA) $(BUILD_PLATFORMPROPERTIES_JAVA)
+TARGETS += $(BUILD_VERSION_JAVA) \
+ $(BUILD_PLATFORMPROPERTIES_JAVA_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU))
################################################################################
ifneq ($(filter $(TOOLCHAIN_TYPE), gcc clang), )
diff --git a/make/modules/java.desktop/Lib.gmk b/make/modules/java.desktop/Lib.gmk
index 1c43950480c4..9d2a82d60321 100644
--- a/make/modules/java.desktop/Lib.gmk
+++ b/make/modules/java.desktop/Lib.gmk
@@ -64,10 +64,8 @@ ifeq ($(ENABLE_JSOUND), true)
EXTRA_HEADER_DIRS := java.base:libjava, \
CFLAGS := $(LIBJSOUND_CFLAGS), \
CXXFLAGS := $(LIBJSOUND_CFLAGS), \
- DISABLED_WARNINGS_gcc := undef unused-variable, \
- DISABLED_WARNINGS_clang := undef unused-variable, \
- DISABLED_WARNINGS_clang_PLATFORM_API_MacOSX_MidiUtils.c := \
- unused-but-set-variable, \
+ DISABLED_WARNINGS_gcc := undef, \
+ DISABLED_WARNINGS_clang := undef, \
DISABLED_WARNINGS_clang_DirectAudioDevice.c := unused-function, \
LIBS_linux := $(ALSA_LIBS), \
LIBS_macosx := \
diff --git a/make/modules/java.desktop/lib/AwtLibraries.gmk b/make/modules/java.desktop/lib/AwtLibraries.gmk
index 887dfab01dff..ca98434d3830 100644
--- a/make/modules/java.desktop/lib/AwtLibraries.gmk
+++ b/make/modules/java.desktop/lib/AwtLibraries.gmk
@@ -134,7 +134,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBAWT, \
DISABLED_WARNINGS_clang_debug_trace.c := format-nonliteral, \
DISABLED_WARNINGS_clang_Trace.c := format-nonliteral, \
DISABLED_WARNINGS_clang_TransformHelper.c := sign-compare, \
- DISABLED_WARNINGS_microsoft := 4244 4996, \
+ DISABLED_WARNINGS_microsoft := 4244 4996 4189, \
DISABLED_WARNINGS_microsoft_awt_Toolkit.cpp := 4267, \
LDFLAGS_windows := -delayload:comctl32.dll -delayload:comdlg32.dll \
-delayload:gdi32.dll -delayload:imm32.dll -delayload:ole32.dll \
diff --git a/make/modules/java.desktop/lib/ClientLibraries.gmk b/make/modules/java.desktop/lib/ClientLibraries.gmk
index 2326505d11c2..a09e54ce5ab4 100644
--- a/make/modules/java.desktop/lib/ClientLibraries.gmk
+++ b/make/modules/java.desktop/lib/ClientLibraries.gmk
@@ -343,7 +343,7 @@ else
expansion-to-defined dangling-reference maybe-uninitialized
HARFBUZZ_DISABLED_WARNINGS_clang := missing-field-initializers \
range-loop-analysis unused-variable
- HARFBUZZ_DISABLED_WARNINGS_microsoft := 4267 4244
+ HARFBUZZ_DISABLED_WARNINGS_microsoft := 4267 4244 4189
LIBFONTMANAGER_CFLAGS += $(HARFBUZZ_CFLAGS)
endif
diff --git a/src/hotspot/cpu/aarch64/bytes_aarch64.hpp b/src/hotspot/cpu/aarch64/bytes_aarch64.hpp
deleted file mode 100644
index 6d4a18d00b5f..000000000000
--- a/src/hotspot/cpu/aarch64/bytes_aarch64.hpp
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_AARCH64_BYTES_AARCH64_HPP
-#define CPU_AARCH64_BYTES_AARCH64_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/byteswap.hpp"
-
-class Bytes: AllStatic {
- public:
- // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering
- // (no special code is needed since x86 CPUs can access unaligned data)
- static inline u2 get_native_u2(address p) { return *(u2*)p; }
- static inline u4 get_native_u4(address p) { return *(u4*)p; }
- static inline u8 get_native_u8(address p) { return *(u8*)p; }
-
- static inline void put_native_u2(address p, u2 x) { *(u2*)p = x; }
- static inline void put_native_u4(address p, u4 x) { *(u4*)p = x; }
- static inline void put_native_u8(address p, u8 x) { *(u8*)p = x; }
-
-
- // Efficient reading and writing of unaligned unsigned data in Java
- // byte ordering (i.e. big-endian ordering). Byte-order reversal is
- // needed since x86 CPUs use little-endian format.
- static inline u2 get_Java_u2(address p) { return byteswap(get_native_u2(p)); }
- static inline u4 get_Java_u4(address p) { return byteswap(get_native_u4(p)); }
- static inline u8 get_Java_u8(address p) { return byteswap(get_native_u8(p)); }
-
- static inline void put_Java_u2(address p, u2 x) { put_native_u2(p, byteswap(x)); }
- static inline void put_Java_u4(address p, u4 x) { put_native_u4(p, byteswap(x)); }
- static inline void put_Java_u8(address p, u8 x) { put_native_u8(p, byteswap(x)); }
-};
-
-#endif // CPU_AARCH64_BYTES_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index d05526890cba..9a35a01c4d2d 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -1569,13 +1569,8 @@ void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
// We are storing into an array that *may* be null-free (the declared type is
// Object[], abstract[], interface[] or VT.ref[]).
- Label test_mark_word;
Register tmp = op->tmp()->as_register();
__ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
- __ tst(tmp, markWord::unlocked_value);
- __ br(Assembler::NE, test_mark_word);
- __ load_prototype_header(tmp, op->array()->as_register());
- __ bind(test_mark_word);
__ tst(tmp, markWord::null_free_array_bit_in_place);
}
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index fba316bf293e..96a4f4d3daaf 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -221,8 +221,8 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register t1,
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
// Try to lock. Transition lock-bits 0b01 => 0b00
- orr(t1_mark, t1_mark, markWord::unlocked_value);
- eor(t3_t, t1_mark, markWord::unlocked_value);
+ orr(t1_mark, t1_mark, markWord::lock_neutral_value);
+ eor(t3_t, t1_mark, markWord::lock_neutral_value);
cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword, memory_order_acquire);
br(Assembler::NE, slow_path);
@@ -383,7 +383,7 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register box, Register t1,
// Try to unlock. Transition lock bits 0b00 => 0b01
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
- orr(t3_t, t1_mark, markWord::unlocked_value);
+ orr(t3_t, t1_mark, markWord::lock_neutral_value);
cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword, memory_order_release);
br(Assembler::EQ, unlocked);
diff --git a/src/hotspot/cpu/aarch64/codeBuffer_aarch64.cpp b/src/hotspot/cpu/aarch64/codeBuffer_aarch64.cpp
index 97d9f7afdfb4..9c8c7393354d 100644
--- a/src/hotspot/cpu/aarch64/codeBuffer_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/codeBuffer_aarch64.cpp
@@ -1,4 +1,5 @@
/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -37,7 +38,7 @@ void CodeBuffer::share_trampoline_for(address dest, int caller_offset) {
if (created) {
_shared_trampoline_requests->maybe_grow();
}
- offsets->add(caller_offset);
+ offsets->push(caller_offset);
_finalize_stubs = true;
}
@@ -50,16 +51,19 @@ static bool emit_shared_trampolines(CodeBuffer* cb, CodeBuffer::SharedTrampoline
MacroAssembler masm(cb);
- auto emit = [&](address dest, const CodeBuffer::Offsets &offsets) {
+ auto emit = [&](address dest, const CodeBuffer::Offsets& offsets) {
assert(cb->stubs()->remaining() >= MacroAssembler::max_trampoline_stub_size(), "pre-allocated trampolines");
- LinkedListIterator it(offsets.head());
- int offset = *it.next();
+ assert(offsets.length() > 0, "must be");
+ // We go backwards
+ const int offset_end = offsets.length() - 1;
+ int offset = offsets.at(offset_end);
address stub = __ emit_trampoline_stub(offset, dest);
assert(stub, "pre-allocated trampolines");
address reloc_pc = cb->stubs()->end() - NativeCallTrampolineStub::instruction_size;
- while (!it.is_empty()) {
- offset = *it.next();
+ // Skip the first one
+ for (int i = offset_end - 1; i >= 0; i--) {
+ offset = offsets.at(i);
address caller_pc = cb->insts()->start() + offset;
cb->stubs()->relocate(reloc_pc, trampoline_stub_Relocation::spec(caller_pc));
}
diff --git a/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp b/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp
index 7cc2a004c40d..7d9acafddc09 100644
--- a/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp
@@ -98,25 +98,6 @@ char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size
result = reserve_at_eor_compatible_address(size, aslr);
}
- // Movk-compatible reservation via probing.
- if (result == nullptr) {
- result = reserve_address_space_for_16bit_move(size, aslr);
- }
-
- // Movk-compatible reservation via overallocation.
- // If that failed, attempt to allocate at any 4G-aligned address. Let the system decide where. For ASLR,
- // we now rely on the system.
- // Compared with the probing done above, this has two disadvantages:
- // - on a kernel with 52-bit address space we may get an address that has bits set between [48, 52).
- // In that case, we may need two movk moves (not yet implemented).
- // - this technique leads to temporary over-reservation of address space; it will spike the vsize of
- // the process. Therefore it may fail if a vsize limit is in place (e.g. ulimit -v).
- if (result == nullptr) {
- constexpr size_t alignment = nth_bit(32);
- log_debug(metaspace, map)("Trying to reserve at a 32-bit-aligned address");
- result = os::reserve_memory_aligned(size, alignment, mtNone);
- }
-
return result;
}
diff --git a/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
index db0d5e007a02..766b2b4dda0a 100644
--- a/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp
@@ -308,8 +308,10 @@ void DowncallLinker::StubGenerator::generate() {
// Restore cpu control state after JNI call
__ restore_cpu_control_state_after_jni(rscratch1, tmp1);
- __ mov(tmp1, _thread_in_vm);
- __ strw(tmp1, Address(rthread, JavaThread::thread_state_offset()));
+ // change thread state
+ __ mov(tmp1, _thread_in_Java);
+ __ lea(tmp2, Address(rthread, JavaThread::thread_state_offset()));
+ __ stlrw(tmp1, tmp2);
// Force this write out before the read below
if (!UseSystemMemoryBarrier) {
@@ -326,11 +328,6 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_after_safepoint_poll);
- // change thread state
- __ mov(tmp1, _thread_in_Java);
- __ lea(tmp2, Address(rthread, JavaThread::thread_state_offset()));
- __ stlrw(tmp1, tmp2);
-
__ block_comment("reguard stack check");
__ ldrb(tmp1, Address(rthread, JavaThread::stack_guard_state_offset()));
__ cmpw(tmp1, StackOverflow::stack_guard_yellow_reserved_disabled);
@@ -356,7 +353,7 @@ void DowncallLinker::StubGenerator::generate() {
__ mov(c_rarg0, rthread);
assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
- __ lea(tmp1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
+ __ lea(tmp1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans)));
__ blr(tmp1);
if (should_save_return_value) {
@@ -388,5 +385,5 @@ void DowncallLinker::StubGenerator::generate() {
//////////////////////////////////////////////////////////////////////////////
- __ flush();
+ // Code will be copied. No ICache sync required.
}
diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
index 74e0395c81ec..11b70e4e8384 100644
--- a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
+++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
@@ -33,14 +33,14 @@ source %{
#include "gc/z/zBarrierSetAssembler.hpp"
-static void z_color(MacroAssembler* masm, const MachNode* node, Register dst, Register src) {
+static void z_color(MacroAssembler* masm, Register dst, Register src) {
assert_different_registers(src, dst);
__ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
__ movzw(dst, barrier_Relocation::unpatched);
__ orr(dst, dst, src, Assembler::LSL, ZPointerLoadShift);
}
-static void z_uncolor(MacroAssembler* masm, const MachNode* node, Register ref) {
+static void z_uncolor(MacroAssembler* masm, Register ref) {
__ lsr(ref, ref, ZPointerLoadShift);
}
@@ -50,7 +50,7 @@ static void z_keep_alive_load_barrier(MacroAssembler* masm, const MachNode* node
__ tst(ref, tmp);
ZLoadBarrierStubC2Aarch64* const stub = ZLoadBarrierStubC2Aarch64::create(node, ref_addr, ref);
__ br(Assembler::NE, *stub->entry());
- z_uncolor(masm, node, ref);
+ z_uncolor(masm, ref);
__ bind(*stub->continuation());
}
@@ -66,7 +66,7 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
}
if (node->barrier_data() == ZBarrierElided) {
- z_uncolor(masm, node, ref);
+ z_uncolor(masm, ref);
return;
}
@@ -81,14 +81,14 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
__ b(*stub->entry());
__ bind(good);
}
- z_uncolor(masm, node, ref);
+ z_uncolor(masm, ref);
__ bind(*stub->continuation());
}
static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
if (node->barrier_data() == ZBarrierElided) {
- z_color(masm, node, rnew_zpointer, rnew_zaddress);
+ z_color(masm, rnew_zpointer, rnew_zaddress);
} else {
bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
@@ -206,7 +206,7 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
- z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
+ z_color(masm, $oldval_tmp$$Register, $oldval$$Register);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword, memory_order_release);
__ cset($res$$Register, Assembler::EQ);
%}
@@ -229,7 +229,7 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
- z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
+ z_color(masm, $oldval_tmp$$Register, $oldval$$Register);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword, memory_order_seq_cst);
__ cset($res$$Register, Assembler::EQ);
%}
@@ -251,10 +251,10 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
- z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
+ z_color(masm, $oldval_tmp$$Register, $oldval$$Register);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
memory_order_release, $res$$Register);
- z_uncolor(masm, this, $res$$Register);
+ z_uncolor(masm, $res$$Register);
%}
ins_pipe(pipe_slow);
@@ -274,10 +274,10 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
- z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
+ z_color(masm, $oldval_tmp$$Register, $oldval$$Register);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
memory_order_seq_cst, $res$$Register);
- z_uncolor(masm, this, $res$$Register);
+ z_uncolor(masm, $res$$Register);
%}
ins_pipe(pipe_slow);
@@ -295,7 +295,7 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
ins_encode %{
z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
__ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
- z_uncolor(masm, this, $prev$$Register);
+ z_uncolor(masm, $prev$$Register);
%}
ins_pipe(pipe_serial);
@@ -313,7 +313,7 @@ instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr)
ins_encode %{
z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
__ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
- z_uncolor(masm, this, $prev$$Register);
+ z_uncolor(masm, $prev$$Register);
%}
ins_pipe(pipe_serial);
diff --git a/src/hotspot/cpu/aarch64/interpreterRT_aarch64.cpp b/src/hotspot/cpu/aarch64/interpreterRT_aarch64.cpp
index c9daef8c6cfa..a92f8f5014cb 100644
--- a/src/hotspot/cpu/aarch64/interpreterRT_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/interpreterRT_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -181,7 +181,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
__ lea(r0, ExternalAddress(Interpreter::result_handler(method()->result_type())));
__ ret(lr);
- __ flush();
+ __ invalidate_icache();
}
diff --git a/src/hotspot/cpu/aarch64/jniFastGetField_aarch64.cpp b/src/hotspot/cpu/aarch64/jniFastGetField_aarch64.cpp
index cffdcf494296..8fd1ffca14cb 100644
--- a/src/hotspot/cpu/aarch64/jniFastGetField_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/jniFastGetField_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2004, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -206,7 +206,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
__ leave();
__ ret(lr);
}
- __ flush ();
+ __ invalidate_icache();
return fast_entry;
}
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 6d280d9e8ab9..74fd6e0d42af 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -5318,12 +5318,6 @@ void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Regi
cmpw(tmp1, tmp2);
}
-void MacroAssembler::load_prototype_header(Register dst, Register src) {
- Register tmp = (dst == rscratch1) ? rscratch2 : rscratch1;
- load_klass(dst, src, tmp);
- ldr(dst, Address(dst, Klass::prototype_header_offset()));
-}
-
void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
@@ -5494,12 +5488,6 @@ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode(address base,
}
}
- const uint64_t shifted_base =
- (uint64_t)base >> shift;
- if ((shifted_base & 0xffff0000ffffffff) == 0) {
- return KlassDecodeMovk;
- }
-
return KlassDecodeFallback;
}
@@ -5545,14 +5533,6 @@ void MacroAssembler::emit_encode_klass_not_null(Register dst, Register src, Regi
lsr(dst, dst, shift);
break;
- case KlassDecodeMovk:
- if (shift != 0) {
- ubfx(dst, src, shift, 32);
- } else {
- movw(dst, src);
- }
- break;
-
case KlassDecodeFallback: {
mov(tmp, base);
sub(dst, src, tmp);
@@ -5609,16 +5589,6 @@ void MacroAssembler::emit_decode_klass_not_null(Register dst, Register src, Regi
eor(dst, dst, (uint64_t)base);
break;
- case KlassDecodeMovk: { // 1-3 instructions
- const uint64_t shifted_base =
- (uint64_t)base >> shift;
-
- if (dst != src) movw(dst, src);
- movk(dst, shifted_base >> 32, 32);
- lsl(dst, dst, shift);
- break;
- }
-
case KlassDecodeFallback: { // 3-4 instructions
mov(tmp, base);
add(dst, tmp, src, LSL, shift);
@@ -7920,13 +7890,13 @@ void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register t1, R
// Try to lock. Transition lock bits 0b01 => 0b00
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
- orr(mark, mark, markWord::unlocked_value);
+ orr(mark, mark, markWord::lock_neutral_value);
if (Arguments::is_valhalla_enabled()) {
// Mask inline_type bit such that we go to the slow path if object is an inline type
andr(mark, mark, ~((int) markWord::inline_type_bit_in_place));
}
- eor(t, mark, markWord::unlocked_value);
+ eor(t, mark, markWord::lock_neutral_value);
cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword, memory_order_acquire);
br(Assembler::NE, slow);
@@ -7985,16 +7955,16 @@ void MacroAssembler::fast_unlock(Register obj, Register t1, Register t2, Registe
tbnz(mark, log2i_exact(markWord::monitor_value), push_and_slow);
#ifdef ASSERT
- // Check header not unlocked (0b01).
+ // Check header not unlocked / lock-neutral (0b01).
Label not_unlocked;
- tbz(mark, log2i_exact(markWord::unlocked_value), not_unlocked);
+ tbz(mark, log2i_exact(markWord::lock_neutral_value), not_unlocked);
stop("fast_unlock already unlocked");
bind(not_unlocked);
#endif
// Try to unlock. Transition lock bits 0b00 => 0b01
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
- orr(t, mark, markWord::unlocked_value);
+ orr(t, mark, markWord::lock_neutral_value);
cmpxchg(obj, mark, t, Assembler::xword, memory_order_release);
br(Assembler::EQ, unlocked);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index 826f88fe85c3..ba46983c7a07 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -104,7 +104,6 @@ class MacroAssembler: public Assembler {
KlassDecodeNone,
KlassDecodeZero,
KlassDecodeXor,
- KlassDecodeMovk,
KlassDecodeFallback
};
@@ -1000,8 +999,6 @@ class MacroAssembler: public Assembler {
// stored using routines that take a jobject.
void store_heap_oop_null(Address dst);
- void load_prototype_header(Register dst, Register src);
-
void store_klass_gap(Register dst, Register src);
// This dummy is to prevent a call to store_heap_oop from
diff --git a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
index 638e57b03fee..9620aba88f8c 100644
--- a/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/runtime_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -249,8 +249,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
// Jump to interpreter
__ ret(lr);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
UncommonTrapBlob *ut_blob = UncommonTrapBlob::create(&buffer, oop_maps,
SimpleRuntimeFrame::framesize >> 1);
@@ -391,8 +390,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
__ br(r8);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Set exception blob
ExceptionBlob* ex_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
@@ -400,5 +398,3 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
return ex_blob;
}
#endif // COMPILER2
-
-
diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
index 60065ab19406..e2b8e1a1ed94 100644
--- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
@@ -1612,7 +1612,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
assert(vep_offset != -1, "Must be set");
#endif
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod* nm = nmethod::new_native_nmethod(method,
compile_id,
masm->code(),
@@ -1646,7 +1646,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
in_sig_bt,
in_regs);
int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
- __ flush();
+ // Code will be copied. No ICache sync required.
int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually
return nmethod::new_native_nmethod(method,
compile_id,
@@ -2047,9 +2047,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
Label safepoint_in_progress, safepoint_in_progress_done;
- __ mov(rscratch1, _thread_in_vm);
-
- __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
+ // change thread state
+ __ mov(rscratch1, _thread_in_Java);
+ __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
+ __ stlrw(rscratch1, rscratch2);
// Force this write out before the read below
if (!UseSystemMemoryBarrier) {
@@ -2067,11 +2068,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ bind(safepoint_in_progress_done);
}
- // change thread state
- __ mov(rscratch1, _thread_in_Java);
- __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
- __ stlrw(rscratch1, rscratch2);
-
if (method->is_object_wait0()) {
// Check preemption for Object.wait()
__ ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
@@ -2273,7 +2269,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
#ifndef PRODUCT
assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
#endif
- __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
+ __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans)));
__ blr(rscratch1);
// Restore any method result value
@@ -2316,7 +2312,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
}
}
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod *nm = nmethod::new_native_nmethod(method,
compile_id,
@@ -2657,8 +2653,7 @@ void SharedRuntime::generate_deopt_blob() {
// Jump to interpreter
__ ret(lr);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
_deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
_deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
@@ -2806,8 +2801,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
__ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
#endif
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Fill-out other meta info
SafepointBlob* sp_blob = SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
@@ -2902,9 +2896,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
__ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
__ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
- // -------------
- // make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// return the blob
// frame_size_words or bytes??
@@ -3058,7 +3050,7 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
__ ret(lr);
- __ flush();
+ // Code will be copied. No ICache sync required.
return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
}
@@ -3309,9 +3301,7 @@ RuntimeStub* SharedRuntime::generate_return_value_stub(address destination) {
__ leave();
__ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
- // -------------
- // make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, frame_size_in_words, oop_maps, false);
AOTCodeCache::store_code_blob(*stub, AOTCodeEntry::SharedBlob, StubInfo::blob(id));
diff --git a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
index 9c53800dd34b..c317629d6cc9 100644
--- a/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp
@@ -1422,7 +1422,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
__ verify_sve_vector_length();
// change thread state
- __ mov(rscratch1, _thread_in_vm);
+ __ mov(rscratch1, _thread_in_Java);
__ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
__ stlrw(rscratch1, rscratch2);
@@ -1447,18 +1447,13 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// hand.
//
__ mov(c_rarg0, rthread);
- __ lea(rscratch2, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
+ __ lea(rscratch2, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans)));
__ blr(rscratch2);
__ get_method(rmethod);
__ reinit_heapbase();
__ bind(Continue);
}
- // change thread state
- __ mov(rscratch1, _thread_in_Java);
- __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
- __ stlrw(rscratch1, rscratch2);
-
// Check preemption for Object.wait()
Label not_preempted;
__ ldr(rscratch1, Address(rthread, JavaThread::preempt_alternate_return_offset()));
diff --git a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
index 7a0e5aaf3b4e..7f678317a9da 100644
--- a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2022, Arm Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -310,7 +310,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
//////////////////////////////////////////////////////////////////////////////
- _masm->flush();
+ // Code will be copied. No ICache sync required.
#ifndef PRODUCT
stringStream ss;
diff --git a/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
index a070b4f66024..c81bf733a1fb 100644
--- a/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -132,7 +132,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
__ ldr(rscratch1, Address(rmethod, entry_offset));
__ br(rscratch1);
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
return s;
@@ -233,7 +233,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
assert(SharedRuntime::get_handle_wrong_method_stub() != nullptr, "check initialization order");
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
return s;
diff --git a/src/hotspot/cpu/arm/bytes_arm.hpp b/src/hotspot/cpu/arm/bytes_arm.hpp
deleted file mode 100644
index 6ebf5a61e4f7..000000000000
--- a/src/hotspot/cpu/arm/bytes_arm.hpp
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_ARM_BYTES_ARM_HPP
-#define CPU_ARM_BYTES_ARM_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/macros.hpp"
-
-#ifndef VM_LITTLE_ENDIAN
-#define VM_LITTLE_ENDIAN 1
-#endif
-
-class Bytes: AllStatic {
-
- public:
- static inline u2 get_Java_u2(address p) {
- return (u2(p[0]) << 8) | u2(p[1]);
- }
-
- static inline u4 get_Java_u4(address p) {
- return u4(p[0]) << 24 |
- u4(p[1]) << 16 |
- u4(p[2]) << 8 |
- u4(p[3]);
- }
-
- static inline u8 get_Java_u8(address p) {
- return u8(p[0]) << 56 |
- u8(p[1]) << 48 |
- u8(p[2]) << 40 |
- u8(p[3]) << 32 |
- u8(p[4]) << 24 |
- u8(p[5]) << 16 |
- u8(p[6]) << 8 |
- u8(p[7]);
- }
-
- static inline void put_Java_u2(address p, u2 x) {
- p[0] = x >> 8;
- p[1] = x;
- }
-
- static inline void put_Java_u4(address p, u4 x) {
- ((u1*)p)[0] = x >> 24;
- ((u1*)p)[1] = x >> 16;
- ((u1*)p)[2] = x >> 8;
- ((u1*)p)[3] = x;
- }
-
- static inline void put_Java_u8(address p, u8 x) {
- ((u1*)p)[0] = x >> 56;
- ((u1*)p)[1] = x >> 48;
- ((u1*)p)[2] = x >> 40;
- ((u1*)p)[3] = x >> 32;
- ((u1*)p)[4] = x >> 24;
- ((u1*)p)[5] = x >> 16;
- ((u1*)p)[6] = x >> 8;
- ((u1*)p)[7] = x;
- }
-
-#ifdef VM_LITTLE_ENDIAN
-
- static inline u2 get_native_u2(address p) {
- return (intptr_t(p) & 1) == 0 ? *(u2*)p : u2(p[0]) | (u2(p[1]) << 8);
- }
-
- static inline u4 get_native_u4(address p) {
- switch (intptr_t(p) & 3) {
- case 0: return *(u4*)p;
- case 2: return u4(((u2*)p)[0]) |
- u4(((u2*)p)[1]) << 16;
- default: return u4(p[0]) |
- u4(p[1]) << 8 |
- u4(p[2]) << 16 |
- u4(p[3]) << 24;
- }
- }
-
- static inline u8 get_native_u8(address p) {
- switch (intptr_t(p) & 7) {
- case 0: return *(u8*)p;
- case 4: return u8(((u4*)p)[0]) |
- u8(((u4*)p)[1]) << 32;
- case 2: return u8(((u2*)p)[0]) |
- u8(((u2*)p)[1]) << 16 |
- u8(((u2*)p)[2]) << 32 |
- u8(((u2*)p)[3]) << 48;
- default: return u8(p[0]) |
- u8(p[1]) << 8 |
- u8(p[2]) << 16 |
- u8(p[3]) << 24 |
- u8(p[4]) << 32 |
- u8(p[5]) << 40 |
- u8(p[6]) << 48 |
- u8(p[7]) << 56;
- }
- }
-
- static inline void put_native_u2(address p, u2 x) {
- if ((intptr_t(p) & 1) == 0) {
- *(u2*)p = x;
- } else {
- p[0] = x;
- p[1] = x >> 8;
- }
- }
-
- static inline void put_native_u4(address p, u4 x) {
- switch (intptr_t(p) & 3) {
- case 0: *(u4*)p = x;
- break;
- case 2: ((u2*)p)[0] = x;
- ((u2*)p)[1] = x >> 16;
- break;
- default: ((u1*)p)[0] = x;
- ((u1*)p)[1] = x >> 8;
- ((u1*)p)[2] = x >> 16;
- ((u1*)p)[3] = x >> 24;
- break;
- }
- }
-
- static inline void put_native_u8(address p, u8 x) {
- switch (intptr_t(p) & 7) {
- case 0: *(u8*)p = x;
- break;
- case 4: ((u4*)p)[0] = x;
- ((u4*)p)[1] = x >> 32;
- break;
- case 2: ((u2*)p)[0] = x;
- ((u2*)p)[1] = x >> 16;
- ((u2*)p)[2] = x >> 32;
- ((u2*)p)[3] = x >> 48;
- break;
- default: ((u1*)p)[0] = x;
- ((u1*)p)[1] = x >> 8;
- ((u1*)p)[2] = x >> 16;
- ((u1*)p)[3] = x >> 24;
- ((u1*)p)[4] = x >> 32;
- ((u1*)p)[5] = x >> 40;
- ((u1*)p)[6] = x >> 48;
- ((u1*)p)[7] = x >> 56;
- }
- }
-
-#else
-
- static inline u2 get_native_u2(address p) { return get_Java_u2(p); }
- static inline u4 get_native_u4(address p) { return get_Java_u4(p); }
- static inline u8 get_native_u8(address p) { return get_Java_u8(p); }
- static inline void put_native_u2(address p, u2 x) { put_Java_u2(p, x); }
- static inline void put_native_u4(address p, u4 x) { put_Java_u4(p, x); }
- static inline void put_native_u8(address p, u8 x) { put_Java_u8(p, x); }
-
-#endif // VM_LITTLE_ENDIAN
-};
-
-#endif // CPU_ARM_BYTES_ARM_HPP
diff --git a/src/hotspot/cpu/arm/jniFastGetField_arm.cpp b/src/hotspot/cpu/arm/jniFastGetField_arm.cpp
index 3a5dd10e82eb..fc7e439effa2 100644
--- a/src/hotspot/cpu/arm/jniFastGetField_arm.cpp
+++ b/src/hotspot/cpu/arm/jniFastGetField_arm.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,9 +26,11 @@
#include "asm/macroAssembler.hpp"
#include "code/codeBlob.hpp"
#include "memory/resourceArea.hpp"
+#include "oops/instanceKlass.hpp"
#include "prims/jniFastGetField.hpp"
#include "prims/jvm_misc.hpp"
#include "prims/jvmtiExport.hpp"
+#include "runtime/jfieldIDWorkaround.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/safepoint.hpp"
@@ -138,10 +140,10 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
#endif // !__ABI_HARD__
) {
// Only ldr and ldrb support embedded shift, other loads do not
- __ add(Robj, Robj, AsmOperand(R2, lsr, 2));
+ __ add(Robj, Robj, AsmOperand(R2, lsr, jfieldIDWorkaround::offset_shift));
field_addr = Address(Robj);
} else {
- field_addr = Address(Robj, R2, lsr, 2);
+ field_addr = Address(Robj, R2, lsr, jfieldIDWorkaround::offset_shift);
}
assert(count < LIST_CAPACITY, "LIST_CAPACITY too small");
speculative_load_pclist[count] = __ pc();
@@ -210,7 +212,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
__ bind_literal(safepoint_counter_addr);
- __ flush();
+ __ invalidate_icache();
guarantee((__ pc() - fast_entry) <= BUFFER_SIZE, "BUFFER_SIZE too small");
diff --git a/src/hotspot/cpu/arm/macroAssembler_arm.cpp b/src/hotspot/cpu/arm/macroAssembler_arm.cpp
index 6715effa68fd..ce79fc57ccd4 100644
--- a/src/hotspot/cpu/arm/macroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/macroAssembler_arm.cpp
@@ -1780,7 +1780,7 @@ void MacroAssembler::fast_lock(Register obj, Register t1, Register t2, Register
Register new_hdr = t2;
ldr(new_hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
bic(new_hdr, new_hdr, markWord::lock_mask_in_place); // new header (00)
- orr(old_hdr, new_hdr, markWord::unlocked_value); // old header (01)
+ orr(old_hdr, new_hdr, markWord::lock_neutral_value); // old header (01)
Label dummy;
@@ -1829,7 +1829,7 @@ void MacroAssembler::fast_unlock(Register obj, Register t1, Register t2, Registe
Register new_hdr = t2;
ldr(old_hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
bic(old_hdr, old_hdr, markWord::lock_mask_in_place); // old header (00)
- orr(new_hdr, old_hdr, markWord::unlocked_value); // new header (01)
+ orr(new_hdr, old_hdr, markWord::lock_neutral_value); // new header (01)
// Try to swing header from locked to unlocked
Label dummy;
diff --git a/src/hotspot/cpu/arm/macroAssembler_arm.hpp b/src/hotspot/cpu/arm/macroAssembler_arm.hpp
index 3119c7141b1f..59c13e05ef6f 100644
--- a/src/hotspot/cpu/arm/macroAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/macroAssembler_arm.hpp
@@ -449,7 +449,7 @@ class MacroAssembler: public Assembler {
int should_not_call_this() {
raw_push(FP, LR);
should_not_reach_here();
- flush();
+ invalidate_icache();
return 2; // frame_size_in_words (FP+LR)
}
diff --git a/src/hotspot/cpu/arm/runtime_arm.cpp b/src/hotspot/cpu/arm/runtime_arm.cpp
index 29fd0aa0a103..5a1845ac0593 100644
--- a/src/hotspot/cpu/arm/runtime_arm.cpp
+++ b/src/hotspot/cpu/arm/runtime_arm.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -176,7 +176,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
__ mov(SP, FP);
__ pop(RegisterSet(FP) | RegisterSet(PC));
- masm->flush();
+ masm->invalidate_icache();
return UncommonTrapBlob::create(&buffer, nullptr, 2 /* LR+FP */);
}
@@ -280,7 +280,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
// -------------
// make sure all code is generated
- masm->flush();
+ masm->invalidate_icache();
return ExceptionBlob::create(&buffer, oop_maps, framesize_in_words);
}
diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
index 593ba159aa7b..ed4d9c8f2e73 100644
--- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
+++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
@@ -850,7 +850,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
in_sig_bt,
in_regs);
int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
- __ flush();
+ __ invalidate_icache();
int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually
return nmethod::new_native_nmethod(method,
compile_id,
@@ -938,8 +938,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ ldr(Rtemp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- assert(markWord::unlocked_value == 1, "adjust this code");
- __ tbz(Rtemp, exact_log2(markWord::unlocked_value), slow_case);
+ assert(markWord::lock_neutral_value == 1, "adjust this code");
+ __ tbz(Rtemp, exact_log2(markWord::lock_neutral_value), slow_case);
__ bics(Rtemp, Rtemp, ~markWord::hash_mask_in_place);
__ mov(R0, AsmOperand(Rtemp, lsr, markWord::hash_shift), ne);
@@ -1263,9 +1263,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ c2bool(R0);
}
- // Do a safepoint check
+ // Perform thread state transition
Label call_safepoint_runtime, return_to_java;
- __ mov(Rtemp, _thread_in_vm);
+ __ mov(Rtemp, _thread_in_Java);
__ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
// make sure the store is observed before reading the SafepointSynchronize state and further mem refs
@@ -1273,6 +1273,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
}
+ // Do a safepoint check
__ safepoint_poll(R2, call_safepoint_runtime);
__ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
__ cmp(R3, 0);
@@ -1280,12 +1281,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ bind(return_to_java);
- // Perform thread state transition and reguard stack yellow pages if needed
+ // Reguard stack yellow pages if needed
Label reguard, reguard_done;
- __ mov(Rtemp, _thread_in_Java);
__ ldr_s32(R2, Address(Rthread, JavaThread::stack_guard_state_offset()));
- __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
-
__ cmp(R2, StackOverflow::stack_guard_yellow_reserved_disabled);
__ b(reguard, eq);
__ bind(reguard_done);
@@ -1336,7 +1334,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ bind(call_safepoint_runtime);
push_result_registers(masm, ret_type);
__ mov(R0, Rthread);
- __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
+ __ call(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans));
pop_result_registers(masm, ret_type);
__ b(return_to_java);
@@ -1385,7 +1383,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ b(unlock_done);
}
- __ flush();
+ __ invalidate_icache();
return nmethod::new_native_nmethod(method,
compile_id,
masm->code(),
@@ -1654,7 +1652,7 @@ void SharedRuntime::generate_deopt_blob() {
__ pop(RegisterSet(FP) | RegisterSet(PC));
- __ flush();
+ __ invalidate_icache();
_deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset,
reexecute_offset, frame_size_in_words);
@@ -1734,7 +1732,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
__ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
- __ flush();
+ __ invalidate_icache();
return SafepointBlob::create(&buffer, oop_maps, frame_size_words);
}
@@ -1794,7 +1792,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
__ mov(Rexception_pc, LR);
__ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
- __ flush();
+ __ invalidate_icache();
return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_words, oop_maps, true);
}
diff --git a/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp b/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp
index 99f30a8f2669..b7e055b317b9 100644
--- a/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp
+++ b/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1013,8 +1013,8 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
__ restore_default_fp_mode();
}
- // Do safepoint check
- __ mov(Rtemp, _thread_in_vm);
+ // Perform Native->Java thread transition
+ __ mov(Rtemp, _thread_in_Java);
__ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
// Force this write out before the read below
@@ -1033,6 +1033,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
saved_result_fp = fnoreg;
#endif // __ABI_HARD__
+ // Do safepoint check
{
Label call, skip_call;
__ safepoint_poll(Rtemp, call);
@@ -1041,7 +1042,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
__ b(skip_call, eq);
__ bind(call);
__ mov(R0, Rthread);
- __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none);
+ __ call(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans), relocInfo::none);
__ bind(skip_call);
#if R9_IS_SCRATCHED
@@ -1049,10 +1050,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
#endif
}
- // Perform Native->Java thread transition
- __ mov(Rtemp, _thread_in_Java);
- __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
-
// Zero handles and last_java_sp
__ reset_last_Java_frame(Rtemp);
__ ldr(R3, Address(Rthread, JavaThread::active_handles_offset()));
diff --git a/src/hotspot/cpu/arm/vtableStubs_arm.cpp b/src/hotspot/cpu/arm/vtableStubs_arm.cpp
index 80b3cb3a400f..3f34fa76969e 100644
--- a/src/hotspot/cpu/arm/vtableStubs_arm.cpp
+++ b/src/hotspot/cpu/arm/vtableStubs_arm.cpp
@@ -110,7 +110,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
address ame_addr = __ pc();
__ ldr(PC, Address(Rmethod, Method::from_compiled_offset()));
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
return s;
@@ -205,7 +205,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
assert(SharedRuntime::get_handle_wrong_method_stub() != nullptr, "check initialization order");
__ jump(SharedRuntime::get_handle_wrong_method_stub(), relocInfo::runtime_call_type, Rtemp);
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
return s;
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index 77c7f63cd062..87c12f3e4ed5 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -539,6 +539,10 @@ class Assembler : public AbstractAssembler {
STXVL_OPCODE = (31u << OPCODE_SHIFT | 397u << 1),
LXVD2X_OPCODE = (31u << OPCODE_SHIFT | 844u << 1),
STXVD2X_OPCODE = (31u << OPCODE_SHIFT | 972u << 1),
+ LXVW4X_OPCODE = (31u << OPCODE_SHIFT | 780u << 1),
+ STXVW4X_OPCODE = (31u << OPCODE_SHIFT | 908u << 1),
+ LXVB16X_OPCODE = (31u << OPCODE_SHIFT | 876u << 1),
+ STXVB16X_OPCODE= (31u << OPCODE_SHIFT | 1004u << 1),
MTVSRD_OPCODE = (31u << OPCODE_SHIFT | 179u << 1),
MTVSRDD_OPCODE = (31u << OPCODE_SHIFT | 435u << 1),
MTVSRWZ_OPCODE = (31u << OPCODE_SHIFT | 243u << 1),
@@ -1365,10 +1369,6 @@ class Assembler : public AbstractAssembler {
return (0 == addr % a);
}
- void flush() {
- AbstractAssembler::flush();
- }
-
inline void emit_int32(int); // shadows AbstractAssembler::emit_int32
inline void emit_data(int);
inline void emit_data(int, RelocationHolder const&);
@@ -2386,8 +2386,17 @@ class Assembler : public AbstractAssembler {
inline void lxvd2x( VectorSRegister d, Register a, Register b);
inline void stxvd2x( VectorSRegister d, Register a);
inline void stxvd2x( VectorSRegister d, Register a, Register b);
+ inline void lxvw4x( VectorSRegister d, Register a);
+ inline void lxvw4x( VectorSRegister d, Register a, Register b);
+ inline void stxvw4x( VectorSRegister d, Register a);
+ inline void stxvw4x( VectorSRegister d, Register a, Register b);
// Power9
+ inline void lxvb16x( VectorSRegister d, Register a);
+ inline void lxvb16x( VectorSRegister d, Register a, Register b);
+ inline void stxvb16x( VectorSRegister d, Register a);
+ inline void stxvb16x( VectorSRegister d, Register a, Register b);
+
inline void lxv( VectorSRegister d, int si16, Register a);
inline void stxv( VectorSRegister d, int si16, Register a);
inline void lxvx( VectorSRegister d, Register a, Register b);
@@ -2590,6 +2599,15 @@ class Assembler : public AbstractAssembler {
inline void vec_perm(VectorRegister first_dest, VectorRegister second, VectorRegister perm);
inline void vec_perm(VectorRegister dest, VectorRegister first, VectorRegister second, VectorRegister perm);
+ // Load/Store unaligned vectors with offs (multiple of 16). Byte versions require vp for Power8 LE.
+ inline void load_byte_vector_unaligned(VectorRegister dest, int offs, Register base, Register tmp,
+ VectorRegister vp); // vp should be pre-computed (see generator below)
+ inline void store_byte_vector_unaligned(VectorRegister val, int offs, Register base, Register tmp,
+ VectorRegister vp, VectorRegister vtmp = vnoreg); // clobbers val if no vtmp provided
+ inline void compute_vp_for_byte_vector_unaligned(VectorRegister dest, VectorRegister vtmp);
+ inline void load_word_vector_unaligned(VectorRegister dest, int offs, Register base, Register tmp);
+ inline void store_word_vector_unaligned(VectorRegister val, int offs, Register base, Register tmp);
+
// RegisterOrConstant versions.
// These emitters choose between the versions using two registers and
// those with register and immediate, depending on the content of roc.
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 22b9e268dcdf..7929e895923b 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -856,6 +856,14 @@ inline void Assembler::lxvd2x( VectorSRegister d, Register s1) { e
inline void Assembler::lxvd2x( VectorSRegister d, Register s1, Register s2) { emit_int32( LXVD2X_OPCODE | vsrt(d) | ra0mem(s1) | rb(s2)); }
inline void Assembler::stxvd2x( VectorSRegister d, Register s1) { emit_int32( STXVD2X_OPCODE | vsrs(d) | ra(0) | rb(s1)); }
inline void Assembler::stxvd2x( VectorSRegister d, Register s1, Register s2) { emit_int32( STXVD2X_OPCODE | vsrs(d) | ra0mem(s1) | rb(s2)); }
+inline void Assembler::lxvw4x( VectorSRegister d, Register s1) { emit_int32( LXVW4X_OPCODE | vsrt(d) | ra(0) | rb(s1)); }
+inline void Assembler::lxvw4x( VectorSRegister d, Register s1, Register s2) { emit_int32( LXVW4X_OPCODE | vsrt(d) | ra0mem(s1) | rb(s2)); }
+inline void Assembler::stxvw4x( VectorSRegister d, Register s1) { emit_int32( STXVW4X_OPCODE | vsrs(d) | ra(0) | rb(s1)); }
+inline void Assembler::stxvw4x( VectorSRegister d, Register s1, Register s2) { emit_int32( STXVW4X_OPCODE | vsrs(d) | ra0mem(s1) | rb(s2)); }
+inline void Assembler::lxvb16x( VectorSRegister d, Register s1) { emit_int32( LXVB16X_OPCODE | vsrt(d) | ra(0) | rb(s1)); }
+inline void Assembler::lxvb16x( VectorSRegister d, Register s1, Register s2) { emit_int32( LXVB16X_OPCODE | vsrt(d) | ra0mem(s1) | rb(s2)); }
+inline void Assembler::stxvb16x(VectorSRegister d, Register s1) { emit_int32( STXVB16X_OPCODE| vsrs(d) | ra(0) | rb(s1)); }
+inline void Assembler::stxvb16x(VectorSRegister d, Register s1, Register s2) { emit_int32( STXVB16X_OPCODE| vsrs(d) | ra0mem(s1) | rb(s2)); }
inline void Assembler::mtvsrd( VectorSRegister d, Register a) { emit_int32( MTVSRD_OPCODE | vsrt(d) | ra(a)); }
inline void Assembler::mtvsrdd( VectorSRegister d, Register a, Register b) { emit_int32( MTVSRDD_OPCODE | vsrt(d) | ra(a) | rb(b)); }
inline void Assembler::mfvsrd( Register d, VectorSRegister a) { emit_int32( MFVSRD_OPCODE | vsrs(a) | ra(d)); }
@@ -1232,6 +1240,108 @@ inline void Assembler::vec_perm(VectorRegister dest, VectorRegister first, Vecto
#endif
}
+inline void Assembler::load_byte_vector_unaligned(VectorRegister dest, int offs, Register base, Register tmp,
+ VectorRegister vp) {
+ VectorSRegister vsr = dest->to_vsr();
+ if (PowerArchitecturePPC64 >= 9) {
+#if !defined(VM_LITTLE_ENDIAN)
+ lxv(vsr, offs, base); // all vector load/store instructions use the same byte order on BE
+#else
+ if (offs == 0) {
+ lxvb16x(vsr, base);
+ } else {
+ li(tmp, offs);
+ lxvb16x(vsr, base, tmp);
+ }
+#endif
+ } else { // Power8 only supports very limited instructions
+ if (offs == 0) {
+ lxvd2x(vsr, base);
+ } else {
+ li(tmp, offs);
+ lxvd2x(vsr, base, tmp);
+ }
+#if defined(VM_LITTLE_ENDIAN)
+ // need to swap bytes in both double-words
+ vperm(dest, dest, dest, vp);
+#endif
+ }
+}
+
+inline void Assembler::store_byte_vector_unaligned(VectorRegister val, int offs, Register base, Register tmp,
+ VectorRegister vp, VectorRegister vtmp) {
+ VectorSRegister vsr = val->to_vsr();
+ if (PowerArchitecturePPC64 >= 9) {
+#if !defined(VM_LITTLE_ENDIAN)
+ stxv(vsr, offs, base); // all vector load/store instructions use the same byte order on BE
+#else
+ if (offs == 0) {
+ stxvb16x(vsr, base);
+ } else {
+ li(tmp, offs);
+ stxvb16x(vsr, base, tmp);
+ }
+#endif
+ } else { // Power8 only supports very limited instructions
+#if defined(VM_LITTLE_ENDIAN)
+ // need to swap bytes in both double-words
+ if (vtmp != vnoreg) {
+ vperm(vtmp, val, val, vp);
+ vsr = vtmp->to_vsr();
+ } else {
+ vperm(val, val, val, vp); // clobbers val!
+ }
+#endif
+ if (offs == 0) {
+ stxvd2x(vsr, base);
+ } else {
+ li(tmp, offs);
+ stxvd2x(vsr, base, tmp);
+ }
+ }
+}
+
+inline void Assembler::compute_vp_for_byte_vector_unaligned(VectorRegister dest, VectorRegister vtmp) {
+#if defined(VM_LITTLE_ENDIAN)
+ if (PowerArchitecturePPC64 < 9) {
+ li(R0, 0);
+ vspltisb(vtmp, 7); // vtmp = [7, ..., 7]
+ lvsl(dest, R0); // dest = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+ vxor(dest, dest, vtmp); // dest = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]
+ }
+#endif
+}
+
+inline void Assembler::load_word_vector_unaligned(VectorRegister dest, int offs, Register base, Register tmp) {
+ VectorSRegister vsr = dest->to_vsr();
+#if !defined(VM_LITTLE_ENDIAN)
+ if (PowerArchitecturePPC64 >= 9) {
+ lxv(vsr, offs, base); // all vector load/store instructions use the same byte order on BE
+ } else
+#endif
+ if (offs == 0) {
+ lxvw4x(vsr, base);
+ } else {
+ li(tmp, offs);
+ lxvw4x(vsr, base, tmp);
+ }
+}
+
+inline void Assembler::store_word_vector_unaligned(VectorRegister val, int offs, Register base, Register tmp) {
+ VectorSRegister vsr = val->to_vsr();
+#if !defined(VM_LITTLE_ENDIAN)
+ if (PowerArchitecturePPC64 >= 9) {
+ stxv(vsr, offs, base); // all vector load/store instructions use the same byte order on BE
+ } else
+#endif
+ if (offs == 0) {
+ stxvw4x(vsr, base);
+ } else {
+ li(tmp, offs);
+ stxvw4x(vsr, base, tmp);
+ }
+}
+
inline void Assembler::load_const(Register d, void* x, Register tmp) {
load_const(d, (long)x, tmp);
}
diff --git a/src/hotspot/cpu/ppc/bytes_ppc.hpp b/src/hotspot/cpu/ppc/bytes_ppc.hpp
deleted file mode 100644
index d6076a9c5b2e..000000000000
--- a/src/hotspot/cpu/ppc/bytes_ppc.hpp
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2022 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_PPC_BYTES_PPC_HPP
-#define CPU_PPC_BYTES_PPC_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/byteswap.hpp"
-
-class Bytes: AllStatic {
- public:
- // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering
- // PowerPC needs to check for alignment.
-
- // Can I count on address always being a pointer to an unsigned char? Yes.
-
-#if defined(VM_LITTLE_ENDIAN)
-
- static inline u2 get_native_u2(address p) {
- return (intptr_t(p) & 1) == 0
- ? *(u2*)p
- : ( u2(p[1]) << 8 )
- | ( u2(p[0]) );
- }
-
- static inline u4 get_native_u4(address p) {
- switch (intptr_t(p) & 3) {
- case 0: return *(u4*)p;
-
- case 2: return ( u4( ((u2*)p)[1] ) << 16 )
- | ( u4( ((u2*)p)[0] ) );
-
- default: return ( u4(p[3]) << 24 )
- | ( u4(p[2]) << 16 )
- | ( u4(p[1]) << 8 )
- | u4(p[0]);
- }
- }
-
- static inline u8 get_native_u8(address p) {
- switch (intptr_t(p) & 7) {
- case 0: return *(u8*)p;
-
- case 4: return ( u8( ((u4*)p)[1] ) << 32 )
- | ( u8( ((u4*)p)[0] ) );
-
- case 2: return ( u8( ((u2*)p)[3] ) << 48 )
- | ( u8( ((u2*)p)[2] ) << 32 )
- | ( u8( ((u2*)p)[1] ) << 16 )
- | ( u8( ((u2*)p)[0] ) );
-
- default: return ( u8(p[7]) << 56 )
- | ( u8(p[6]) << 48 )
- | ( u8(p[5]) << 40 )
- | ( u8(p[4]) << 32 )
- | ( u8(p[3]) << 24 )
- | ( u8(p[2]) << 16 )
- | ( u8(p[1]) << 8 )
- | u8(p[0]);
- }
- }
-
-
-
- static inline void put_native_u2(address p, u2 x) {
- if ( (intptr_t(p) & 1) == 0 ) *(u2*)p = x;
- else {
- p[1] = x >> 8;
- p[0] = x;
- }
- }
-
- static inline void put_native_u4(address p, u4 x) {
- switch ( intptr_t(p) & 3 ) {
- case 0: *(u4*)p = x;
- break;
-
- case 2: ((u2*)p)[1] = x >> 16;
- ((u2*)p)[0] = x;
- break;
-
- default: ((u1*)p)[3] = x >> 24;
- ((u1*)p)[2] = x >> 16;
- ((u1*)p)[1] = x >> 8;
- ((u1*)p)[0] = x;
- break;
- }
- }
-
- static inline void put_native_u8(address p, u8 x) {
- switch ( intptr_t(p) & 7 ) {
- case 0: *(u8*)p = x;
- break;
-
- case 4: ((u4*)p)[1] = x >> 32;
- ((u4*)p)[0] = x;
- break;
-
- case 2: ((u2*)p)[3] = x >> 48;
- ((u2*)p)[2] = x >> 32;
- ((u2*)p)[1] = x >> 16;
- ((u2*)p)[0] = x;
- break;
-
- default: ((u1*)p)[7] = x >> 56;
- ((u1*)p)[6] = x >> 48;
- ((u1*)p)[5] = x >> 40;
- ((u1*)p)[4] = x >> 32;
- ((u1*)p)[3] = x >> 24;
- ((u1*)p)[2] = x >> 16;
- ((u1*)p)[1] = x >> 8;
- ((u1*)p)[0] = x;
- }
- }
-
- // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering)
- // (no byte-order reversal is needed since Power CPUs are big-endian oriented).
- static inline u2 get_Java_u2(address p) { return byteswap(get_native_u2(p)); }
- static inline u4 get_Java_u4(address p) { return byteswap(get_native_u4(p)); }
- static inline u8 get_Java_u8(address p) { return byteswap(get_native_u8(p)); }
-
- static inline void put_Java_u2(address p, u2 x) { put_native_u2(p, byteswap(x)); }
- static inline void put_Java_u4(address p, u4 x) { put_native_u4(p, byteswap(x)); }
- static inline void put_Java_u8(address p, u8 x) { put_native_u8(p, byteswap(x)); }
-
-#else // !defined(VM_LITTLE_ENDIAN)
-
- static inline u2 get_native_u2(address p) {
- return (intptr_t(p) & 1) == 0
- ? *(u2*)p
- : ( u2(p[0]) << 8 )
- | ( u2(p[1]) );
- }
-
- static inline u4 get_native_u4(address p) {
- switch (intptr_t(p) & 3) {
- case 0: return *(u4*)p;
-
- case 2: return ( u4( ((u2*)p)[0] ) << 16 )
- | ( u4( ((u2*)p)[1] ) );
-
- default: return ( u4(p[0]) << 24 )
- | ( u4(p[1]) << 16 )
- | ( u4(p[2]) << 8 )
- | u4(p[3]);
- }
- }
-
- static inline u8 get_native_u8(address p) {
- switch (intptr_t(p) & 7) {
- case 0: return *(u8*)p;
-
- case 4: return ( u8( ((u4*)p)[0] ) << 32 )
- | ( u8( ((u4*)p)[1] ) );
-
- case 2: return ( u8( ((u2*)p)[0] ) << 48 )
- | ( u8( ((u2*)p)[1] ) << 32 )
- | ( u8( ((u2*)p)[2] ) << 16 )
- | ( u8( ((u2*)p)[3] ) );
-
- default: return ( u8(p[0]) << 56 )
- | ( u8(p[1]) << 48 )
- | ( u8(p[2]) << 40 )
- | ( u8(p[3]) << 32 )
- | ( u8(p[4]) << 24 )
- | ( u8(p[5]) << 16 )
- | ( u8(p[6]) << 8 )
- | u8(p[7]);
- }
- }
-
-
-
- static inline void put_native_u2(address p, u2 x) {
- if ( (intptr_t(p) & 1) == 0 ) { *(u2*)p = x; }
- else {
- p[0] = x >> 8;
- p[1] = x;
- }
- }
-
- static inline void put_native_u4(address p, u4 x) {
- switch ( intptr_t(p) & 3 ) {
- case 0: *(u4*)p = x;
- break;
-
- case 2: ((u2*)p)[0] = x >> 16;
- ((u2*)p)[1] = x;
- break;
-
- default: ((u1*)p)[0] = x >> 24;
- ((u1*)p)[1] = x >> 16;
- ((u1*)p)[2] = x >> 8;
- ((u1*)p)[3] = x;
- break;
- }
- }
-
- static inline void put_native_u8(address p, u8 x) {
- switch ( intptr_t(p) & 7 ) {
- case 0: *(u8*)p = x;
- break;
-
- case 4: ((u4*)p)[0] = x >> 32;
- ((u4*)p)[1] = x;
- break;
-
- case 2: ((u2*)p)[0] = x >> 48;
- ((u2*)p)[1] = x >> 32;
- ((u2*)p)[2] = x >> 16;
- ((u2*)p)[3] = x;
- break;
-
- default: ((u1*)p)[0] = x >> 56;
- ((u1*)p)[1] = x >> 48;
- ((u1*)p)[2] = x >> 40;
- ((u1*)p)[3] = x >> 32;
- ((u1*)p)[4] = x >> 24;
- ((u1*)p)[5] = x >> 16;
- ((u1*)p)[6] = x >> 8;
- ((u1*)p)[7] = x;
- }
- }
-
- // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering)
- // (no byte-order reversal is needed since Power CPUs are big-endian oriented).
- static inline u2 get_Java_u2(address p) { return get_native_u2(p); }
- static inline u4 get_Java_u4(address p) { return get_native_u4(p); }
- static inline u8 get_Java_u8(address p) { return get_native_u8(p); }
-
- static inline void put_Java_u2(address p, u2 x) { put_native_u2(p, x); }
- static inline void put_Java_u4(address p, u4 x) { put_native_u4(p, x); }
- static inline void put_Java_u8(address p, u8 x) { put_native_u8(p, x); }
-
-#endif // VM_LITTLE_ENDIAN
-};
-
-#endif // CPU_PPC_BYTES_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
index f86b0a9d4fc9..74a8e17dd700 100644
--- a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
@@ -155,6 +155,7 @@ LoadFlattenedArrayStub::LoadFlattenedArrayStub(LIR_Opr array, LIR_Opr index, LIR
void LoadFlattenedArrayStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
+ __ extsw(_index->as_register(), _index->as_register()); // see CCallingConventionRequiresIntsAsLongs
// Pass arguments on stack.
__ std(_array->as_register(), -16, R1_SP);
__ std(_index->as_register(), -8, R1_SP);
@@ -182,6 +183,7 @@ StoreFlattenedArrayStub::StoreFlattenedArrayStub(LIR_Opr array, LIR_Opr index, L
void StoreFlattenedArrayStub::emit_code(LIR_Assembler* ce) {
__ bind(_entry);
+ __ extsw(_index->as_register(), _index->as_register()); // see CCallingConventionRequiresIntsAsLongs
// Pass arguments on stack.
__ std(_array->as_register(), -24, R1_SP);
__ std(_index->as_register(), -16, R1_SP);
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
index d6051da562a3..cd9bb80d9507 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
@@ -3127,13 +3127,8 @@ void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
// We are storing into an array that *may* be null-free (the declared type is
// Object[], abstract[], interface[] or VT.ref[]).
- Label test_mark_word;
Register tmp = op->tmp()->as_register();
__ ld(tmp, oopDesc::mark_offset_in_bytes(), op->array()->as_register());
- __ andi_(R0, tmp, markWord::unlocked_value);
- __ bne(CR0, test_mark_word);
- __ load_prototype_header(tmp, op->array()->as_register());
- __ bind(test_mark_word);
__ andi(R0, tmp, markWord::null_free_array_bit_in_place);
__ cmpwi(BOOL_RESULT, R0, 0);
}
diff --git a/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
index d550c33b1122..909d1e585825 100644
--- a/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
+++ b/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2020, 2026 SAP SE. All rights reserved.
* Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -297,7 +297,7 @@ void DowncallLinker::StubGenerator::generate() {
Label L_after_reguard;
if (_needs_transition) {
- __ li(tmp, _thread_in_vm);
+ __ li(tmp, _thread_in_Java);
__ release();
__ stw(tmp, in_bytes(JavaThread::thread_state_offset()), R16_thread);
if (!UseSystemMemoryBarrier) {
@@ -311,11 +311,6 @@ void DowncallLinker::StubGenerator::generate() {
__ bne(CR0, L_safepoint_poll_slow_path);
__ bind(L_after_safepoint_poll);
- // change thread state
- __ li(tmp, _thread_in_Java);
- __ lwsync(); // Acquire safepoint and suspend state, release thread state.
- __ stw(tmp, in_bytes(JavaThread::thread_state_offset()), R16_thread);
-
__ block_comment("reguard stack check");
__ lwz(tmp, in_bytes(JavaThread::stack_guard_state_offset()), R16_thread);
__ cmpwi(CR0, tmp, StackOverflow::stack_guard_yellow_reserved_disabled);
@@ -340,7 +335,7 @@ void DowncallLinker::StubGenerator::generate() {
out_reg_spiller.generate_spill(_masm, out_spill_offset);
}
- __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, JavaThread::check_special_condition_for_native_trans), R0);
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, SharedRuntime::check_special_condition_for_native_trans), R0);
__ mr(R3_ARG1, R16_thread);
__ call_c(call_target_address);
@@ -374,5 +369,5 @@ void DowncallLinker::StubGenerator::generate() {
//////////////////////////////////////////////////////////////////////////////
- __ flush();
+ // Code will be copied. No ICache sync required.
}
diff --git a/src/hotspot/cpu/ppc/interpreterRT_ppc.cpp b/src/hotspot/cpu/ppc/interpreterRT_ppc.cpp
index dd2503bd54b8..233713b0ecd8 100644
--- a/src/hotspot/cpu/ppc/interpreterRT_ppc.cpp
+++ b/src/hotspot/cpu/ppc/interpreterRT_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -127,7 +127,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
__ load_const(R3_RET, AbstractInterpreter::result_handler(method()->result_type()));
__ blr();
- __ flush();
+ __ invalidate_icache();
}
#undef __
diff --git a/src/hotspot/cpu/ppc/jniFastGetField_ppc.cpp b/src/hotspot/cpu/ppc/jniFastGetField_ppc.cpp
index ac3d2d5dba85..033a25b62f2d 100644
--- a/src/hotspot/cpu/ppc/jniFastGetField_ppc.cpp
+++ b/src/hotspot/cpu/ppc/jniFastGetField_ppc.cpp
@@ -154,7 +154,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
__ load_const_optimized(R12, slow_case_addr, R0);
__ call_c_and_return_to_caller(R12); // tail call
- __ flush();
+ __ invalidate_icache();
return fast_entry;
}
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index e7bf14dad340..c48071291062 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -2734,7 +2734,7 @@ void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register
// Check for monitor (0b10) or locked (0b00).
ld(mark, oopDesc::mark_offset_in_bytes(), obj);
andi_(R0, mark, markWord::lock_mask_in_place);
- cmpldi(CR0, R0, markWord::unlocked_value);
+ cmpldi(CR0, R0, markWord::lock_neutral_value);
bgt(CR0, inflated);
bne(CR0, slow_path);
@@ -2913,7 +2913,7 @@ void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Registe
#ifdef ASSERT
// Check header not unlocked (0b01).
Label not_unlocked;
- andi_(t, mark, markWord::unlocked_value);
+ andi_(t, mark, markWord::lock_neutral_value);
beq(CR0, not_unlocked);
stop("fast_unlock already unlocked");
bind(not_unlocked);
@@ -3374,11 +3374,6 @@ void MacroAssembler::load_metadata(Register dst, Register src) {
}
}
-void MacroAssembler::load_prototype_header(Register dst, Register src) {
- load_klass(dst, src);
- ld(dst, Klass::prototype_header_offset(), dst);
-}
-
void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info) {
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
bs->flat_field_copy(this, decorators, src, dst, inline_layout_info);
@@ -4838,17 +4833,18 @@ void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj,
}
bind(retry);
- STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
+ STATIC_ASSERT(markWord::fast_locked_value == 0); // Or need to change this!
+ STATIC_ASSERT(markWord::lock_neutral_value == 1); // Or need to change this!
if (!is_unlock) {
ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
- xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
+ xori(tmp, tmp, markWord::lock_neutral_value); // flip lock-neutral bit
andi_(R0, tmp, markWord::lock_mask_in_place | markWord::inline_type_bit_in_place);
- bne(CR0, failed); // failed if new header doesn't contain locked_value (which is 0) or belongs to an inline type
+ bne(CR0, failed); // failed if new header doesn't contain fast_locked_value (which is 0) or belongs to an inline type
} else {
ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
andi_(R0, tmp, markWord::lock_mask_in_place);
- bne(CR0, failed); // failed if old header doesn't contain locked_value (which is 0)
- ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
+ bne(CR0, failed); // failed if old header doesn't contain fast_locked_value (which is 0)
+ ori(tmp, tmp, markWord::lock_neutral_value); // set lock-neutral bit
}
stdcx_(tmp, obj);
bne(CR0, retry);
@@ -4900,7 +4896,7 @@ void MacroAssembler::fast_lock(Register box, Register obj, Register t1, Register
// Check header for monitor (0b10) or locked (0b00).
ld(mark, oopDesc::mark_offset_in_bytes(), obj);
- xori(t, mark, markWord::unlocked_value);
+ xori(t, mark, markWord::lock_neutral_value);
andi_(t, t, markWord::lock_mask_in_place);
bne(CR0, slow);
@@ -4974,7 +4970,7 @@ void MacroAssembler::fast_unlock(Register obj, Register t1, Label& slow) {
#ifdef ASSERT
// Check header not unlocked (0b01).
Label not_unlocked;
- andi_(t, mark, markWord::unlocked_value);
+ andi_(t, mark, markWord::lock_neutral_value);
beq(CR0, not_unlocked);
stop("fast_unlock already unlocked");
bind(not_unlocked);
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index 533d03230a4d..839e5dcd69d9 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -846,8 +846,6 @@ class MacroAssembler: public Assembler {
void flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info);
- void load_prototype_header(Register dst, Register src);
-
void inline_layout_info(Register holder_klass, Register index, Register layout_info);
// inline type data payload offsets...
diff --git a/src/hotspot/cpu/ppc/runtime_ppc.cpp b/src/hotspot/cpu/ppc/runtime_ppc.cpp
index ab658e9de58b..a1ba80cad5b7 100644
--- a/src/hotspot/cpu/ppc/runtime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/runtime_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -141,8 +141,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
__ mtlr(R4_ARG2);
__ bctr();
- // Make sure all code is generated.
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Set exception blob.
return ExceptionBlob::create(&buffer, oop_maps,
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 553934953873..ae86f80cff4c 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -2169,7 +2169,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
assert(vep_offset != -1, "Must be set");
#endif
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod* nm = nmethod::new_native_nmethod(method,
compile_id,
masm->code(),
@@ -2198,7 +2198,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
in_sig_bt,
in_regs);
int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
- __ flush();
+ // Code will be copied. No ICache sync required.
int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually
return nmethod::new_native_nmethod(method,
compile_id,
@@ -2617,8 +2617,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
}
// Publish thread state
- // Transition from _thread_in_native to _thread_in_vm.
- __ li(R0, _thread_in_vm);
+ // Transition from _thread_in_native.
+ __ li(R0, _thread_in_Java);
__ release();
// TODO: PPC port assert(4 == JavaThread::sz_thread_state(), "unexpected field size");
__ stw(R0, thread_(thread_state));
@@ -2642,7 +2642,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
Register suspend_flags = r_temp_6;
// No synchronization in progress nor yet synchronized
- // (cmp-br-isync on one path, release (same as acquire on PPC64) on the other path).
__ safepoint_poll(sync, sync_state, true /* at_return */, false /* in_nmethod */);
// Not suspended.
@@ -2656,28 +2655,15 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
// lets us share the oopMap we used when we went native rather than create
// a distinct one for this pc.
__ bind(sync);
- __ isync();
address entry_point =
- CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans);
+ CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans);
save_native_result(masm, ret_type, workspace_slot_offset);
__ call_VM_leaf(entry_point, R16_thread);
restore_native_result(masm, ret_type, workspace_slot_offset);
__ bind(no_block);
- // Publish thread state.
- // --------------------------------------------------------------------------
-
- // Thread state is _thread_in_vm. Any safepoint blocking has
- // already happened so we can now change state to _thread_in_Java.
-
- // Transition from _thread_in_vm to _thread_in_Java.
- __ li(R0, _thread_in_Java);
- __ lwsync(); // Acquire safepoint and suspend state, release thread state.
- // TODO: PPC port assert(4 == JavaThread::sz_thread_state(), "unexpected field size");
- __ stw(R0, thread_(thread_state));
-
// Check preemption for Object.wait()
if (method->is_object_wait0()) {
Label not_preempted;
@@ -2836,7 +2822,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
// Done.
// --------------------------------------------------------------------------
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod *nm = nmethod::new_native_nmethod(method,
compile_id,
@@ -3203,8 +3189,7 @@ void SharedRuntime::generate_deopt_blob() {
__ unimplemented("deopt blob needed only with compiler");
#endif
- // Make sure all code is generated
- __ flush();
+ // Code will be copied. No ICache sync required.
_deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset,
reexecute_offset, first_frame_size_in_bytes / wordSize);
@@ -3341,7 +3326,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
// Return to the interpreter entry point.
__ blr();
- masm->flush();
+ // Code will be copied. No ICache sync required.
return UncommonTrapBlob::create(&buffer, oop_maps, frame_size_in_bytes/wordSize);
}
@@ -3447,8 +3432,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
__ blr();
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Fill-out other meta info
// CodeBlob frame size is in words.
@@ -3534,9 +3518,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
__ std(R11_scratch1, in_bytes(JavaThread::vm_result_oop_offset()), R16_thread);
__ b64_patchable(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
- // -------------
- // Make sure all code is generated.
- masm->flush();
+ // Code will be copied. No ICache sync required.
// return the blob
// frame_size_words or bytes??
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index c1a6b54df0bf..2d51119bcc9f 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -580,7 +580,8 @@ class StubGenerator: public StubCodeGenerator {
//
//
address generate_ghash_processBlocks() {
- StubCodeMark mark(this, "StubRoutines", "ghash");
+ StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
+ StubCodeMark mark(this, stub_id);
address start = __ function_entry();
// Registers for parameters
@@ -2781,10 +2782,8 @@ class StubGenerator: public StubCodeGenerator {
Register to = R4_ARG2; // destination array address
Register key = R5_ARG3; // round key array
- Register keylen = R8;
- Register temp = R9;
- Register keypos = R10;
- Register fifteen = R12;
+ Register keylen = R6;
+ Register tmp = R7;
VectorRegister vRet = VR0;
@@ -2793,68 +2792,27 @@ class StubGenerator: public StubCodeGenerator {
VectorRegister vKey3 = VR3;
VectorRegister vKey4 = VR4;
- VectorRegister fromPerm = VR5;
- VectorRegister keyPerm = VR6;
- VectorRegister toPerm = VR7;
- VectorRegister fSplt = VR8;
-
- VectorRegister vTmp1 = VR9;
- VectorRegister vTmp2 = VR10;
- VectorRegister vTmp3 = VR11;
- VectorRegister vTmp4 = VR12;
+ VectorRegister vp = VR6; // permute vector for byte vector accesses on P8 LE
- __ li (fifteen, 15);
+ __ compute_vp_for_byte_vector_unaligned(vp, /*temp*/ vRet);
// load unaligned from[0-15] to vRet
- __ lvx (vRet, from);
- __ lvx (vTmp1, fifteen, from);
- __ lvsl (fromPerm, from);
-#ifdef VM_LITTLE_ENDIAN
- __ vspltisb (fSplt, 0x0f);
- __ vxor (fromPerm, fromPerm, fSplt);
-#endif
- __ vperm (vRet, vRet, vTmp1, fromPerm);
+ __ load_byte_vector_unaligned(vRet, 0, from, tmp, vp);
+
+ // load the 1st round key to vKey1
+ __ load_word_vector_unaligned(vKey1, 0, key, tmp);
// load keylen (44 or 52 or 60)
__ lwz (keylen, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT), key);
- // to load keys
- __ load_perm (keyPerm, key);
-#ifdef VM_LITTLE_ENDIAN
- __ vspltisb (vTmp2, -16);
- __ vrld (keyPerm, keyPerm, vTmp2);
- __ vrld (keyPerm, keyPerm, vTmp2);
- __ vsldoi (keyPerm, keyPerm, keyPerm, 8);
-#endif
-
- // load the 1st round key to vTmp1
- __ lvx (vTmp1, key);
- __ li (keypos, 16);
- __ lvx (vKey1, keypos, key);
- __ vec_perm (vTmp1, vKey1, keyPerm);
-
// 1st round
- __ vxor (vRet, vRet, vTmp1);
-
- // load the 2nd round key to vKey1
- __ li (keypos, 32);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vKey2, keyPerm);
-
- // load the 3rd round key to vKey2
- __ li (keypos, 48);
- __ lvx (vKey3, keypos, key);
- __ vec_perm (vKey2, vKey3, keyPerm);
-
- // load the 4th round key to vKey3
- __ li (keypos, 64);
- __ lvx (vKey4, keypos, key);
- __ vec_perm (vKey3, vKey4, keyPerm);
+ __ vxor (vRet, vRet, vKey1);
- // load the 5th round key to vKey4
- __ li (keypos, 80);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey4, vTmp1, keyPerm);
+ // load the 2nd - 5th round key to vKey1 - vKey4
+ __ load_word_vector_unaligned(vKey1, 16, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 32, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 48, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 64, key, tmp);
// 2nd - 5th rounds
__ vcipher (vRet, vRet, vKey1);
@@ -2862,25 +2820,11 @@ class StubGenerator: public StubCodeGenerator {
__ vcipher (vRet, vRet, vKey3);
__ vcipher (vRet, vRet, vKey4);
- // load the 6th round key to vKey1
- __ li (keypos, 96);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vTmp1, vKey2, keyPerm);
-
- // load the 7th round key to vKey2
- __ li (keypos, 112);
- __ lvx (vKey3, keypos, key);
- __ vec_perm (vKey2, vKey3, keyPerm);
-
- // load the 8th round key to vKey3
- __ li (keypos, 128);
- __ lvx (vKey4, keypos, key);
- __ vec_perm (vKey3, vKey4, keyPerm);
-
- // load the 9th round key to vKey4
- __ li (keypos, 144);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey4, vTmp1, keyPerm);
+ // load the 6th - 9th round key to vKey1 - vKey4
+ __ load_word_vector_unaligned(vKey1, 80, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 96, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 112, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 128, key, tmp);
// 6th - 9th rounds
__ vcipher (vRet, vRet, vKey1);
@@ -2888,15 +2832,9 @@ class StubGenerator: public StubCodeGenerator {
__ vcipher (vRet, vRet, vKey3);
__ vcipher (vRet, vRet, vKey4);
- // load the 10th round key to vKey1
- __ li (keypos, 160);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vTmp1, vKey2, keyPerm);
-
- // load the 11th round key to vKey2
- __ li (keypos, 176);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey2, vTmp1, keyPerm);
+ // load the 10th - 11th round key to vKey1 - vKey2
+ __ load_word_vector_unaligned(vKey1, 144, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 160, key, tmp);
// if all round keys are loaded, skip next 4 rounds
__ cmpwi (CR0, keylen, 44);
@@ -2906,15 +2844,9 @@ class StubGenerator: public StubCodeGenerator {
__ vcipher (vRet, vRet, vKey1);
__ vcipher (vRet, vRet, vKey2);
- // load the 12th round key to vKey1
- __ li (keypos, 192);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vTmp1, vKey2, keyPerm);
-
- // load the 13th round key to vKey2
- __ li (keypos, 208);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey2, vTmp1, keyPerm);
+ // load the 12th - 13th round key to vKey1 - vKey2
+ __ load_word_vector_unaligned(vKey1, 176, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 192, key, tmp);
// if all round keys are loaded, skip next 2 rounds
__ cmpwi (CR0, keylen, 52);
@@ -2929,15 +2861,9 @@ class StubGenerator: public StubCodeGenerator {
__ vcipher (vRet, vRet, vKey1);
__ vcipher (vRet, vRet, vKey2);
- // load the 14th round key to vKey1
- __ li (keypos, 224);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vTmp1, vKey2, keyPerm);
-
- // load the 15th round key to vKey2
- __ li (keypos, 240);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey2, vTmp1, keyPerm);
+ // load the 14th - 15th round key to vKey1 - vKey2
+ __ load_word_vector_unaligned(vKey1, 208, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 224, key, tmp);
__ bind(L_doLast);
@@ -2945,23 +2871,8 @@ class StubGenerator: public StubCodeGenerator {
__ vcipher (vRet, vRet, vKey1);
__ vcipherlast (vRet, vRet, vKey2);
-#ifdef VM_LITTLE_ENDIAN
- // toPerm = 0x0F0E0D0C0B0A09080706050403020100
- __ lvsl (toPerm, keypos); // keypos is a multiple of 16
- __ vxor (toPerm, toPerm, fSplt);
-
- // Swap Bytes
- __ vperm (vRet, vRet, vRet, toPerm);
-#endif
-
// store result (unaligned)
- // Note: We can't use a read-modify-write sequence which touches additional Bytes.
- Register lo = temp, hi = fifteen; // Reuse
- __ vsldoi (vTmp1, vRet, vRet, 8);
- __ mfvrd (hi, vRet);
- __ mfvrd (lo, vTmp1);
- __ std (hi, 0 LITTLE_ENDIAN_ONLY(+ 8), to);
- __ std (lo, 0 BIG_ENDIAN_ONLY(+ 8), to);
+ __ store_byte_vector_unaligned(vRet, 0, to, tmp, vp);
__ blr();
@@ -2989,10 +2900,8 @@ class StubGenerator: public StubCodeGenerator {
Register to = R4_ARG2; // destination array address
Register key = R5_ARG3; // round key array
- Register keylen = R8;
- Register temp = R9;
- Register keypos = R10;
- Register fifteen = R12;
+ Register keylen = R6;
+ Register tmp = R7;
VectorRegister vRet = VR0;
@@ -3002,41 +2911,16 @@ class StubGenerator: public StubCodeGenerator {
VectorRegister vKey4 = VR4;
VectorRegister vKey5 = VR5;
- VectorRegister fromPerm = VR6;
- VectorRegister keyPerm = VR7;
- VectorRegister toPerm = VR8;
- VectorRegister fSplt = VR9;
-
- VectorRegister vTmp1 = VR10;
- VectorRegister vTmp2 = VR11;
- VectorRegister vTmp3 = VR12;
- VectorRegister vTmp4 = VR13;
+ VectorRegister vp = VR6; // permute vector for byte vector accesses on P8 LE
- __ li (fifteen, 15);
+ __ compute_vp_for_byte_vector_unaligned(vp, /*temp*/ vRet);
// load unaligned from[0-15] to vRet
- __ lvx (vRet, from);
- __ lvx (vTmp1, fifteen, from);
- __ lvsl (fromPerm, from);
-#ifdef VM_LITTLE_ENDIAN
- __ vspltisb (fSplt, 0x0f);
- __ vxor (fromPerm, fromPerm, fSplt);
-#endif
- __ vperm (vRet, vRet, vTmp1, fromPerm); // align [and byte swap in LE]
+ __ load_byte_vector_unaligned(vRet, 0, from, tmp, vp);
// load keylen (44 or 52 or 60)
__ lwz (keylen, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT), key);
- // to load keys
- __ load_perm (keyPerm, key);
-#ifdef VM_LITTLE_ENDIAN
- __ vxor (vTmp2, vTmp2, vTmp2);
- __ vspltisb (vTmp2, -16);
- __ vrld (keyPerm, keyPerm, vTmp2);
- __ vrld (keyPerm, keyPerm, vTmp2);
- __ vsldoi (keyPerm, keyPerm, keyPerm, 8);
-#endif
-
__ cmpwi (CR0, keylen, 44);
__ beq (CR0, L_do44);
@@ -3048,32 +2932,12 @@ class StubGenerator: public StubCodeGenerator {
__ bne (CR0, L_error);
#endif
- // load the 15th round key to vKey1
- __ li (keypos, 240);
- __ lvx (vKey1, keypos, key);
- __ li (keypos, 224);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vKey2, vKey1, keyPerm);
-
- // load the 14th round key to vKey2
- __ li (keypos, 208);
- __ lvx (vKey3, keypos, key);
- __ vec_perm (vKey2, vKey3, vKey2, keyPerm);
-
- // load the 13th round key to vKey3
- __ li (keypos, 192);
- __ lvx (vKey4, keypos, key);
- __ vec_perm (vKey3, vKey4, vKey3, keyPerm);
-
- // load the 12th round key to vKey4
- __ li (keypos, 176);
- __ lvx (vKey5, keypos, key);
- __ vec_perm (vKey4, vKey5, vKey4, keyPerm);
-
- // load the 11th round key to vKey5
- __ li (keypos, 160);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey5, vTmp1, vKey5, keyPerm);
+ // load the 15th - 11th round key to vKey1 - vKey5
+ __ load_word_vector_unaligned(vKey1, 224, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 208, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 192, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 176, key, tmp);
+ __ load_word_vector_unaligned(vKey5, 160, key, tmp);
// 1st - 5th rounds
__ vxor (vRet, vRet, vKey1);
@@ -3087,22 +2951,10 @@ class StubGenerator: public StubCodeGenerator {
__ align(32);
__ bind (L_do52);
- // load the 13th round key to vKey1
- __ li (keypos, 208);
- __ lvx (vKey1, keypos, key);
- __ li (keypos, 192);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vKey2, vKey1, keyPerm);
-
- // load the 12th round key to vKey2
- __ li (keypos, 176);
- __ lvx (vKey3, keypos, key);
- __ vec_perm (vKey2, vKey3, vKey2, keyPerm);
-
- // load the 11th round key to vKey3
- __ li (keypos, 160);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey3, vTmp1, vKey3, keyPerm);
+ // load the 13th - 11th round key to vKey1 - vKey3
+ __ load_word_vector_unaligned(vKey1, 192, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 176, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 160, key, tmp);
// 1st - 3rd rounds
__ vxor (vRet, vRet, vKey1);
@@ -3115,41 +2967,19 @@ class StubGenerator: public StubCodeGenerator {
__ bind (L_do44);
// load the 11th round key to vKey1
- __ li (keypos, 176);
- __ lvx (vKey1, keypos, key);
- __ li (keypos, 160);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey1, vTmp1, vKey1, keyPerm);
+ __ load_word_vector_unaligned(vKey1, 160, key, tmp);
// 1st round
__ vxor (vRet, vRet, vKey1);
__ bind (L_doLast);
- // load the 10th round key to vKey1
- __ li (keypos, 144);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vKey2, vTmp1, keyPerm);
-
- // load the 9th round key to vKey2
- __ li (keypos, 128);
- __ lvx (vKey3, keypos, key);
- __ vec_perm (vKey2, vKey3, vKey2, keyPerm);
-
- // load the 8th round key to vKey3
- __ li (keypos, 112);
- __ lvx (vKey4, keypos, key);
- __ vec_perm (vKey3, vKey4, vKey3, keyPerm);
-
- // load the 7th round key to vKey4
- __ li (keypos, 96);
- __ lvx (vKey5, keypos, key);
- __ vec_perm (vKey4, vKey5, vKey4, keyPerm);
-
- // load the 6th round key to vKey5
- __ li (keypos, 80);
- __ lvx (vTmp1, keypos, key);
- __ vec_perm (vKey5, vTmp1, vKey5, keyPerm);
+ // load the 10th - 6th round key to vKey1 - vKey5
+ __ load_word_vector_unaligned(vKey1, 144, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 128, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 112, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 96, key, tmp);
+ __ load_word_vector_unaligned(vKey5, 80, key, tmp);
// last 10th - 6th rounds
__ vncipher (vRet, vRet, vKey1);
@@ -3158,29 +2988,12 @@ class StubGenerator: public StubCodeGenerator {
__ vncipher (vRet, vRet, vKey4);
__ vncipher (vRet, vRet, vKey5);
- // load the 5th round key to vKey1
- __ li (keypos, 64);
- __ lvx (vKey2, keypos, key);
- __ vec_perm (vKey1, vKey2, vTmp1, keyPerm);
-
- // load the 4th round key to vKey2
- __ li (keypos, 48);
- __ lvx (vKey3, keypos, key);
- __ vec_perm (vKey2, vKey3, vKey2, keyPerm);
-
- // load the 3rd round key to vKey3
- __ li (keypos, 32);
- __ lvx (vKey4, keypos, key);
- __ vec_perm (vKey3, vKey4, vKey3, keyPerm);
-
- // load the 2nd round key to vKey4
- __ li (keypos, 16);
- __ lvx (vKey5, keypos, key);
- __ vec_perm (vKey4, vKey5, vKey4, keyPerm);
-
- // load the 1st round key to vKey5
- __ lvx (vTmp1, key);
- __ vec_perm (vKey5, vTmp1, vKey5, keyPerm);
+ // load the 5th - 1st round key to vKey1 - vKey5
+ __ load_word_vector_unaligned(vKey1, 64, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 48, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 32, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 16, key, tmp);
+ __ load_word_vector_unaligned(vKey5, 0, key, tmp);
// last 5th - 1th rounds
__ vncipher (vRet, vRet, vKey1);
@@ -3189,23 +3002,8 @@ class StubGenerator: public StubCodeGenerator {
__ vncipher (vRet, vRet, vKey4);
__ vncipherlast (vRet, vRet, vKey5);
-#ifdef VM_LITTLE_ENDIAN
- // toPerm = 0x0F0E0D0C0B0A09080706050403020100
- __ lvsl (toPerm, keypos); // keypos is a multiple of 16
- __ vxor (toPerm, toPerm, fSplt);
-
- // Swap Bytes
- __ vperm (vRet, vRet, vRet, toPerm);
-#endif
-
// store result (unaligned)
- // Note: We can't use a read-modify-write sequence which touches additional Bytes.
- Register lo = temp, hi = fifteen; // Reuse
- __ vsldoi (vTmp1, vRet, vRet, 8);
- __ mfvrd (hi, vRet);
- __ mfvrd (lo, vTmp1);
- __ std (hi, 0 LITTLE_ENDIAN_ONLY(+ 8), to);
- __ std (lo, 0 BIG_ENDIAN_ONLY(+ 8), to);
+ __ store_byte_vector_unaligned(vRet, 0, to, tmp, vp);
__ blr();
@@ -3216,6 +3014,306 @@ class StubGenerator: public StubCodeGenerator {
return start;
}
+ // ==========================================================================
+ // AES helper functions for PPC64
+ //
+ // These emit the AES round instructions.
+ // Each call to these helpers emits a sequence of vcipher/vncipher
+ // instructions.
+ //
+ // ==========================================================================
+ // Emits the AES encrypt round instructions.
+ //
+ // vRet: in/out — the AES state (plaintext in, ciphertext out)
+ // key: register holding pointer to expanded key array
+ // keylen: register holding key length (44/52/60)
+ //
+ void aes_encrypt_rounds(VectorRegister vRet,
+ Register key, Register keylen, Register tmp,
+ VectorRegister vKey1, VectorRegister vKey2,
+ VectorRegister vKey3, VectorRegister vKey4) {
+ Label L_doLast;
+
+ // round 0: AddRoundKey
+ __ load_word_vector_unaligned(vKey1, 0, key, tmp);
+ __ vxor (vRet, vRet, vKey1);
+
+ // rounds 2-5
+ __ load_word_vector_unaligned(vKey1, 16, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 32, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 48, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 64, key, tmp);
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipher (vRet, vRet, vKey2);
+ __ vcipher (vRet, vRet, vKey3);
+ __ vcipher (vRet, vRet, vKey4);
+
+ // rounds 6-9
+ __ load_word_vector_unaligned(vKey1, 80, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 96, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 112, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 128, key, tmp);
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipher (vRet, vRet, vKey2);
+ __ vcipher (vRet, vRet, vKey3);
+ __ vcipher (vRet, vRet, vKey4);
+
+ // rounds 10-11
+ __ load_word_vector_unaligned(vKey1, 144, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 160, key, tmp);
+
+ __ cmpwi (CR0, keylen, 44); // AES-128 -> final rounds
+ __ beq (CR0, L_doLast);
+
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipher (vRet, vRet, vKey2);
+
+ // rounds 12-13
+ __ load_word_vector_unaligned(vKey1, 176, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 192, key, tmp);
+
+ __ cmpwi (CR0, keylen, 52); // AES-192 -> final rounds
+ __ beq (CR0, L_doLast);
+#ifdef ASSERT
+ __ cmpwi (CR0, keylen, 60);
+ __ asm_assert_eq(FILE_AND_LINE ": aes_encrypt_rounds - invalid key length");
+#endif
+
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipher (vRet, vRet, vKey2);
+
+ // rounds 14-15
+ __ load_word_vector_unaligned(vKey1, 208, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 224, key, tmp);
+
+ __ bind(L_doLast);
+ __ vcipher (vRet, vRet, vKey1);
+ __ vcipherlast (vRet, vRet, vKey2);
+ }
+
+
+ // ==========================================================================
+ // Emits the AES decrypt round instructions.
+ //
+ // vRet: in/out — the AES state (ciphertext in, plaintext out)
+ // key: register holding pointer to expanded key array
+ // keylen: register holding key length (44/52/60)
+ //
+ void aes_decrypt_rounds(VectorRegister vRet,
+ Register key, Register keylen, Register tmp,
+ VectorRegister vKey1, VectorRegister vKey2,
+ VectorRegister vKey3, VectorRegister vKey4,
+ VectorRegister vKey5) {
+ Label L_doLast, L_do44, L_do52;
+
+ __ cmpwi (CR0, keylen, 44);
+ __ beq (CR0, L_do44);
+
+ __ cmpwi (CR0, keylen, 52);
+ __ beq (CR0, L_do52);
+
+#ifdef ASSERT
+ __ cmpwi (CR0, keylen, 60);
+ __ asm_assert_eq(FILE_AND_LINE ": aes_decrypt_rounds - invalid key length");
+#endif
+ // ---- AES-256: round keys 15-11 ----
+ __ load_word_vector_unaligned(vKey1, 224, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 208, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 192, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 176, key, tmp);
+ __ load_word_vector_unaligned(vKey5, 160, key, tmp);
+
+ __ vxor (vRet, vRet, vKey1);
+ __ vncipher (vRet, vRet, vKey2);
+ __ vncipher (vRet, vRet, vKey3);
+ __ vncipher (vRet, vRet, vKey4);
+ __ vncipher (vRet, vRet, vKey5);
+ __ b (L_doLast);
+
+ __ align(32);
+ // ---- AES-192: round keys 13-11 ----
+ __ bind (L_do52);
+ __ load_word_vector_unaligned(vKey1, 192, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 176, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 160, key, tmp);
+
+ __ vxor (vRet, vRet, vKey1);
+ __ vncipher (vRet, vRet, vKey2);
+ __ vncipher (vRet, vRet, vKey3);
+ __ b (L_doLast);
+
+ __ align(32);
+ // ---- AES-128: round key 11 ----
+ __ bind (L_do44);
+ __ load_word_vector_unaligned(vKey1, 160, key, tmp);
+ __ vxor (vRet, vRet, vKey1);
+
+ // ---- Common rounds 10-1 ----
+ __ bind (L_doLast);
+ __ load_word_vector_unaligned(vKey1, 144, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 128, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 112, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 96, key, tmp);
+ __ load_word_vector_unaligned(vKey5, 80, key, tmp);
+
+ __ vncipher (vRet, vRet, vKey1);
+ __ vncipher (vRet, vRet, vKey2);
+ __ vncipher (vRet, vRet, vKey3);
+ __ vncipher (vRet, vRet, vKey4);
+ __ vncipher (vRet, vRet, vKey5);
+ __ load_word_vector_unaligned(vKey1, 64, key, tmp);
+ __ load_word_vector_unaligned(vKey2, 48, key, tmp);
+ __ load_word_vector_unaligned(vKey3, 32, key, tmp);
+ __ load_word_vector_unaligned(vKey4, 16, key, tmp);
+ __ load_word_vector_unaligned(vKey5, 0, key, tmp);
+ __ vncipher (vRet, vRet, vKey1);
+ __ vncipher (vRet, vRet, vKey2);
+ __ vncipher (vRet, vRet, vKey3);
+ __ vncipher (vRet, vRet, vKey4);
+ __ vncipherlast (vRet, vRet, vKey5);
+ }
+
+ // ==========================================================================
+ // CBC Encrypt stub — using helper functions
+ // from: R3_ARG1 - source byte array address (plaintext)
+ // to: R4_ARG2 - destination byte array address (ciphertext)
+ // key: R5_ARG3 - round key array
+ // rvec: R6_ARG4 - r vector byte array address (initialization vector)
+ // input_len: R7_ARG5 - length of input in bytes
+ //
+ // Returns:
+ // R3_RET - number of bytes processed
+ //
+ address generate_cipherBlockChaining_encryptAESCrypt() {
+ assert(UseAESIntrinsics, "need AES instructions support");
+ StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
+ StubCodeMark mark(this, stub_id);
+
+ address start = __ function_entry();
+
+ Label L_enc_loop;
+
+ Register from = R3_ARG1;
+ Register to = R4_ARG2;
+ Register key = R5_ARG3;
+ Register rvec = R6_ARG4;
+ Register input_len = R7_ARG5;
+
+ Register keylen = R8;
+ Register tmp = R9;
+ Register len = R10;
+
+ VectorRegister vRet = VR0;
+ VectorRegister vKey1 = VR1;
+ VectorRegister vKey2 = VR2;
+ VectorRegister vKey3 = VR3;
+ VectorRegister vKey4 = VR4;
+ VectorRegister vIn = VR5;
+ VectorRegister vp = VR6; // permute vector for P8 LE byte accesses
+ VectorRegister vTmp = VR7;
+
+ __ mr (len, input_len);
+
+ // vp must be computed once, before any byte vector access. Clobbers R0.
+ __ compute_vp_for_byte_vector_unaligned(vp, /*temp*/ vRet);
+
+ __ load_byte_vector_unaligned(vRet, 0, rvec, tmp, vp);
+
+ __ lwz (keylen, arrayOopDesc::length_offset_in_bytes() -
+ arrayOopDesc::base_offset_in_bytes(T_INT), key);
+
+ __ align(32);
+ __ bind(L_enc_loop);
+ __ load_byte_vector_unaligned(vIn, 0, from, tmp, vp);
+ __ addi (from, from, 16);
+ __ vxor (vRet, vRet, vIn); // CBC XOR
+ aes_encrypt_rounds(vRet, key, keylen, tmp, vKey1, vKey2, vKey3, vKey4);
+ __ store_byte_vector_unaligned(vRet, 0, to, tmp, vp, vTmp);
+ __ addi (to, to, 16);
+ __ addic_ (len, len, -16);
+ __ bne (CR0, L_enc_loop);
+
+ // save the last ciphertext block in rvec; it is the IV for the next call
+ __ store_byte_vector_unaligned(vRet, 0, rvec, tmp, vp, vTmp);
+ __ mr (R3_RET, input_len);
+ __ blr();
+
+ return start;
+ }
+
+ // ==========================================================================
+ // CBC Decrypt stub
+ // Arguments:
+ // R3_ARG1 - from: source byte array address (ciphertext)
+ // R4_ARG2 - to: destination byte array address (plaintext)
+ // R5_ARG3 - key: round key array
+ // R6_ARG4 - rvec: r vector byte array address (in/out), holds the
+ // initialization vector on entry and is updated with
+ // the last ciphertext block on exit
+ // R7_ARG5 - input_len: length of input in bytes, a multiple of 16
+ //
+ // Returns:
+ // R3_RET - number of bytes processed
+ // ==========================================================================
+
+ address generate_cipherBlockChaining_decryptAESCrypt() {
+ assert(UseAESIntrinsics, "need AES instructions support");
+ StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
+ StubCodeMark mark(this, stub_id);
+
+ address start = __ function_entry();
+
+ Label L_dec_loop;
+
+ Register from = R3_ARG1;
+ Register to = R4_ARG2;
+ Register key = R5_ARG3;
+ Register rvec = R6_ARG4;
+ Register input_len = R7_ARG5;
+
+ Register keylen = R8;
+ Register tmp = R9;
+ Register len = R10;
+
+ VectorRegister vRet = VR0;
+ VectorRegister vKey1 = VR1;
+ VectorRegister vKey2 = VR2;
+ VectorRegister vKey3 = VR3;
+ VectorRegister vKey4 = VR4;
+ VectorRegister vKey5 = VR5;
+ VectorRegister vIV = VR6;
+ VectorRegister vSavedCT = VR7;
+ VectorRegister vp = VR8; // permute vector for P8 LE byte accesses
+ VectorRegister vTmp = VR9;
+ __ mr (len, input_len);
+ // vp must be computed before any byte vector access. Clobbers R0.
+ __ compute_vp_for_byte_vector_unaligned(vp, /*temp*/ vRet);
+
+ __ load_byte_vector_unaligned(vIV, 0, rvec, tmp, vp);
+
+ __ lwz (keylen, arrayOopDesc::length_offset_in_bytes() -
+ arrayOopDesc::base_offset_in_bytes(T_INT), key);
+
+ __ align(32);
+ __ bind(L_dec_loop);
+ __ load_byte_vector_unaligned(vRet, 0, from, tmp, vp);
+ __ addi (from, from, 16);
+ __ vor (vSavedCT, vRet, vRet); // AES will destroy vRet
+ aes_decrypt_rounds(vRet, key, keylen, tmp, vKey1, vKey2, vKey3, vKey4, vKey5);
+ __ vxor (vRet, vRet, vIV); // CBC XOR (after decrypt)
+ __ vor (vIV, vSavedCT, vSavedCT); // IV = previous ciphertext
+ __ store_byte_vector_unaligned(vRet, 0, to, tmp, vp, vTmp);
+ __ addi (to, to, 16);
+ __ addic_ (len, len, -16);
+ __ bne (CR0, L_dec_loop);
+
+ __ store_byte_vector_unaligned(vIV, 0, rvec, tmp, vp, vTmp);
+ __ mr (R3_RET, input_len);
+ __ blr();
+
+ return start;
+ }
+
address generate_sha256_implCompress(StubId stub_id) {
assert(UseSHA, "need SHA instructions");
bool multi_block;
@@ -3742,7 +3840,8 @@ class StubGenerator: public StubCodeGenerator {
address generate_floatToFloat16() {
__ align(CodeEntryAlignment);
- StubCodeMark mark(this, "StubRoutines", "floatToFloat16");
+ StubId stub_id = StubId::stubgen_f2hf_id;
+ StubCodeMark mark(this, stub_id);
address start = __ function_entry();
__ f2hf(R3_RET, F1_ARG1, F0);
__ blr();
@@ -3751,7 +3850,8 @@ class StubGenerator: public StubCodeGenerator {
address generate_float16ToFloat() {
__ align(CodeEntryAlignment);
- StubCodeMark mark(this, "StubRoutines", "float16ToFloat");
+ StubId stub_id = StubId::stubgen_hf2f_id;
+ StubCodeMark mark(this, stub_id);
address start = __ function_entry();
__ hf2f(F1_RET, R3_ARG1);
__ blr();
@@ -5092,6 +5192,8 @@ void generate_lookup_secondary_supers_table_stub() {
if (UseAESIntrinsics) {
StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
+ StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
+ StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
}
if (UseSHA256Intrinsics) {
diff --git a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp
index 35042e841e66..69ff7c2a6a65 100644
--- a/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp
@@ -1162,7 +1162,7 @@ address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::M
__ resize_frame_absolute(R21_sender_SP, R11_scratch1, R0);
__ blr();
- __ flush();
+ __ invalidate_icache();
return entry;
}
@@ -1179,7 +1179,7 @@ address TemplateInterpreterGenerator::generate_Float_floatToFloat16_entry() {
__ resize_frame_absolute(R21_sender_SP, R11_scratch1, R0);
__ blr();
- __ flush();
+ __ invalidate_icache();
return entry;
}
@@ -1200,7 +1200,7 @@ address TemplateInterpreterGenerator::generate_Float_float16ToFloat_entry() {
__ resize_frame_absolute(R21_sender_SP, R11_scratch1, R0);
__ blr();
- __ flush();
+ __ invalidate_icache();
return entry;
}
@@ -1487,7 +1487,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// We use release_store_fence to update values like the thread state, where
// we don't want the current thread to continue until all our prior memory
// accesses (including the new thread state) are visible to other threads.
- __ li(R0/*thread_state*/, _thread_in_vm);
+ __ li(R0/*thread_state*/, _thread_in_Java);
__ release();
__ stw(R0/*thread_state*/, thread_(thread_state));
if (!UseSystemMemoryBarrier) {
@@ -1498,10 +1498,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// (a new safepoint can not start since we entered _thread_in_vm).
// We must check here because a current safepoint could be in progress.
- // Acquire isn't strictly necessary here because of the fence, but
- // sync_state is declared to be volatile, so we do it anyway
- // (cmp-br-isync on one path, release (same as acquire on PPC64) on the other path).
-
Label do_safepoint, sync_check_done;
// No synchronization in progress nor yet synchronized.
__ safepoint_poll(do_safepoint, sync_state, true /* at_return */, false /* in_nmethod */);
@@ -1513,13 +1509,12 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
__ beq(CR1, sync_check_done);
__ bind(do_safepoint);
- __ isync();
// Block. We do the call directly and leave the current
// last_Java_frame setup undisturbed. We must save any possible
// native result across the call. No oop is present.
__ mr(R3_ARG1, R16_thread);
- __ call_c(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
+ __ call_c(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans));
__ bind(sync_check_done);
@@ -1540,14 +1535,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// On PPC64, we have stored the result directly after the native call.
//=============================================================================
- // Back in Java
-
- // We use release_store_fence to update values like the thread state, where
- // we don't want the current thread to continue until all our prior memory
- // accesses (including the new thread state) are visible to other threads.
- __ li(R0/*thread_state*/, _thread_in_Java);
- __ lwsync(); // Acquire safepoint and suspend state, release thread state.
- __ stw(R0/*thread_state*/, thread_(thread_state));
if (support_vthread_preemption) {
// Check preemption for Object.wait()
diff --git a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
index ae5410b12dfc..7d0cfeaea094 100644
--- a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
+++ b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2023, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -243,7 +243,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
//////////////////////////////////////////////////////////////////////////////
- _masm->flush();
+ // Code will be copied. No ICache sync required.
#ifndef PRODUCT
stringStream ss;
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.cpp b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
index 8781230d8126..b3b433cea948 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.cpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.cpp
@@ -514,7 +514,7 @@ void VM_Version::determine_features() {
a->blr();
uint32_t *code_end = (uint32_t *)a->pc();
- a->flush();
+ a->invalidate_icache();
_features = VM_Version::unknown_m;
// Print the detection code.
@@ -570,7 +570,7 @@ void VM_Version::config_dscr() {
a->blr();
uint32_t *code_end = (uint32_t *)a->pc();
- a->flush();
+ a->invalidate_icache();
// Print the detection code.
if (PrintAssembly) {
diff --git a/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp b/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
index 73a1cbe090f8..b34f60cdec95 100644
--- a/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -124,7 +124,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
__ mtctr(R12_scratch2);
__ bctr();
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
return s;
@@ -224,7 +224,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
__ mtctr(R11_scratch1);
__ bctr();
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
return s;
diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp
index b657c1f108dd..a689107493b1 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp
@@ -514,20 +514,138 @@ class Assembler : public AbstractAssembler {
rdy = 0b111, // in instruction's rm field, selects dynamic rounding mode.In Rounding Mode register, Invalid.
};
+ // Efficient reading and writing of unaligned data in platform-specific byte ordering
+ // RISC-V needs to check for alignment.
+
+ static inline u2 get_native_u2(address p) {
+ if ((intptr_t(p) & 1) == 0) {
+ return *(u2*)p;
+ } else {
+ return ((u2)(p[1]) << 8) |
+ ((u2)(p[0]));
+ }
+ }
+
+ static inline u4 get_native_u4(address p) {
+ switch (intptr_t(p) & 3) {
+ case 0:
+ return *(u4*)p;
+
+ case 2:
+ return ((u4)(((u2*)p)[1]) << 16) |
+ ((u4)(((u2*)p)[0]));
+
+ default:
+ return ((u4)(p[3]) << 24) |
+ ((u4)(p[2]) << 16) |
+ ((u4)(p[1]) << 8) |
+ ((u4)(p[0]));
+ }
+ }
+
+ static inline u8 get_native_u8(address p) {
+ switch (intptr_t(p) & 7) {
+ case 0:
+ return *(u8*)p;
+
+ case 4:
+ return ((u8)(((u4*)p)[1]) << 32) |
+ ((u8)(((u4*)p)[0]));
+
+ case 2:
+ case 6:
+ return ((u8)(((u2*)p)[3]) << 48) |
+ ((u8)(((u2*)p)[2]) << 32) |
+ ((u8)(((u2*)p)[1]) << 16) |
+ ((u8)(((u2*)p)[0]));
+
+ default:
+ return ((u8)(p[7]) << 56) |
+ ((u8)(p[6]) << 48) |
+ ((u8)(p[5]) << 40) |
+ ((u8)(p[4]) << 32) |
+ ((u8)(p[3]) << 24) |
+ ((u8)(p[2]) << 16) |
+ ((u8)(p[1]) << 8) |
+ ((u8)(p[0]));
+ }
+ }
+
+ static inline void put_native_u2(address p, u2 x) {
+ if ((intptr_t(p) & 1) == 0) {
+ *(u2*)p = x;
+ } else {
+ p[1] = x >> 8;
+ p[0] = x;
+ }
+ }
+
+ static inline void put_native_u4(address p, u4 x) {
+ switch (intptr_t(p) & 3) {
+ case 0:
+ *(u4*)p = x;
+ break;
+
+ case 2:
+ ((u2*)p)[1] = x >> 16;
+ ((u2*)p)[0] = x;
+ break;
+
+ default:
+ ((u1*)p)[3] = x >> 24;
+ ((u1*)p)[2] = x >> 16;
+ ((u1*)p)[1] = x >> 8;
+ ((u1*)p)[0] = x;
+ break;
+ }
+ }
+
+ static inline void put_native_u8(address p, u8 x) {
+ switch (intptr_t(p) & 7) {
+ case 0:
+ *(u8*)p = x;
+ break;
+
+ case 4:
+ ((u4*)p)[1] = x >> 32;
+ ((u4*)p)[0] = x;
+ break;
+
+ case 2:
+ case 6:
+ ((u2*)p)[3] = x >> 48;
+ ((u2*)p)[2] = x >> 32;
+ ((u2*)p)[1] = x >> 16;
+ ((u2*)p)[0] = x;
+ break;
+
+ default:
+ ((u1*)p)[7] = x >> 56;
+ ((u1*)p)[6] = x >> 48;
+ ((u1*)p)[5] = x >> 40;
+ ((u1*)p)[4] = x >> 32;
+ ((u1*)p)[3] = x >> 24;
+ ((u1*)p)[2] = x >> 16;
+ ((u1*)p)[1] = x >> 8;
+ ((u1*)p)[0] = x;
+ break;
+ }
+ }
+
// handle unaligned access
static inline uint16_t ld_c_instr(address addr) {
- return Bytes::get_native_u2(addr);
+ return get_native_u2(addr);
}
static inline void sd_c_instr(address addr, uint16_t c_instr) {
- Bytes::put_native_u2(addr, c_instr);
+ put_native_u2(addr, c_instr);
}
// handle unaligned access
static inline uint32_t ld_instr(address addr) {
- return Bytes::get_native_u4(addr);
+ return get_native_u4(addr);
}
static inline void sd_instr(address addr, uint32_t instr) {
- Bytes::put_native_u4(addr, instr);
+ put_native_u4(addr, instr);
}
static inline uint32_t extract(uint32_t val, unsigned msb, unsigned lsb) {
diff --git a/src/hotspot/cpu/riscv/bytes_riscv.hpp b/src/hotspot/cpu/riscv/bytes_riscv.hpp
deleted file mode 100644
index 9495703a03f8..000000000000
--- a/src/hotspot/cpu/riscv/bytes_riscv.hpp
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2016 SAP SE. All rights reserved.
- * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_RISCV_BYTES_RISCV_HPP
-#define CPU_RISCV_BYTES_RISCV_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/byteswap.hpp"
-
-class Bytes: AllStatic {
- public:
- // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering
- // RISCV needs to check for alignment.
-
- static inline u2 get_native_u2(address p) {
- if ((intptr_t(p) & 1) == 0) {
- return *(u2*)p;
- } else {
- return ((u2)(p[1]) << 8) |
- ((u2)(p[0]));
- }
- }
-
- static inline u4 get_native_u4(address p) {
- switch (intptr_t(p) & 3) {
- case 0:
- return *(u4*)p;
-
- case 2:
- return ((u4)(((u2*)p)[1]) << 16) |
- ((u4)(((u2*)p)[0]));
-
- default:
- return ((u4)(p[3]) << 24) |
- ((u4)(p[2]) << 16) |
- ((u4)(p[1]) << 8) |
- ((u4)(p[0]));
- }
- }
-
- static inline u8 get_native_u8(address p) {
- switch (intptr_t(p) & 7) {
- case 0:
- return *(u8*)p;
-
- case 4:
- return ((u8)(((u4*)p)[1]) << 32) |
- ((u8)(((u4*)p)[0]));
-
- case 2:
- case 6:
- return ((u8)(((u2*)p)[3]) << 48) |
- ((u8)(((u2*)p)[2]) << 32) |
- ((u8)(((u2*)p)[1]) << 16) |
- ((u8)(((u2*)p)[0]));
-
- default:
- return ((u8)(p[7]) << 56) |
- ((u8)(p[6]) << 48) |
- ((u8)(p[5]) << 40) |
- ((u8)(p[4]) << 32) |
- ((u8)(p[3]) << 24) |
- ((u8)(p[2]) << 16) |
- ((u8)(p[1]) << 8) |
- ((u8)(p[0]));
- }
- }
-
- static inline void put_native_u2(address p, u2 x) {
- if ((intptr_t(p) & 1) == 0) {
- *(u2*)p = x;
- } else {
- p[1] = x >> 8;
- p[0] = x;
- }
- }
-
- static inline void put_native_u4(address p, u4 x) {
- switch (intptr_t(p) & 3) {
- case 0:
- *(u4*)p = x;
- break;
-
- case 2:
- ((u2*)p)[1] = x >> 16;
- ((u2*)p)[0] = x;
- break;
-
- default:
- ((u1*)p)[3] = x >> 24;
- ((u1*)p)[2] = x >> 16;
- ((u1*)p)[1] = x >> 8;
- ((u1*)p)[0] = x;
- break;
- }
- }
-
- static inline void put_native_u8(address p, u8 x) {
- switch (intptr_t(p) & 7) {
- case 0:
- *(u8*)p = x;
- break;
-
- case 4:
- ((u4*)p)[1] = x >> 32;
- ((u4*)p)[0] = x;
- break;
-
- case 2:
- case 6:
- ((u2*)p)[3] = x >> 48;
- ((u2*)p)[2] = x >> 32;
- ((u2*)p)[1] = x >> 16;
- ((u2*)p)[0] = x;
- break;
-
- default:
- ((u1*)p)[7] = x >> 56;
- ((u1*)p)[6] = x >> 48;
- ((u1*)p)[5] = x >> 40;
- ((u1*)p)[4] = x >> 32;
- ((u1*)p)[3] = x >> 24;
- ((u1*)p)[2] = x >> 16;
- ((u1*)p)[1] = x >> 8;
- ((u1*)p)[0] = x;
- break;
- }
- }
-
-#ifndef VM_LITTLE_ENDIAN
-#error RISC-V is little endian, the preprocessor macro VM_LITTLE_ENDIAN should be defined.
-#endif
-
- // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering)
- static inline u2 get_Java_u2(address p) { return byteswap(get_native_u2(p)); }
- static inline u4 get_Java_u4(address p) { return byteswap(get_native_u4(p)); }
- static inline u8 get_Java_u8(address p) { return byteswap(get_native_u8(p)); }
-
- static inline void put_Java_u2(address p, u2 x) { put_native_u2(p, byteswap(x)); }
- static inline void put_Java_u4(address p, u4 x) { put_native_u4(p, byteswap(x)); }
- static inline void put_Java_u8(address p, u8 x) { put_native_u8(p, byteswap(x)); }
-};
-
-#endif // CPU_RISCV_BYTES_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index 496e26d3c0b5..b2473a9356e5 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -1262,13 +1262,8 @@ void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
// We are storing into an array that *may* be null-free (the declared type is
// Object[], abstract[], interface[] or VT.ref[]).
- Label test_mark_word;
Register tmp = op->tmp()->as_register();
__ ld(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
- __ test_bit(t0, tmp, exact_log2(markWord::unlocked_value));
- __ bnez(t0, test_mark_word);
- __ load_prototype_header(tmp, op->array()->as_register());
- __ bind(test_mark_word);
__ test_bit(tmp, tmp, exact_log2(markWord::null_free_array_bit_in_place));
}
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index c0504ba23da7..c2684982c140 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -123,8 +123,8 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box,
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
// Try to lock. Transition lock-bits 0b01 => 0b00
- ori(tmp1_mark, tmp1_mark, markWord::unlocked_value);
- xori(tmp3_t, tmp1_mark, markWord::unlocked_value);
+ ori(tmp1_mark, tmp1_mark, markWord::lock_neutral_value);
+ xori(tmp3_t, tmp1_mark, markWord::lock_neutral_value);
cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
/*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ tmp3_t);
bne(tmp1_mark, tmp3_t, slow_path);
@@ -295,7 +295,7 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register box,
// Try to unlock. Transition lock bits 0b00 => 0b01
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
- ori(tmp3_t, tmp1_mark, markWord::unlocked_value);
+ ori(tmp3_t, tmp1_mark, markWord::lock_neutral_value);
cmpxchg(/*addr*/ obj, /*expected*/ tmp1_mark, /*new*/ tmp3_t, Assembler::int64,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ tmp3_t);
beq(tmp1_mark, tmp3_t, unlocked);
@@ -408,6 +408,7 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register box,
// StringLatin1.indexOfChar
void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
Register ch, Register result,
+ Register start_index,
bool isL)
{
Register ch1 = t0;
@@ -500,7 +501,7 @@ void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
addi(index, index, 7);
bind(MATCH);
- mv(result, index);
+ add(result, start_index, index);
bind(NOMATCH);
BLOCK_COMMENT("} string_indexof_char_short");
}
@@ -513,39 +514,40 @@ void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
Register tmp3, Register tmp4,
bool isL)
{
- Label CH1_LOOP, HIT, NOMATCH, DONE, DO_LONG;
+ Label CH1_LOOP, HIT, DONE, SHORT;
Register ch1 = t0;
Register orig_cnt = t1;
- Register mask1 = tmp3;
+ Register mask1 = tmp1;
Register mask2 = tmp2;
- Register match_mask = tmp1;
- Register trailing_char = tmp4;
- Register unaligned_elems = tmp4;
+ Register match_mask = tmp3;
+ Register loop_step = tmp4;
+ Register trailing_chars = tmp4;
+ Register unaligned_chars = tmp4;
+ Register start_index = tmp4;
BLOCK_COMMENT("string_indexof_char {");
- beqz(cnt1, NOMATCH);
+ mv(result, -1);
+ beqz(cnt1, DONE);
subi(t0, cnt1, isL ? 32 : 16);
- bgtz(t0, DO_LONG);
- string_indexof_char_short(str1, cnt1, ch, result, isL);
- j(DONE);
+ mv(start_index, zr);
+ blez(t0, SHORT);
- bind(DO_LONG);
mv(orig_cnt, cnt1);
if (AvoidUnalignedAccesses) {
Label ALIGNED;
- andi(unaligned_elems, str1, 0x7);
- beqz(unaligned_elems, ALIGNED);
- sub(unaligned_elems, unaligned_elems, 8);
- neg(unaligned_elems, unaligned_elems);
+ andi(unaligned_chars, str1, 0x7);
+ beqz(unaligned_chars, ALIGNED);
+ sub(unaligned_chars, unaligned_chars, 8);
+ neg(unaligned_chars, unaligned_chars);
if (!isL) {
- srli(unaligned_elems, unaligned_elems, 1);
+ srli(unaligned_chars, unaligned_chars, 1);
}
// do unaligned part per element
- string_indexof_char_short(str1, unaligned_elems, ch, result, isL);
+ string_indexof_char_short(str1, unaligned_chars, ch, result, zr, isL);
bgez(result, DONE);
mv(orig_cnt, cnt1);
- sub(cnt1, cnt1, unaligned_elems);
+ sub(cnt1, cnt1, unaligned_chars);
bind(ALIGNED);
}
@@ -570,33 +572,47 @@ void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff);
mv(mask2, isL ? mask7f7f : mask7fff);
+ mv(loop_step, 8);
+
bind(CH1_LOOP);
ld(ch1, Address(str1));
- addi(str1, str1, 8);
- subi(cnt1, cnt1, 8);
compute_match_mask(ch1, ch, match_mask, mask1, mask2);
bnez(match_mask, HIT);
- bgtz(cnt1, CH1_LOOP);
- j(NOMATCH);
+ addi(str1, str1, 8);
+ subi(cnt1, cnt1, 8);
+ bge(cnt1, loop_step, CH1_LOOP);
+
+ beqz(cnt1, DONE);
+ if (!isL) {
+ srli(cnt1, cnt1, 1);
+ }
+ // Tail (1..7 chars) after the SWAR loop has advanced str1. cnt1 holds the
+ // remaining char count; the number of chars already scanned by the loop is
+ // (orig_cnt - cnt1). string_indexof_char_short returns an index relative to
+ // the current str1, so we pass that prefix as start_index to recover the
+ // real index.
+ // Note: ch was broadcast across all 8 bytes for the SWAR loop above, but the
+ // short helper compares a single element, so restore ch to a single char.
+ isL ? zext(ch, ch, 8) : zext(ch, ch, 16);
+ sub(start_index, orig_cnt, cnt1);
+
+ bind(SHORT);
+ string_indexof_char_short(str1, cnt1, ch, result, start_index, isL);
+ j(DONE);
bind(HIT);
// count bits of trailing zero chars
- ctzc_bits(trailing_char, match_mask, isL, ch1, result);
- srli(trailing_char, trailing_char, 3);
- addi(cnt1, cnt1, 8);
- ble(cnt1, trailing_char, NOMATCH);
+ ctzc_bits(trailing_chars, match_mask, isL, mask1, mask2);
+ srli(trailing_chars, trailing_chars, 3);
+
// match case
if (!isL) {
srli(cnt1, cnt1, 1);
- srli(trailing_char, trailing_char, 1);
+ srli(trailing_chars, trailing_chars, 1);
}
sub(result, orig_cnt, cnt1);
- add(result, result, trailing_char);
- j(DONE);
-
- bind(NOMATCH);
- mv(result, -1);
+ add(result, result, trailing_chars);
bind(DONE);
BLOCK_COMMENT("} string_indexof_char");
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
index db80d048e927..8fea474dacb2 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
@@ -64,6 +64,7 @@
void string_indexof_char_short(Register str1, Register cnt1,
Register ch, Register result,
+ Register start_index,
bool isL);
void string_indexof_char(Register str1, Register cnt1,
diff --git a/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
index b11abb912ee5..e1511f5b7f34 100644
--- a/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
+++ b/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp
@@ -304,12 +304,14 @@ void DowncallLinker::StubGenerator::generate() {
Label L_reguard;
Label L_after_reguard;
if (_needs_transition) {
+ __ block_comment("{ thread native2java");
// Restore cpu control state after JNI call
__ restore_cpu_control_state_after_jni(t0);
- __ block_comment("{ thread native2java");
- __ mv(t0, _thread_in_vm);
- __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
+ // change thread state
+ __ mv(t1, _thread_in_Java);
+ __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
+ __ sw(t1, Address(xthread, JavaThread::thread_state_offset()));
// Force this write out before the read below
if (!UseSystemMemoryBarrier) {
@@ -322,11 +324,6 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_after_safepoint_poll);
- // change thread state
- __ mv(t0, _thread_in_Java);
- __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
- __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
-
__ block_comment("reguard stack check");
__ lbu(t0, Address(xthread, JavaThread::stack_guard_state_offset()));
__ mv(t1, StackOverflow::stack_guard_yellow_reserved_disabled);
@@ -353,7 +350,7 @@ void DowncallLinker::StubGenerator::generate() {
__ mv(c_rarg0, xthread);
assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
- __ rt_call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
+ __ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans));
if (should_save_return_value) {
out_reg_spiller.generate_fill(_masm, out_spill_offset);
@@ -383,5 +380,5 @@ void DowncallLinker::StubGenerator::generate() {
//////////////////////////////////////////////////////////////////////////////
- __ flush();
+ // Code will be copied. No ICache sync required.
}
diff --git a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
index 0078deb76e8c..a922f5337b2c 100644
--- a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
+++ b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
@@ -33,7 +33,7 @@ source_hpp %{
source %{
#include "gc/z/zBarrierSetAssembler.hpp"
-static void z_color(MacroAssembler* masm, const MachNode* node, Register dst, Register src, Register tmp) {
+static void z_color(MacroAssembler* masm, Register dst, Register src, Register tmp) {
assert_different_registers(dst, tmp);
__ relocate(barrier_Relocation::spec(), [&] {
@@ -43,7 +43,7 @@ static void z_color(MacroAssembler* masm, const MachNode* node, Register dst, Re
__ orr(dst, dst, tmp);
}
-static void z_uncolor(MacroAssembler* masm, const MachNode* node, Register ref) {
+static void z_uncolor(MacroAssembler* masm, Register ref) {
__ srli(ref, ref, ZPointerLoadShift);
}
@@ -63,7 +63,7 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
((node->barrier_data() & ZBarrierPhantom) != 0);
if (node->barrier_data() == ZBarrierElided) {
- z_uncolor(masm, node, ref);
+ z_uncolor(masm, ref);
return;
}
@@ -74,14 +74,14 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
__ j(*stub->entry());
__ bind(good);
- z_uncolor(masm, node, ref);
+ z_uncolor(masm, ref);
__ bind(*stub->continuation());
}
static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
if (node->barrier_data() == ZBarrierElided) {
- z_color(masm, node, rnew_zpointer, rnew_zaddress, tmp);
+ z_color(masm, rnew_zpointer, rnew_zaddress, tmp);
} else {
bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
@@ -145,7 +145,7 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
ins_encode %{
guarantee($mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
- z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+ z_color(masm, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
%}
@@ -168,7 +168,7 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
ins_encode %{
guarantee($mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
- z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+ z_color(masm, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
%}
@@ -189,10 +189,10 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
ins_encode %{
guarantee($mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
- z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+ z_color(masm, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
- z_uncolor(masm, this, $res$$Register);
+ z_uncolor(masm, $res$$Register);
%}
ins_pipe(pipe_slow);
@@ -211,10 +211,10 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
ins_encode %{
guarantee($mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
- z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+ z_color(masm, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
- z_uncolor(masm, this, $res$$Register);
+ z_uncolor(masm, $res$$Register);
%}
ins_pipe(pipe_slow);
@@ -232,7 +232,7 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp, rF
ins_encode %{
z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, $tmp$$Register, true /* is_atomic */);
__ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
- z_uncolor(masm, this, $prev$$Register);
+ z_uncolor(masm, $prev$$Register);
%}
ins_pipe(pipe_serial);
@@ -250,7 +250,7 @@ instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp,
ins_encode %{
z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, $tmp$$Register, true /* is_atomic */);
__ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
- z_uncolor(masm, this, $prev$$Register);
+ z_uncolor(masm, $prev$$Register);
%}
ins_pipe(pipe_serial);
%}
diff --git a/src/hotspot/cpu/riscv/interpreterRT_riscv.cpp b/src/hotspot/cpu/riscv/interpreterRT_riscv.cpp
index c8e488d9d691..fc9f224a7675 100644
--- a/src/hotspot/cpu/riscv/interpreterRT_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interpreterRT_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -168,7 +168,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
__ movptr(x10, ExternalAddress(Interpreter::result_handler(method()->result_type())));
__ ret();
- __ flush();
+ __ invalidate_icache();
}
diff --git a/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp b/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp
index 9755cb9ef16c..dcdbe4a6dc18 100644
--- a/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp
+++ b/src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp
@@ -166,7 +166,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
__ leave();
__ ret();
}
- __ flush();
+ __ invalidate_icache();
return fast_entry;
}
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index ec044b6f824d..2f44cf42dc37 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -162,8 +162,7 @@ uint32_t MacroAssembler::get_membar_kind(address addr) {
assert_cond(addr != nullptr);
assert(is_membar(addr), "no membar found");
- uint32_t insn = Bytes::get_native_u4(addr);
-
+ uint32_t insn = Assembler::ld_instr(addr);
uint32_t predecessor = Assembler::extract(insn, 27, 24);
uint32_t successor = Assembler::extract(insn, 23, 20);
@@ -179,7 +178,7 @@ void MacroAssembler::set_membar_kind(address addr, uint32_t order_kind) {
MacroAssembler::membar_mask_to_pred_succ(order_kind, predecessor, successor);
- uint32_t insn = Bytes::get_native_u4(addr);
+ uint32_t insn = Assembler::ld_instr(addr);
address pInsn = (address) &insn;
Assembler::patch(pInsn, 27, 24, predecessor);
Assembler::patch(pInsn, 23, 20, successor);
@@ -3844,11 +3843,17 @@ void MacroAssembler::encode_heap_oop(Register d, Register s) {
mv(d, s);
}
} else {
- Label notNull;
- sub(d, s, xheapbase);
- bgez(d, notNull);
- mv(d, zr);
- bind(notNull);
+ if (UseZicond) {
+ assert_different_registers(s, t0);
+ sub(t0, s, xheapbase);
+ czero_eqz(d, t0, s); // d = s == 0 ? 0 : t0
+ } else {
+ Label notNull;
+ sub(d, s, xheapbase);
+ bgez(d, notNull);
+ mv(d, zr);
+ bind(notNull);
+ }
if (CompressedOops::shift() != 0) {
assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
srli(d, d, CompressedOops::shift());
@@ -3922,11 +3927,6 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
decode_klass_not_null(dst, tmp);
}
-void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
- load_klass(dst, src, tmp);
- ld(dst, Address(dst, Klass::prototype_header_offset()));
-}
-
void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
@@ -4060,11 +4060,18 @@ void MacroAssembler::decode_heap_oop(Register d, Register s) {
slli(d, s, CompressedOops::shift());
}
} else {
- Label done;
- mv(d, s);
- beqz(s, done);
- shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes);
- bind(done);
+ assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
+ if (UseZicond) {
+ assert_different_registers(s, t0);
+ shadd(t0, s, xheapbase, t0, LogMinObjAlignmentInBytes);
+ czero_eqz(d, t0, s); // d = s == 0 ? 0 : t0
+ } else {
+ Label done;
+ mv(d, s);
+ beqz(s, done);
+ shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes);
+ bind(done);
+ }
}
verify_oop_msg(d, "broken oop in decode_heap_oop");
}
@@ -7125,13 +7132,13 @@ void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register tmp1,
// Try to lock. Transition lock-bits 0b01 => 0b00
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la");
- ori(mark, mark, markWord::unlocked_value);
+ ori(mark, mark, markWord::lock_neutral_value);
if (Arguments::is_valhalla_enabled()) {
// Mask inline_type bit such that we go to the slow path if object is an inline type
andi(mark, mark, ~((int) markWord::inline_type_bit_in_place));
}
- xori(t, mark, markWord::unlocked_value);
+ xori(t, mark, markWord::lock_neutral_value);
cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
/*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t);
bne(mark, t, slow, /* is_far */ true);
@@ -7194,7 +7201,7 @@ void MacroAssembler::fast_unlock(Register obj, Register tmp1, Register tmp2, Reg
#ifdef ASSERT
// Check header not unlocked (0b01).
Label not_unlocked;
- test_bit(t, mark, exact_log2(markWord::unlocked_value));
+ test_bit(t, mark, exact_log2(markWord::lock_neutral_value));
beqz(t, not_unlocked);
stop("fast_unlock already unlocked");
bind(not_unlocked);
@@ -7202,7 +7209,7 @@ void MacroAssembler::fast_unlock(Register obj, Register tmp1, Register tmp2, Reg
// Try to unlock. Transition lock bits 0b00 => 0b01
assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
- ori(t, mark, markWord::unlocked_value);
+ ori(t, mark, markWord::lock_neutral_value);
cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64,
/*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ t);
beq(mark, t, unlocked);
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index 9af9fad06d08..be684c5ec908 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -201,7 +201,6 @@ class MacroAssembler: public Assembler {
void access_store_at(BasicType type, DecoratorSet decorators, Address dst,
Register val, Register tmp1, Register tmp2, Register tmp3);
void load_klass(Register dst, Register src, Register tmp = t0);
- void load_prototype_header(Register dst, Register src, Register tmp = t0);
void load_narrow_klass_compact(Register dst, Register src);
void load_narrow_klass(Register dst, Register src);
void store_klass(Register dst, Register src, Register tmp = t0);
@@ -1842,7 +1841,7 @@ class MacroAssembler: public Assembler {
static bool is_pc_relative_at(address branch);
static bool is_membar(address addr) {
- return (Bytes::get_native_u4(addr) & 0x7f) == 0b1111 && extract_funct3(addr) == 0;
+ return (Assembler::ld_instr(addr) & 0x7f) == 0b1111 && extract_funct3(addr) == 0;
}
static uint32_t get_membar_kind(address addr);
static void set_membar_kind(address addr, uint32_t order_kind);
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
index 5d1cac72ade2..6f51395898a3 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
@@ -234,7 +234,7 @@ void NativeMovConstReg::verify() {
intptr_t NativeMovConstReg::data() const {
address addr = MacroAssembler::target_addr_for_insn(instruction_address());
if (maybe_cpool_ref(instruction_address())) {
- return Bytes::get_native_u8(addr);
+ return MacroAssembler::get_native_u8(addr);
} else {
return (intptr_t)addr;
}
@@ -243,7 +243,7 @@ intptr_t NativeMovConstReg::data() const {
void NativeMovConstReg::set_data(intptr_t x) {
if (maybe_cpool_ref(instruction_address())) {
address addr = MacroAssembler::target_addr_for_insn(instruction_address());
- Bytes::put_native_u8(addr, x);
+ MacroAssembler::put_native_u8(addr, x);
} else {
// Store x into the instruction stream.
MacroAssembler::pd_patch_instruction_size(instruction_address(), (address)x);
@@ -259,11 +259,11 @@ void NativeMovConstReg::set_data(intptr_t x) {
while (iter.next()) {
if (iter.type() == relocInfo::oop_type) {
oop* oop_addr = iter.oop_reloc()->oop_addr();
- Bytes::put_native_u8((address)oop_addr, x);
+ MacroAssembler::put_native_u8((address)oop_addr, x);
break;
} else if (iter.type() == relocInfo::metadata_type) {
Metadata** metadata_addr = iter.metadata_reloc()->metadata_addr();
- Bytes::put_native_u8((address)metadata_addr, x);
+ MacroAssembler::put_native_u8((address)metadata_addr, x);
break;
}
}
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
index b28e33759b2f..90f32c9b25db 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
@@ -78,19 +78,19 @@ class NativeInstruction {
protected:
address addr_at(int offset) const { return address(this) + offset; }
- jint int_at(int offset) const { return (jint) Bytes::get_native_u4(addr_at(offset)); }
- juint uint_at(int offset) const { return Bytes::get_native_u4(addr_at(offset)); }
- address ptr_at(int offset) const { return (address) Bytes::get_native_u8(addr_at(offset)); }
- oop oop_at(int offset) const { return cast_to_oop(Bytes::get_native_u8(addr_at(offset))); }
+ jint int_at(int offset) const { return (jint) MacroAssembler::get_native_u4(addr_at(offset)); }
+ juint uint_at(int offset) const { return MacroAssembler::get_native_u4(addr_at(offset)); }
+ address ptr_at(int offset) const { return (address) MacroAssembler::get_native_u8(addr_at(offset)); }
+ oop oop_at(int offset) const { return cast_to_oop(MacroAssembler::get_native_u8(addr_at(offset))); }
- void set_int_at(int offset, jint i) { Bytes::put_native_u4(addr_at(offset), i); }
- void set_uint_at(int offset, jint i) { Bytes::put_native_u4(addr_at(offset), i); }
- void set_ptr_at(int offset, address ptr) { Bytes::put_native_u8(addr_at(offset), (u8)ptr); }
- void set_oop_at(int offset, oop o) { Bytes::put_native_u8(addr_at(offset), cast_from_oop(o)); }
+ void set_int_at(int offset, jint i) { MacroAssembler::put_native_u4(addr_at(offset), i); }
+ void set_uint_at(int offset, juint i) { MacroAssembler::put_native_u4(addr_at(offset), i); }
+ void set_ptr_at(int offset, address ptr) { MacroAssembler::put_native_u8(addr_at(offset), (u8)ptr); }
+ void set_oop_at(int offset, oop o) { MacroAssembler::put_native_u8(addr_at(offset), cast_from_oop(o)); }
- static void set_data64_at(address dest, uint64_t data) { Bytes::put_native_u8(dest, (u8)data); }
- static uint64_t get_data64_at(address src) { return Bytes::get_native_u8(src); }
+ static void set_data64_at(address dest, uint64_t data) { MacroAssembler::put_native_u8(dest, (u8)data); }
+ static uint64_t get_data64_at(address src) { return MacroAssembler::get_native_u8(src); }
public:
inline friend NativeInstruction* nativeInstruction_at(address addr);
diff --git a/src/hotspot/cpu/riscv/relocInfo_riscv.cpp b/src/hotspot/cpu/riscv/relocInfo_riscv.cpp
index ccd8b8919969..09264327516e 100644
--- a/src/hotspot/cpu/riscv/relocInfo_riscv.cpp
+++ b/src/hotspot/cpu/riscv/relocInfo_riscv.cpp
@@ -44,7 +44,7 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
if (MacroAssembler::is_load_pc_relative_at(addr())) {
address constptr = (address)code()->oop_addr_at(reloc->oop_index());
bytes = MacroAssembler::pd_patch_instruction_size(addr(), constptr);
- assert((address)Bytes::get_native_u8(constptr) == x, "error in oop relocation");
+ assert((address)MacroAssembler::get_native_u8(constptr) == x, "error in oop relocation");
} else {
bytes = MacroAssembler::patch_oop(addr(), x);
}
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index 033e4a4222e0..f8b9acd18a90 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -2850,6 +2850,29 @@ operand immIpowerOf2() %{
interface(CONST_INTER);
%}
+// Int immediate: the mask of a bitfield extract, i.e. contiguous low-order
+// ones, whose width is log2i_exact(mask + 1), e.g. 0xfff, 0xffff, 0x7fffffff.
+// Masks that fit into a 12-bit immediate are excluded: a shift followed by
+// andi is already as short as the shift pair used to extract the field.
+operand immI_bitmask() %{
+ predicate(is_power_of_2((juint)n->get_int() + 1) &&
+ !Assembler::is_simm12((int64_t)n->get_int()));
+ match(ConI);
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
+// Long Immediate: low 16-bit mask
+operand immL_16bits()
+%{
+ predicate(n->get_long() == 0xFFFFL);
+ match(ConL);
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
// Long Immediate: low 32-bit mask
operand immL_32bits()
%{
@@ -2860,6 +2883,17 @@ operand immL_32bits()
interface(CONST_INTER);
%}
+// Long immediate: the mask of a bitfield extract, see immI_bitmask above,
+// e.g. 0xfff, 0xffffffff, 0x7fffffffffffffff.
+operand immL_bitmask() %{
+ predicate(is_power_of_2((julong)n->get_long() + 1) &&
+ !Assembler::is_simm12(n->get_long()));
+ match(ConL);
+ op_cost(0);
+ format %{ %}
+ interface(CONST_INTER);
+%}
+
// 64 bit unit decrement
operand immL_M1()
%{
@@ -7254,6 +7288,86 @@ instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
ins_pipe(ialu_reg_shift);
%}
+// Unsigned bitfield extract: (src >>> rshift) & (2^width - 1)
+instruct bitfieldExtractI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask) %{
+ match(Set dst (AndI (URShiftI src rshift) mask));
+ // The field must not extend past the high end of the int.
+ predicate(log2i_exact((juint)n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 0x1f) <= 32);
+
+ ins_cost(ALU_COST * 2);
+ format %{ "slli $dst, $src, 64 - (($rshift & 0x1f) + width($mask))\n\t"
+ "srli $dst, $dst, 64 - width($mask)\t#@bitfieldExtractI" %}
+
+ ins_encode %{
+ int rshift = $rshift$$constant & 0x1f;
+ int width = log2i_exact((juint)$mask$$constant + 1);
+ __ slli(as_Register($dst$$reg), as_Register($src$$reg), 64 - (rshift + width));
+ __ srli(as_Register($dst$$reg), as_Register($dst$$reg), 64 - width);
+ %}
+
+ ins_pipe(ialu_reg_shift);
+%}
+
+// Unsigned bitfield extract from an int with a zero-extending conversion to long.
+instruct bitfieldExtractI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask) %{
+ match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
+ predicate(log2i_exact((juint)n->in(1)->in(2)->get_int() + 1) +
+ (n->in(1)->in(1)->in(2)->get_int() & 0x1f) <= 32);
+
+ ins_cost(ALU_COST * 2);
+ format %{ "slli $dst, $src, 64 - (($rshift & 0x1f) + width($mask))\n\t"
+ "srli $dst, $dst, 64 - width($mask)\t#@bitfieldExtractI2L" %}
+
+ ins_encode %{
+ int rshift = $rshift$$constant & 0x1f;
+ int width = log2i_exact((juint)$mask$$constant + 1);
+ __ slli(as_Register($dst$$reg), as_Register($src$$reg), 64 - (rshift + width));
+ __ srli(as_Register($dst$$reg), as_Register($dst$$reg), 64 - width);
+ %}
+
+ ins_pipe(ialu_reg_shift);
+%}
+
+instruct bitfieldExtractL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask) %{
+ match(Set dst (AndL (URShiftL src rshift) mask));
+ // The field must not extend past the high end of the long.
+ predicate(log2i_exact((julong)n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 0x3f) <= 64);
+
+ ins_cost(ALU_COST * 2);
+ format %{ "slli $dst, $src, 64 - (($rshift & 0x3f) + width($mask))\n\t"
+ "srli $dst, $dst, 64 - width($mask)\t#@bitfieldExtractL" %}
+
+ ins_encode %{
+ int rshift = $rshift$$constant & 0x3f;
+ int width = log2i_exact((julong)$mask$$constant + 1);
+ __ slli(as_Register($dst$$reg), as_Register($src$$reg), 64 - (rshift + width));
+ __ srli(as_Register($dst$$reg), as_Register($dst$$reg), 64 - width);
+ %}
+
+ ins_pipe(ialu_reg_shift);
+%}
+
+// Extract an int sized field out of a long. The mask is at most 31 bits wide,
+// so the zero extended result is a valid int. This is the shape emitted by
+// LibraryCallKit::inline_native_hashcode.
+instruct bitfieldExtractL2I(iRegINoSp dst, iRegL src, immI rshift, immI_bitmask mask) %{
+ match(Set dst (AndI (ConvL2I (URShiftL src rshift)) mask));
+ predicate(log2i_exact((juint)n->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 0x3f) <= 64);
+
+ ins_cost(ALU_COST * 2);
+ format %{ "slli $dst, $src, 64 - (($rshift & 0x3f) + width($mask))\n\t"
+ "srli $dst, $dst, 64 - width($mask)\t#@bitfieldExtractL2I" %}
+
+ ins_encode %{
+ int rshift = $rshift$$constant & 0x3f;
+ int width = log2i_exact((juint)$mask$$constant + 1);
+ __ slli(as_Register($dst$$reg), as_Register($src$$reg), 64 - (rshift + width));
+ __ srli(as_Register($dst$$reg), as_Register($dst$$reg), 64 - width);
+ %}
+
+ ins_pipe(ialu_reg_shift);
+%}
+
// Shift Right Arithmetic Register
// Only the low 6 bits of src2 are considered for the shift amount, all other bits are ignored.
instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
diff --git a/src/hotspot/cpu/riscv/riscv_b.ad b/src/hotspot/cpu/riscv/riscv_b.ad
index a13efa96fc16..cf61b1d43a6e 100644
--- a/src/hotspot/cpu/riscv/riscv_b.ad
+++ b/src/hotspot/cpu/riscv/riscv_b.ad
@@ -183,7 +183,37 @@ instruct convI2UL_reg_reg_b(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask) %{
__ zext_w(as_Register($dst$$reg), as_Register($src$$reg));
%}
- ins_pipe(ialu_reg_shift);
+ ins_pipe(ialu_reg);
+%}
+
+// And with a low 16-bit mask
+instruct andL_16bits_b(iRegLNoSp dst, iRegL src, immL_16bits mask) %{
+ predicate(UseZbb);
+ match(Set dst (AndL src mask));
+
+ format %{ "zext.h $dst, $src\t#@andL_16bits_b" %}
+
+ ins_cost(ALU_COST);
+ ins_encode %{
+ __ zext_h(as_Register($dst$$reg), as_Register($src$$reg));
+ %}
+
+ ins_pipe(ialu_reg);
+%}
+
+// And with a low 32-bit mask
+instruct andL_32bits_b(iRegLNoSp dst, iRegL src, immL_32bits mask) %{
+ predicate(UseZba);
+ match(Set dst (AndL src mask));
+
+ format %{ "zext.w $dst, $src\t#@andL_32bits_b" %}
+
+ ins_cost(ALU_COST);
+ ins_encode %{
+ __ zext_w(as_Register($dst$$reg), as_Register($src$$reg));
+ %}
+
+ ins_pipe(ialu_reg);
%}
// BSWAP instructions
diff --git a/src/hotspot/cpu/riscv/riscv_v.ad b/src/hotspot/cpu/riscv/riscv_v.ad
index ef0ce89133ed..a4cd5c838715 100644
--- a/src/hotspot/cpu/riscv/riscv_v.ad
+++ b/src/hotspot/cpu/riscv/riscv_v.ad
@@ -4518,7 +4518,7 @@ instruct vmaskAllL(vRegMask dst, iRegL src) %{
// ------------------------------ Vector mask basic OPs ------------------------
-// vector mask logical ops: and/and-not/or/xor
+// vector mask logical ops
instruct vmask_and(vRegMask dst, vRegMask src1, vRegMask src2) %{
match(Set dst (AndVMask src1 src2));
@@ -4559,6 +4559,136 @@ instruct vmask_and_notL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1)
ins_pipe(pipe_slow);
%}
+instruct vmask_or_notI(vRegMask dst, vRegMask src1, vRegMask src2, immI_M1 m1) %{
+ match(Set dst (OrVMask src1 (XorVMask src2 (MaskAll m1))));
+ format %{ "vmask_or_notI $dst, $src1, $src2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmorn_mm(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src1$$reg),
+ as_VectorRegister($src2$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_or_notL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1) %{
+ match(Set dst (OrVMask src1 (XorVMask src2 (MaskAll m1))));
+ format %{ "vmask_or_notL $dst, $src1, $src2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmorn_mm(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src1$$reg),
+ as_VectorRegister($src2$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_nandI(vRegMask dst, vRegMask src1, vRegMask src2, immI_M1 m1) %{
+ match(Set dst (XorVMask (AndVMask src1 src2) (MaskAll m1)));
+ format %{ "vmask_nandI $dst, $src1, $src2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmnand_mm(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src1$$reg),
+ as_VectorRegister($src2$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_nandL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1) %{
+ match(Set dst (XorVMask (AndVMask src1 src2) (MaskAll m1)));
+ format %{ "vmask_nandL $dst, $src1, $src2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmnand_mm(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src1$$reg),
+ as_VectorRegister($src2$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_norI(vRegMask dst, vRegMask src1, vRegMask src2, immI_M1 m1) %{
+ match(Set dst (XorVMask (OrVMask src1 src2) (MaskAll m1)));
+ format %{ "vmask_norI $dst, $src1, $src2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmnor_mm(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src1$$reg),
+ as_VectorRegister($src2$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_norL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1) %{
+ match(Set dst (XorVMask (OrVMask src1 src2) (MaskAll m1)));
+ format %{ "vmask_norL $dst, $src1, $src2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmnor_mm(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src1$$reg),
+ as_VectorRegister($src2$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_xnorI(vRegMask dst, vRegMask src1, vRegMask src2, immI_M1 m1) %{
+ match(Set dst (XorVMask (XorVMask src1 src2) (MaskAll m1)));
+ match(Set dst (XorVMask src1 (XorVMask src2 (MaskAll m1))));
+ format %{ "vmask_xnorI $dst, $src1, $src2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmxnor_mm(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src1$$reg),
+ as_VectorRegister($src2$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_xnorL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1) %{
+ match(Set dst (XorVMask (XorVMask src1 src2) (MaskAll m1)));
+ match(Set dst (XorVMask src1 (XorVMask src2 (MaskAll m1))));
+ format %{ "vmask_xnorL $dst, $src1, $src2" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmxnor_mm(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src1$$reg),
+ as_VectorRegister($src2$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_notI(vRegMask dst, vRegMask src, immI_M1 m1) %{
+ match(Set dst (XorVMask src (MaskAll m1)));
+ format %{ "vmask_notI $dst, $src" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmnot_m(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct vmask_notL(vRegMask dst, vRegMask src, immL_M1 m1) %{
+ match(Set dst (XorVMask src (MaskAll m1)));
+ format %{ "vmask_notL $dst, $src" %}
+ ins_encode %{
+ BasicType bt = Matcher::vector_element_basic_type(this);
+ __ vsetvli_helper(bt, Matcher::vector_length(this));
+ __ vmnot_m(as_VectorRegister($dst$$reg),
+ as_VectorRegister($src$$reg));
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct vmask_or(vRegMask dst, vRegMask src1, vRegMask src2) %{
match(Set dst (OrVMask src1 src2));
format %{ "vmask_or $dst, $src1, $src2" %}
diff --git a/src/hotspot/cpu/riscv/runtime_riscv.cpp b/src/hotspot/cpu/riscv/runtime_riscv.cpp
index 5a1fdbe773a1..62c61bf436a8 100644
--- a/src/hotspot/cpu/riscv/runtime_riscv.cpp
+++ b/src/hotspot/cpu/riscv/runtime_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2024, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -246,8 +246,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
// Jump to interpreter
__ ret();
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
UncommonTrapBlob* ut_blob = UncommonTrapBlob::create(&buffer, oop_maps,
SimpleRuntimeFrame::framesize >> 1);
@@ -389,8 +388,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
__ jr(t1);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Set exception blob
ExceptionBlob* ex_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
index f28230f23bc5..f1bc7fc0d5e3 100644
--- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
+++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
@@ -1392,7 +1392,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
assert(vep_offset != -1, "Must be set");
#endif
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod* nm = nmethod::new_native_nmethod(method,
compile_id,
masm->code(),
@@ -1430,7 +1430,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
in_sig_bt,
in_regs);
int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
- __ flush();
+ // Code will be copied. No ICache sync required.
int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually
return nmethod::new_native_nmethod(method,
compile_id,
@@ -1817,9 +1817,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
Label safepoint_in_progress, safepoint_in_progress_done;
- __ mv(t0, _thread_in_vm);
-
- __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
+ // change thread state
+ __ mv(t1, _thread_in_Java);
+ __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
+ __ sw(t1, Address(xthread, JavaThread::thread_state_offset()));
// Force this write out before the read below
if (!UseSystemMemoryBarrier) {
@@ -1834,12 +1835,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ bind(safepoint_in_progress_done);
}
- // change thread state
- __ la(t1, Address(xthread, JavaThread::thread_state_offset()));
- __ mv(t0, _thread_in_Java);
- __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
- __ sw(t0, Address(t1));
-
if (method->is_object_wait0()) {
// Check preemption for Object.wait()
__ ld(t1, Address(xthread, JavaThread::preempt_alternate_return_offset()));
@@ -2040,7 +2035,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
#ifndef PRODUCT
assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
#endif
- __ rt_call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
+ __ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans));
// Restore any method result value
restore_native_result(masm, ret_type, stack_slots);
@@ -2082,7 +2077,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
}
}
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod *nm = nmethod::new_native_nmethod(method,
compile_id,
@@ -2421,8 +2416,7 @@ void SharedRuntime::generate_deopt_blob() {
// Jump to interpreter
__ ret();
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
_deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
assert(_deopt_blob != nullptr, "create deoptimization blob fail!");
@@ -2567,8 +2561,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
__ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
#endif
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Fill-out other meta info
SafepointBlob* sp_blob = SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
@@ -2663,9 +2656,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
__ ld(x10, Address(xthread, Thread::pending_exception_offset()));
__ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
- // -------------
- // make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// return the blob
RuntimeStub* rs_blob = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index 260d31fc7cdc..7cefb1eef2c9 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -1240,8 +1240,8 @@ class StubGenerator: public StubCodeGenerator {
void verify_oop_array(size_t size, Register a, Register count, Register temp) {
Label loop, end;
__ mv(t1, zr);
- __ slli(t0, count, exact_log2(size));
__ bind(loop);
+ __ slli(t0, count, exact_log2(size));
__ bgeu(t1, t0, end);
__ add(temp, a, t1);
diff --git a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
index ef23498da5c7..bf552678f965 100644
--- a/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp
@@ -1210,11 +1210,9 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
__ push(ltos);
// change thread state
- // Force all preceding writes to be observed prior to thread state change
+ __ mv(t1, _thread_in_Java);
__ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
-
- __ mv(t0, _thread_in_vm);
- __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
+ __ sw(t1, Address(xthread, JavaThread::thread_state_offset()));
// Force this write out before the read below
if (!UseSystemMemoryBarrier) {
@@ -1236,19 +1234,12 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// hand.
//
__ mv(c_rarg0, xthread);
- __ rt_call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
+ __ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans));
__ get_method(xmethod);
__ reinit_heapbase();
__ bind(Continue);
}
- // change thread state
- // Force all preceding writes to be observed prior to thread state change
- __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
-
- __ mv(t0, _thread_in_Java);
- __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
-
// Check preemption for Object.wait()
Label not_preempted;
__ ld(t1, Address(xthread, JavaThread::preempt_alternate_return_offset()));
diff --git a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
index 0fccce171bbc..70dafd94fe23 100644
--- a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
+++ b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -330,7 +330,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
//////////////////////////////////////////////////////////////////////////////
- __ flush();
+ // Code will be copied. No ICache sync required.
#ifndef PRODUCT
stringStream ss;
diff --git a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
index 4fc70e7656f2..73948ef6429d 100644
--- a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
@@ -137,7 +137,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
__ ld(t1, Address(xmethod, entry_offset));
__ jr(t1);
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
return s;
@@ -246,7 +246,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
assert(SharedRuntime::get_handle_wrong_method_stub() != nullptr, "check initialization order");
__ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
return s;
diff --git a/src/hotspot/cpu/s390/bytes_s390.hpp b/src/hotspot/cpu/s390/bytes_s390.hpp
deleted file mode 100644
index ed6418bd4511..000000000000
--- a/src/hotspot/cpu/s390/bytes_s390.hpp
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2022 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_S390_BYTES_S390_HPP
-#define CPU_S390_BYTES_S390_HPP
-
-#include "memory/allStatic.hpp"
-
-class Bytes: AllStatic {
- public:
- // Efficient reading and writing of unaligned unsigned data in
- // platform-specific byte ordering.
-
- // Use regular load and store for unaligned access.
- //
- // On z/Architecture, unaligned loads and stores are supported when using the
- // "traditional" load (LH, L/LY, LG) and store (STH, ST/STY, STG) instructions.
- // The penalty for unaligned access is just very few (two or three) ticks,
- // plus another few (two or three) ticks if the access crosses a cache line boundary.
- //
- // In short, it makes no sense on z/Architecture to piecemeal get or put unaligned data.
-
- static inline u2 get_native_u2(address p) { return *(u2*)p; }
- static inline u4 get_native_u4(address p) { return *(u4*)p; }
- static inline u8 get_native_u8(address p) { return *(u8*)p; }
-
- static inline void put_native_u2(address p, u2 x) { *(u2*)p = x; }
- static inline void put_native_u4(address p, u4 x) { *(u4*)p = x; }
- static inline void put_native_u8(address p, u8 x) { *(u8*)p = x; }
-
- // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering)
- static inline u2 get_Java_u2(address p) { return get_native_u2(p); }
- static inline u4 get_Java_u4(address p) { return get_native_u4(p); }
- static inline u8 get_Java_u8(address p) { return get_native_u8(p); }
-
- static inline void put_Java_u2(address p, u2 x) { put_native_u2(p, x); }
- static inline void put_Java_u4(address p, u4 x) { put_native_u4(p, x); }
- static inline void put_Java_u8(address p, u8 x) { put_native_u8(p, x); }
-};
-
-#endif // CPU_S390_BYTES_S390_HPP
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index d3c143c97aa8..14e72183e90b 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -3109,13 +3109,8 @@ void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
// We are storing into an array that *may* be null-free (the declared type is
// Object[], abstract[], interface[] or VT.ref[]).
- Label test_mark_word;
Register tmp = op->tmp()->as_register();
__ z_lg(tmp, oopDesc::mark_offset_in_bytes(), op->array()->as_register());
- __ z_tmll(tmp, markWord::unlocked_value);
- __ z_brnaz(test_mark_word);
- __ load_prototype_header(tmp, op->array()->as_register());
- __ bind(test_mark_word);
__ z_tmll(tmp, markWord::null_free_array_bit_in_place);
}
diff --git a/src/hotspot/cpu/s390/downcallLinker_s390.cpp b/src/hotspot/cpu/s390/downcallLinker_s390.cpp
index 4fe4c31567a0..49d7ab0d487e 100644
--- a/src/hotspot/cpu/s390/downcallLinker_s390.cpp
+++ b/src/hotspot/cpu/s390/downcallLinker_s390.cpp
@@ -247,7 +247,8 @@ void DowncallLinker::StubGenerator::generate() {
if (_needs_transition) {
__ block_comment("thread_native2java {");
- __ set_thread_state(_thread_in_vm);
+ // change thread state
+ __ set_thread_state(_thread_in_Java);
if (!UseSystemMemoryBarrier) {
__ z_fence(); // Order state change wrt. safepoint poll.
@@ -260,9 +261,6 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_after_safepoint_poll);
- // change thread state
- __ set_thread_state(_thread_in_Java);
-
__ block_comment("reguard_stack_check {");
__ z_cli(Address(Z_thread,
JavaThread::stack_guard_state_offset() + in_ByteSize(sizeof(StackOverflow::StackGuardState) - 1)),
@@ -288,7 +286,7 @@ void DowncallLinker::StubGenerator::generate() {
// Need to save the native result registers around any runtime calls.
out_reg_spiller.generate_spill(_masm, out_spill_offset);
- __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, JavaThread::check_special_condition_for_native_trans));
+ __ load_const_optimized(call_target_address, CAST_FROM_FN_PTR(uint64_t, SharedRuntime::check_special_condition_for_native_trans));
__ z_lgr(Z_ARG1, Z_thread);
__ call(call_target_address);
@@ -316,5 +314,5 @@ void DowncallLinker::StubGenerator::generate() {
//////////////////////////////////////////////////////////////////////////////
- __ flush();
+ // Code will be copied. No ICache sync required.
}
diff --git a/src/hotspot/cpu/s390/interpreterRT_s390.cpp b/src/hotspot/cpu/s390/interpreterRT_s390.cpp
index c9d7adbc36a8..75d8a648a262 100644
--- a/src/hotspot/cpu/s390/interpreterRT_s390.cpp
+++ b/src/hotspot/cpu/s390/interpreterRT_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -139,7 +139,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
iterate(fingerprint);
__ load_const_optimized(Z_RET, AbstractInterpreter::result_handler(method()->result_type()));
__ z_br(Z_R14);
- __ flush();
+ __ invalidate_icache();
}
#undef __
diff --git a/src/hotspot/cpu/s390/jniFastGetField_s390.cpp b/src/hotspot/cpu/s390/jniFastGetField_s390.cpp
index 00c9316c3551..3456afdc674d 100644
--- a/src/hotspot/cpu/s390/jniFastGetField_s390.cpp
+++ b/src/hotspot/cpu/s390/jniFastGetField_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -138,7 +138,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
__ load_const_optimized(Robj, slow_case_addr);
__ z_br(Robj); // tail call
- __ flush();
+ __ invalidate_icache();
return fast_entry;
}
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index e8971e7630e6..bb4a10a5f80b 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -4236,11 +4236,6 @@ void MacroAssembler::load_metadata(Register dst, Register src) {
}
}
-void MacroAssembler::load_prototype_header(Register dst, Register src) {
- load_klass(dst, src);
- z_lg(dst, Address(dst, Klass::prototype_header_offset()));
-}
-
void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
assert_different_registers(dst_oop, klass, Z_R0);
@@ -6377,7 +6372,7 @@ void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register temp1
{ // Try to lock. Transition lock bits 0b01 => 0b00
const Register locked_obj = top;
- z_oill(mark, markWord::unlocked_value);
+ z_oill(mark, markWord::lock_neutral_value);
if (Arguments::is_valhalla_enabled()) {
static_assert((uint32_t)markWord::inline_type_bit_in_place <= 0x7FFFFFFF,
"inline_type_bit_in_place must fit in low 32 bits for z_nilf");
@@ -6386,7 +6381,7 @@ void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register temp1
}
z_lgr(locked_obj, mark);
// Clear lock-bits from locked_obj (locked state)
- z_xilf(locked_obj, markWord::unlocked_value);
+ z_xilf(locked_obj, markWord::lock_neutral_value);
z_csg(mark, locked_obj, mark_offset, obj);
branch_optimized(Assembler::bcondNotEqual, slow);
}
@@ -6458,7 +6453,7 @@ void MacroAssembler::fast_unlock(Register obj, Register temp1, Register temp2, L
#ifdef ASSERT
// Check header not unlocked (0b01).
NearLabel not_unlocked;
- z_tmll(mark, markWord::unlocked_value);
+ z_tmll(mark, markWord::lock_neutral_value);
z_braz(not_unlocked);
stop("fast_unlock already unlocked");
bind(not_unlocked);
@@ -6467,7 +6462,7 @@ void MacroAssembler::fast_unlock(Register obj, Register temp1, Register temp2, L
{ // Try to unlock. Transition lock bits 0b00 => 0b01
Register unlocked_obj = top;
z_lgr(unlocked_obj, mark);
- z_oill(unlocked_obj, markWord::unlocked_value);
+ z_oill(unlocked_obj, markWord::lock_neutral_value);
z_csg(mark, unlocked_obj, mark_offset, obj);
branch_optimized(Assembler::bcondEqual, unlocked);
}
@@ -6538,7 +6533,7 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
{ // Try to lock. Transition lock bits 0b01 => 0b00
assert(mark_offset == 0, "required to avoid a lea");
const Register locked_obj = top;
- z_oill(mark, markWord::unlocked_value);
+ z_oill(mark, markWord::lock_neutral_value);
if (Arguments::is_valhalla_enabled()) {
static_assert((uint32_t)markWord::inline_type_bit_in_place <= 0x7FFFFFFF,
"inline_type_bit_in_place must fit in low 32 bits for z_nilf");
@@ -6547,7 +6542,7 @@ void MacroAssembler::compiler_fast_lock_object(Register obj, Register box, Regis
}
z_lgr(locked_obj, mark);
// Clear lock-bits from locked_obj (locked state)
- z_xilf(locked_obj, markWord::unlocked_value);
+ z_xilf(locked_obj, markWord::lock_neutral_value);
z_csg(mark, locked_obj, mark_offset, obj);
branch_optimized(Assembler::bcondNotEqual, slow_path);
}
@@ -6725,7 +6720,7 @@ void MacroAssembler::compiler_fast_unlock_object(Register obj, Register box, Reg
#ifdef ASSERT
// Check header not unlocked (0b01).
NearLabel not_unlocked;
- z_tmll(mark, markWord::unlocked_value);
+ z_tmll(mark, markWord::lock_neutral_value);
z_braz(not_unlocked);
stop("fast_unlock already unlocked");
bind(not_unlocked);
@@ -6734,7 +6729,7 @@ void MacroAssembler::compiler_fast_unlock_object(Register obj, Register box, Reg
{ // Try to unlock. Transition lock bits 0b00 => 0b01
Register unlocked_obj = top;
z_lgr(unlocked_obj, mark);
- z_oill(unlocked_obj, markWord::unlocked_value);
+ z_oill(unlocked_obj, markWord::lock_neutral_value);
z_csg(mark, unlocked_obj, mark_offset, obj);
branch_optimized(Assembler::bcondEqual, unlocked);
}
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index 3fd182144724..2b831d1a49cf 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -863,7 +863,6 @@ class MacroAssembler: public Assembler {
void decode_klass_not_null(Register dst);
void load_klass(Register klass, Address mem);
void load_klass(Register klass, Register src_oop);
- void load_prototype_header(Register dst, Register src);
void store_klass(Register klass, Register dst_oop, Register ck = noreg); // Klass will get compressed if ck not provided.
void store_klass_gap(Register s, Register dst_oop);
void load_narrow_klass_compact(Register dst, Register src);
diff --git a/src/hotspot/cpu/s390/runtime_s390.cpp b/src/hotspot/cpu/s390/runtime_s390.cpp
index 658fba069b4c..ddfe8afe2a2d 100644
--- a/src/hotspot/cpu/s390/runtime_s390.cpp
+++ b/src/hotspot/cpu/s390/runtime_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -142,8 +142,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
__ z_br(handle_exception);
- // Make sure all code is generated.
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Set exception blob.
OopMapSet *oop_maps = nullptr;
diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
index b6b22102d011..aad194900450 100644
--- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
+++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
* Copyright (c) 2026 IBM Corporation. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -2220,7 +2220,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
assert(vep_offset != -1, "Must be set");
#endif
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod* nm = nmethod::new_native_nmethod(method,
compile_id,
masm->code(),
@@ -2250,7 +2250,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
int frame_complete = ((intptr_t)__ pc()) - start; // Not complete, period.
- __ flush();
+ // Code will be copied. No ICache sync required.
int stack_slots = SharedRuntime::out_preserve_stack_slots(); // No out slots at all, actually.
@@ -2745,8 +2745,13 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
break;
}
- // Transition from _thread_in_native to _thread_in_vm.
- __ set_thread_state(_thread_in_vm);
+ // Transition from _thread_in_native to _thread_in_Java.
+ __ set_thread_state(_thread_in_Java);
+
+ // Force this write out before the read below.
+ if (!UseSystemMemoryBarrier) {
+ __ z_fence();
+ }
// Safepoint synchronization
//--------------------------------------------------------------------
@@ -2760,11 +2765,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
save_native_result(masm, ret_type, workspace_slot_offset); // Make Z_R2 available as work reg.
- // Force this write out before the read below.
- if (!UseSystemMemoryBarrier) {
- __ z_fence();
- }
-
__ safepoint_poll(sync, Z_R1);
__ load_and_test_int(Z_R0, Address(Z_thread, JavaThread::suspend_flags_offset()));
@@ -2776,9 +2776,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
// a distinct one for this pc.
//
__ bind(sync);
- __ z_acquire();
- address entry_point = CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans);
+ address entry_point = CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans);
__ call_VM_leaf(entry_point, Z_thread);
@@ -2786,13 +2785,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
restore_native_result(masm, ret_type, workspace_slot_offset);
}
- //--------------------------------------------------------------------
- // Thread state is _thread_in_vm. Any safepoint blocking has
- // already happened so we can now change state to _thread_in_Java.
- //--------------------------------------------------------------------
- // Transition from _thread_in_vm to _thread_in_Java.
- __ set_thread_state(_thread_in_Java);
-
// Check preemption for Object.wait()
if (method->is_object_wait0()) {
NearLabel not_preempted;
@@ -2968,7 +2960,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
__ restore_return_pc();
__ z_br(Z_R1_scratch);
- __ flush();
+ // Code will be copied. No ICache sync required.
//////////////////////////////////////////////////////////////////////
// end of code generation
//////////////////////////////////////////////////////////////////////
@@ -3543,8 +3535,7 @@ void SharedRuntime::generate_deopt_blob() {
// return to the interpreter entry point.
__ z_br(Z_R14);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
_deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, RegisterSaver::live_reg_frame_size(RegisterSaver::all_registers, SuperwordUseVX)/wordSize);
_deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
@@ -3682,7 +3673,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
// return to the interpreter entry point
__ z_br(Z_R14);
- masm->flush();
+ // Code will be copied. No ICache sync required.
return UncommonTrapBlob::create(&buffer, nullptr, framesize_in_bytes/wordSize);
}
#endif // COMPILER2
@@ -3780,8 +3771,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
__ z_br(Z_R14);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Fill-out other meta info
return SafepointBlob::create(&buffer, oop_maps, RegisterSaver::live_reg_frame_size(RegisterSaver::all_registers, save_vectors)/wordSize);
@@ -3863,8 +3853,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
__ z_br(Z_R1_scratch);
// -------------
- // make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// return the blob
// frame_size_words or bytes??
diff --git a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
index c0a1b06954da..ddcba7b25339 100644
--- a/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
+++ b/src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp
@@ -1575,13 +1575,14 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// In order for GC to work, don't clear the last_Java_sp until after
// blocking.
- __ set_thread_state(_thread_in_vm);
+ // Transition from _thread_in_native to _thread_in_Java.
+ // Force this write out before the read below;
+ __ set_thread_state(_thread_in_Java);
if (!UseSystemMemoryBarrier) {
__ z_fence();
}
- // Now before we return to java we must look for a current safepoint
- // (a new safepoint can not start since we entered _thread_in_vm).
+ // Now before we return to java we must look for a current safepoint.
// We must check here because a current safepoint could be in progress.
// Check for safepoint operation in progress and/or pending suspend requests.
@@ -1593,17 +1594,13 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
__ z_bre(Continue); // 0 -> no flag set -> not suspended
__ bind(do_safepoint);
__ z_lgr(Z_ARG1, Z_thread);
- __ call_c(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
+ __ call_c(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans));
__ bind(Continue);
}
//=============================================================================
// Back in Interpreter Frame.
- // We are in _thread_in_vm here and back in the normal
- // interpreter frame. We don't have to do anything special about
- // safepoints and we can switch to Java mode anytime we are ready.
-
// Note: frame::interpreter_frame_result has a dependency on how the
// method result is saved across the call to post_method_exit. For
// native methods it assumes that the non-FPU/non-void result is
@@ -1614,10 +1611,6 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
//=============================================================================
// Back in Java.
- // Memory ordering: Z does not reorder store/load with subsequent
- // load. That's strong enough.
- __ set_thread_state(_thread_in_Java);
-
__ reset_last_Java_frame();
// We reset the JNI handle block only after unboxing the result; see below.
diff --git a/src/hotspot/cpu/s390/upcallLinker_s390.cpp b/src/hotspot/cpu/s390/upcallLinker_s390.cpp
index de57e5e0cc49..e940f0df2b93 100644
--- a/src/hotspot/cpu/s390/upcallLinker_s390.cpp
+++ b/src/hotspot/cpu/s390/upcallLinker_s390.cpp
@@ -271,7 +271,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
//////////////////////////////////////////////////////////////////////////////
- _masm->flush();
+ // Code will be copied. No ICache sync required.
#ifndef PRODUCT
stringStream ss;
diff --git a/src/hotspot/cpu/s390/vm_version_s390.cpp b/src/hotspot/cpu/s390/vm_version_s390.cpp
index 95ca00d4d1a6..eb0a5a63223e 100644
--- a/src/hotspot/cpu/s390/vm_version_s390.cpp
+++ b/src/hotspot/cpu/s390/vm_version_s390.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1142,7 +1142,7 @@ void VM_Version::determine_features() {
a->z_br(Z_R14);
address code_end = a->pc();
- a->flush();
+ a->invalidate_icache();
cbuf.insts()->set_end(code_end);
diff --git a/src/hotspot/cpu/s390/vtableStubs_s390.cpp b/src/hotspot/cpu/s390/vtableStubs_s390.cpp
index de4049ccacfe..596f3bc1c229 100644
--- a/src/hotspot/cpu/s390/vtableStubs_s390.cpp
+++ b/src/hotspot/cpu/s390/vtableStubs_s390.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -141,7 +141,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
__ z_lg(Z_R1_scratch, in_bytes(Method::from_compiled_offset()), Z_method);
__ z_br(Z_R1_scratch);
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
return s;
@@ -235,7 +235,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
assert(slop_delta >= 0, "negative slop(%d) encountered, adjust code size estimate!", slop_delta);
__ z_br(Z_R1_scratch);
- masm->flush();
+ masm->invalidate_icache();
bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
return s;
diff --git a/src/hotspot/cpu/x86/bytes_x86.hpp b/src/hotspot/cpu/x86/bytes_x86.hpp
deleted file mode 100644
index 3f7f42342600..000000000000
--- a/src/hotspot/cpu/x86/bytes_x86.hpp
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_X86_BYTES_X86_HPP
-#define CPU_X86_BYTES_X86_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/align.hpp"
-#include "utilities/byteswap.hpp"
-#include "utilities/macros.hpp"
-
-class Bytes: AllStatic {
- public:
- // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering
- template
- static inline T get_native(const void* p) {
- assert(p != nullptr, "null pointer");
-
- T x;
-
- if (is_aligned(p, sizeof(T))) {
- x = *(T*)p;
- } else {
- memcpy(&x, p, sizeof(T));
- }
-
- return x;
- }
-
- template
- static inline void put_native(void* p, T x) {
- assert(p != nullptr, "null pointer");
-
- if (is_aligned(p, sizeof(T))) {
- *(T*)p = x;
- } else {
- memcpy(p, &x, sizeof(T));
- }
- }
-
- static inline u2 get_native_u2(address p) { return get_native((void*)p); }
- static inline u4 get_native_u4(address p) { return get_native((void*)p); }
- static inline u8 get_native_u8(address p) { return get_native((void*)p); }
- static inline void put_native_u2(address p, u2 x) { put_native((void*)p, x); }
- static inline void put_native_u4(address p, u4 x) { put_native((void*)p, x); }
- static inline void put_native_u8(address p, u8 x) { put_native((void*)p, x); }
-
- // Efficient reading and writing of unaligned unsigned data in Java
- // byte ordering (i.e. big-endian ordering). Byte-order reversal is
- // needed since x86 CPUs use little-endian format.
- template
- static inline T get_Java(const address p) {
- T x = get_native(p);
-
- if (Endian::is_Java_byte_ordering_different()) {
- x = byteswap(x);
- }
-
- return x;
- }
-
- template
- static inline void put_Java(address p, T x) {
- if (Endian::is_Java_byte_ordering_different()) {
- x = byteswap(x);
- }
-
- put_native(p, x);
- }
-
- static inline u2 get_Java_u2(address p) { return get_Java(p); }
- static inline u4 get_Java_u4(address p) { return get_Java(p); }
- static inline u8 get_Java_u8(address p) { return get_Java(p); }
-
- static inline void put_Java_u2(address p, u2 x) { put_Java(p, x); }
- static inline void put_Java_u4(address p, u4 x) { put_Java(p, x); }
- static inline void put_Java_u8(address p, u8 x) { put_Java(p, x); }
-};
-
-#endif // CPU_X86_BYTES_X86_HPP
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index 84f99215f156..23a457913e5a 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -262,7 +262,6 @@ void LIR_Assembler::osr_entry() {
//
// build frame
- ciMethod* m = compilation()->method();
__ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
// OSR buffer is
@@ -1339,7 +1338,6 @@ void LIR_Assembler::type_profile_helper(Register mdo,
void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
// we always need a stub for the failure case.
- CodeStub* stub = op->stub();
Register obj = op->object()->as_register();
Register k_RInfo = op->tmp1()->as_register();
Register klass_RInfo = op->tmp2()->as_register();
@@ -1577,13 +1575,8 @@ void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
// We are storing into an array that *may* be null-free (the declared type is
// Object[], abstract[], interface[] or VT.ref[]).
- Label test_mark_word;
Register tmp = op->tmp()->as_register();
__ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
- __ testl(tmp, markWord::unlocked_value);
- __ jccb(Assembler::notZero, test_mark_word);
- __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
- __ bind(test_mark_word);
__ testl(tmp, markWord::null_free_array_bit_in_place);
}
@@ -2341,7 +2334,7 @@ void LIR_Assembler::emit_static_call_stub() {
return;
}
- int start = __ offset();
+ DEBUG_ONLY(int start = __ offset();)
// make sure that the displacement word of the call ends up word aligned
__ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
@@ -2937,7 +2930,6 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
ciMethod* method = op->profiled_method();
int bci = op->profiled_bci();
- ciMethod* callee = op->profiled_callee();
Register tmp_load_klass = rscratch1;
// Update counter for all call types
diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
index 95de5906b62a..2642372bc373 100644
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
@@ -674,7 +674,7 @@ LIR_Opr LIRGenerator::atomic_cmpxchg(BasicType type, LIR_Opr addr, LIRItem& cmp_
}
LIR_Opr LIRGenerator::atomic_xchg(BasicType type, LIR_Opr addr, LIRItem& value) {
- bool is_oop = is_reference_type(type);
+ DEBUG_ONLY(bool is_oop = is_reference_type(type);)
LIR_Opr result = new_register(type);
value.load_item();
// Because we want a 2-arg form of xchg and xadd
@@ -920,7 +920,6 @@ void LIRGenerator::do_update_CRC32(Intrinsic* x) {
assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions support");
// Make all state_for calls early since they can emit code
LIR_Opr result = rlock_result(x);
- int flags = 0;
switch (x->id()) {
case vmIntrinsics::_updateCRC32: {
LIRItem crc(x->argument_at(0), this);
diff --git a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
index 20b6ee0a15ff..ecb997e878ac 100644
--- a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
@@ -814,7 +814,6 @@ OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
OopMapSet* Runtime1::generate_code_for(StubId id, StubAssembler* sasm) {
// for better readability
- const bool must_gc_arguments = true;
const bool dont_gc_arguments = false;
// default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 6e08f438a4af..e69a210117ef 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -313,8 +313,8 @@ void C2_MacroAssembler::fast_lock(Register obj, Register box, Register rax_reg,
// Try to lock. Transition lock bits 0b01 => 0b00
movptr(rax_reg, mark);
- orptr(rax_reg, markWord::unlocked_value);
- andptr(mark, ~(int32_t)markWord::unlocked_value);
+ orptr(rax_reg, markWord::lock_neutral_value);
+ andptr(mark, ~(int32_t)markWord::lock_neutral_value);
lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::notEqual, slow_path);
@@ -511,7 +511,7 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register t,
// Try to unlock. Transition lock bits 0b00 => 0b01
movptr(reg_rax, mark);
andptr(reg_rax, ~(int32_t)markWord::lock_mask_in_place);
- orptr(mark, markWord::unlocked_value);
+ orptr(mark, markWord::lock_neutral_value);
lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::notEqual, push_and_slow_path);
jmp(unlocked);
@@ -2243,7 +2243,6 @@ void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRe
void C2_MacroAssembler::reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
assert_different_registers(src2, vtmp1);
- int vector_len = Assembler::AVX_256bit;
vextracti64x4_high(vtmp1, src2);
reduce_operation_256(T_SHORT, opcode, vtmp1, vtmp1, src2);
reduce16S(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
@@ -2507,7 +2506,6 @@ XMMRegister C2_MacroAssembler::get_lane(BasicType typ, XMMRegister dst, XMMRegis
int esize = type2aelembytes(typ);
int elem_per_lane = 16/esize;
int lane = elemindex / elem_per_lane;
- int eindex = elemindex % elem_per_lane;
if (lane >= 2) {
assert(UseAVX > 2, "required");
@@ -5188,7 +5186,7 @@ void C2_MacroAssembler::vector_castF2X_avx(BasicType to_elem_bt, XMMRegister dst
void C2_MacroAssembler::vector_castF2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
Register rscratch, int vec_enc) {
- int to_elem_sz = type2aelembytes(to_elem_bt);
+ DEBUG_ONLY(int to_elem_sz = type2aelembytes(to_elem_bt);)
assert(to_elem_sz <= 4, "");
vcvttps2dq(dst, src, vec_enc);
vector_cast_fp_to_int_special_cases_evex(T_FLOAT, dst, src, xtmp1, xtmp2, ktmp1, ktmp2, rscratch, float_sign_flip, vec_enc);
diff --git a/src/hotspot/cpu/x86/c2_init_x86.cpp b/src/hotspot/cpu/x86/c2_init_x86.cpp
index 4d8db39bb0c5..0a74d0f7f3ac 100644
--- a/src/hotspot/cpu/x86/c2_init_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_init_x86.cpp
@@ -36,7 +36,6 @@ void Compile::pd_compiler2_init() {
if (UseAVX < 3) {
int delta = XMMRegister::max_slots_per_register * XMMRegister::number_of_registers;
int bottom = ConcreteRegisterImpl::max_fpr;
- int top = bottom + delta;
int middle = bottom + (delta / 2);
int xmm_slots = XMMRegister::max_slots_per_register;
int lower = xmm_slots / 2;
diff --git a/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp b/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp
index b71fb46f0758..6585b60f8a0b 100644
--- a/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp
+++ b/src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp
@@ -220,7 +220,7 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
assert(StubInfo::entry_count(stub_id) == 1, "sanity check");
GrowableArray extras;
- const int expected_extra_count = 2 * NUMBER_OF_CASES;
+ DEBUG_ONLY(const int expected_extra_count = 2 * NUMBER_OF_CASES;)
address start = stubgen->load_archive_data(stub_id, nullptr, &extras);
if (start != nullptr) {
assert(extras.length() == expected_extra_count,
@@ -1009,7 +1009,6 @@ static void broadcast_first_and_last_needle(Register needle, Register needle_len
MacroAssembler *_masm) {
bool isUL = (ae == StrIntrinsicNode::UL);
bool isUU = (ae == StrIntrinsicNode::UU);
- bool isU = (isUU || isUL);
Label L_short;
// Always need needle broadcast to ymm registers
@@ -1776,8 +1775,6 @@ static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error,
bool isU = isUL || isUU; // At least one is UTF-16
const XMMRegister byte_1 = XMM_BYTE_1;
- int jmp_ndx = 0;
-
////////////////////////////////////////////////
// On entry to each case, the register state is:
//
diff --git a/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
index 2480e68e7b86..5d6286786683 100644
--- a/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
+++ b/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp
@@ -310,7 +310,8 @@ void DowncallLinker::StubGenerator::generate() {
__ block_comment("{ thread native2java");
__ restore_cpu_control_state_after_jni(rscratch1);
- __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_vm);
+ // change thread state
+ __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
// Force this write out before the read below
if (!UseSystemMemoryBarrier) {
@@ -325,9 +326,6 @@ void DowncallLinker::StubGenerator::generate() {
__ bind(L_after_safepoint_poll);
- // change thread state
- __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
-
__ block_comment("reguard stack check");
__ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_yellow_reserved_disabled);
__ jcc(Assembler::equal, L_reguard);
@@ -351,7 +349,7 @@ void DowncallLinker::StubGenerator::generate() {
}
__ mov(c_rarg0, r15_thread);
- runtime_call(_masm, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
+ runtime_call(_masm, CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans));
if (should_save_return_value) {
out_reg_spiller.generate_fill(_masm, out_spill_rsp_offset);
@@ -381,5 +379,5 @@ void DowncallLinker::StubGenerator::generate() {
}
//////////////////////////////////////////////////////////////////////////////
- __ flush();
+ // Code will be copied. No ICache sync required.
}
diff --git a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
index 0c640dde285c..af74e29fb6fe 100644
--- a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
+++ b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
@@ -34,14 +34,14 @@ source %{
#include "c2_intelJccErratum_x86.hpp"
#include "gc/z/zBarrierSetAssembler.hpp"
-static void z_color(MacroAssembler* masm, const MachNode* node, Register ref) {
+static void z_color(MacroAssembler* masm, Register ref) {
__ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
__ shlq(ref, barrier_Relocation::unpatched);
__ orq_imm32(ref, barrier_Relocation::unpatched);
__ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterOr);
}
-static void z_uncolor(MacroAssembler* masm, const MachNode* node, Register ref) {
+static void z_uncolor(MacroAssembler* masm, Register ref) {
__ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
__ shrq(ref, barrier_Relocation::unpatched);
}
@@ -53,7 +53,7 @@ static void z_keep_alive_load_barrier(MacroAssembler* masm, const MachNode* node
ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref);
__ jcc(Assembler::notEqual, *stub->entry());
- z_uncolor(masm, node, ref);
+ z_uncolor(masm, ref);
__ bind(*stub->continuation());
}
@@ -69,7 +69,7 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
return;
}
- z_uncolor(masm, node, ref);
+ z_uncolor(masm, ref);
if (node->barrier_data() == ZBarrierElided) {
return;
}
@@ -87,7 +87,7 @@ static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address
if (rnew_zaddress != noreg) {
// noreg means null; no need to color
__ movptr(rnew_zpointer, rnew_zaddress);
- z_color(masm, node, rnew_zpointer);
+ z_color(masm, rnew_zpointer);
}
} else {
bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
@@ -200,10 +200,10 @@ instruct zCompareAndExchangeP(indirect mem, no_rax_RegP newval, rRegP tmp, rax_R
assert_different_registers($oldval$$Register, $newval$$Register);
const Address mem_addr = Address($mem$$Register, 0);
z_store_barrier(masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
- z_color(masm, this, $oldval$$Register);
+ z_color(masm, $oldval$$Register);
__ lock();
__ cmpxchgptr($tmp$$Register, mem_addr);
- z_uncolor(masm, this, $oldval$$Register);
+ z_uncolor(masm, $oldval$$Register);
%}
ins_pipe(pipe_cmpxchg);
@@ -223,7 +223,7 @@ instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rax_
assert_different_registers($oldval$$Register, $mem$$Register);
const Address mem_addr = Address($mem$$Register, 0);
z_store_barrier(masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
- z_color(masm, this, $oldval$$Register);
+ z_color(masm, $oldval$$Register);
__ lock();
__ cmpxchgptr($tmp$$Register, mem_addr);
__ setcc(Assembler::equal, $res$$Register);
@@ -245,7 +245,7 @@ instruct zXChgP(indirect mem, rRegP newval, rRegP tmp, rFlagsReg cr) %{
z_store_barrier(masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
__ movptr($newval$$Register, $tmp$$Register);
__ xchgptr($newval$$Register, mem_addr);
- z_uncolor(masm, this, $newval$$Register);
+ z_uncolor(masm, $newval$$Register);
%}
ins_pipe(pipe_cmpxchg);
diff --git a/src/hotspot/cpu/x86/icache_x86.hpp b/src/hotspot/cpu/x86/icache_x86.hpp
index 92e4fbf15690..a8bdf7f0ec6a 100644
--- a/src/hotspot/cpu/x86/icache_x86.hpp
+++ b/src/hotspot/cpu/x86/icache_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,16 +28,6 @@
// Interface for updating the instruction cache. Whenever the VM modifies
// code, part of the processor instruction cache potentially has to be flushed.
-// On the x86, this is a no-op -- the I-cache is guaranteed to be consistent
-// after the next jump, and the VM never modifies instructions directly ahead
-// of the instruction fetch path.
-
-// [phh] It's not clear that the above comment is correct, because on an MP
-// system where the dcaches are not snooped, only the thread doing the invalidate
-// will see the update. Even in the snooped case, a memory fence would be
-// necessary if stores weren't ordered. Fortunately, they are on all known
-// x86 implementations.
-
class ICache : public AbstractICache {
public:
enum {
diff --git a/src/hotspot/cpu/x86/interpreterRT_x86_64.cpp b/src/hotspot/cpu/x86/interpreterRT_x86_64.cpp
index 8909df5b3f08..31b074c552be 100644
--- a/src/hotspot/cpu/x86/interpreterRT_x86_64.cpp
+++ b/src/hotspot/cpu/x86/interpreterRT_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -295,7 +295,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
__ lea(rax, ExternalAddress(Interpreter::result_handler(method()->result_type())));
__ ret(0);
- __ flush();
+ __ invalidate_icache();
}
diff --git a/src/hotspot/cpu/x86/jniFastGetField_x86_64.cpp b/src/hotspot/cpu/x86/jniFastGetField_x86_64.cpp
index 2c4d34c7cd54..caa167357f16 100644
--- a/src/hotspot/cpu/x86/jniFastGetField_x86_64.cpp
+++ b/src/hotspot/cpu/x86/jniFastGetField_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2004, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -120,7 +120,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
// tail call
__ jump (RuntimeAddress(slow_case_addr), rscratch1);
- __ flush ();
+ __ invalidate_icache();
return fast_entry;
}
@@ -208,7 +208,7 @@ address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) {
// tail call
__ jump (RuntimeAddress(slow_case_addr), rscratch1);
- __ flush ();
+ __ invalidate_icache();
return fast_entry;
}
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index dd6a6ed51ed4..20ca09968b89 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -5582,11 +5582,6 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
decode_klass_not_null(dst, tmp);
}
-void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
- load_klass(dst, src, tmp);
- movptr(dst, Address(dst, Klass::prototype_header_offset()));
-}
-
void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
assert(!UseCompactObjectHeaders, "not with compact headers");
assert_different_registers(src, tmp);
@@ -10626,8 +10621,8 @@ void MacroAssembler::fast_lock(Register basic_lock, Register obj, Register reg_r
// Try to lock. Transition lock bits 0b01 => 0b00
movptr(tmp, reg_rax);
- andptr(tmp, ~(int32_t)markWord::unlocked_value);
- orptr(reg_rax, markWord::unlocked_value);
+ andptr(tmp, ~(int32_t)markWord::lock_neutral_value);
+ orptr(reg_rax, markWord::lock_neutral_value);
if (Arguments::is_valhalla_enabled()) {
// Mask inline_type bit such that we go to the slow path if object is an inline type
andptr(reg_rax, ~((int) markWord::inline_type_bit_in_place));
@@ -10682,7 +10677,7 @@ void MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register tmp, L
#ifdef ASSERT
// Check header not unlocked (0b01).
Label not_unlocked;
- testptr(reg_rax, markWord::unlocked_value);
+ testptr(reg_rax, markWord::lock_neutral_value);
jcc(Assembler::zero, not_unlocked);
stop("fast_unlock already unlocked");
bind(not_unlocked);
@@ -10690,7 +10685,7 @@ void MacroAssembler::fast_unlock(Register obj, Register reg_rax, Register tmp, L
// Try to unlock. Transition lock bits 0b00 => 0b01
movptr(tmp, reg_rax);
- orptr(tmp, markWord::unlocked_value);
+ orptr(tmp, markWord::lock_neutral_value);
lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::equal, unlocked);
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 7f424966468e..b7c6b379c022 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -407,8 +407,6 @@ class MacroAssembler: public Assembler {
// stored using routines that take a jobject.
void store_heap_oop_null(Address dst);
- void load_prototype_header(Register dst, Register src, Register tmp);
-
void store_klass_gap(Register dst, Register src);
// This dummy is to prevent a call to store_heap_oop from
diff --git a/src/hotspot/cpu/x86/runtime_x86_64.cpp b/src/hotspot/cpu/x86/runtime_x86_64.cpp
index 5bf65299a0c5..cfd4e8e9df0c 100644
--- a/src/hotspot/cpu/x86/runtime_x86_64.cpp
+++ b/src/hotspot/cpu/x86/runtime_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -231,8 +231,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
// Jump to interpreter
__ ret(0);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
UncommonTrapBlob *ut_blob = UncommonTrapBlob::create(&buffer, oop_maps,
SimpleRuntimeFrame::framesize >> 1);
@@ -370,8 +369,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
__ jmp(r8);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Set exception blob
ExceptionBlob* ex_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
index 8d257565c939..0eb629f6ee8e 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
@@ -2021,7 +2021,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
assert(vep_offset != -1, "Must be set");
#endif
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod* nm = nmethod::new_native_nmethod(method,
compile_id,
masm->code(),
@@ -2050,7 +2050,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
in_sig_bt,
in_regs);
int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
- __ flush();
+ // Code will be copied. No ICache sync required.
int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually
return nmethod::new_native_nmethod(method,
compile_id,
@@ -2456,7 +2456,8 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
default : ShouldNotReachHere();
}
- __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_vm);
+ // change thread state
+ __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
// Force this write out before the read below
if (!UseSystemMemoryBarrier) {
@@ -2488,7 +2489,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ mov(r12, rsp); // remember sp
__ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
__ andptr(rsp, -16); // align stack as required by ABI
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
+ __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans)));
__ mov(rsp, r12); // restore sp
__ reinit_heapbase();
// Restore any method result value
@@ -2496,9 +2497,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ bind(Continue);
}
- // change thread state
- __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
-
if (method->is_object_wait0()) {
// Check preemption for Object.wait()
__ movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset()));
@@ -2712,7 +2710,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
- __ flush();
+ // Code will be copied. No ICache sync required.
nmethod *nm = nmethod::new_native_nmethod(method,
compile_id,
@@ -3064,8 +3062,7 @@ void SharedRuntime::generate_deopt_blob() {
// Jump to interpreter
__ ret(0);
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
_deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
_deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
@@ -3248,8 +3245,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
__ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
#endif
- // Make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// Fill-out other meta info
SafepointBlob* sp_blob = SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
@@ -3340,9 +3336,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
__ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
__ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
- // -------------
- // make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
// return the blob
// frame_size_words or bytes??
@@ -3864,7 +3858,7 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
__ bind(skip);
__ ret(0);
- __ flush();
+ // Code will be copied. No ICache sync required.
return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
}
@@ -4021,9 +4015,7 @@ RuntimeStub* SharedRuntime::generate_return_value_stub(address destination) {
__ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
__ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
- // -------------
- // make sure all code is generated
- masm->flush();
+ // Code will be copied. No ICache sync required.
RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, false);
AOTCodeCache::store_code_blob(*stub, AOTCodeEntry::SharedBlob, StubInfo::blob(id));
diff --git a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp
index 631d23801d33..38dfe1e171b0 100644
--- a/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp
@@ -955,8 +955,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
__ push(ltos);
// change thread state
- __ movl(Address(thread, JavaThread::thread_state_offset()),
- _thread_in_vm);
+ __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_Java);
// Force this write out before the read below
if (!UseSystemMemoryBarrier) {
@@ -987,15 +986,12 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
__ mov(r12, rsp); // remember sp (can only use r12 if not using call_VM)
__ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
__ andptr(rsp, -16); // align stack as required by ABI
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
+ __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::check_special_condition_for_native_trans)));
__ mov(rsp, r12); // restore sp
__ reinit_heapbase();
__ bind(Continue);
}
- // change thread state
- __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_Java);
-
// Check preemption for Object.wait()
Label not_preempted;
__ movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset()));
diff --git a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
index edc83fa7c562..eb5cda566060 100644
--- a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
+++ b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -351,7 +351,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
__ lea(c_rarg0, Address(rsp, frame_data_offset));
// stack already aligned
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::on_exit)));
- __ reinit_heapbase();
+ assert(!UseCompressedOops || !abi.is_volatile_reg(r12_heapbase), "r12_heapbase is not a volatile_reg!");
__ block_comment("} on_exit");
restore_callee_saved_registers(_masm, abi, reg_save_area_offset);
@@ -363,7 +363,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
//////////////////////////////////////////////////////////////////////////////
- _masm->flush();
+ // Code will be copied. No ICache sync required.
#ifndef PRODUCT
stringStream ss;
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 12cdadf026b6..61b012e74351 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -1576,12 +1576,18 @@ void VM_Version::get_processor_features() {
if (FLAG_IS_DEFAULT(UseUnalignedLoadStores)) {
FLAG_SET_DEFAULT(UseUnalignedLoadStores, true);
}
+ }
+
#ifdef COMPILER2
+ // Enable UseFPUForSpilling on Zen1/Zen2 (family 0x17) and Hygon Dhyana (family 0x18).
+ // On Zen3 (family 0x19) and beyond it should be default off.
+ if (cpu_family() >= 0x17 && cpu_family() < 0x19) {
if (supports_sse4_2() && FLAG_IS_DEFAULT(UseFPUForSpilling)) {
FLAG_SET_DEFAULT(UseFPUForSpilling, true);
}
-#endif
}
+#endif // COMPILER2
+
}
if (is_intel()) { // Intel cpus specific settings
diff --git a/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp b/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
index 81929e78d585..f47d1ad2ad5b 100644
--- a/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
+++ b/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -131,7 +131,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
address ame_addr = __ pc();
__ jmp( Address(rbx, entry_offset));
- masm->flush();
+ masm->invalidate_icache();
slop_bytes += index_dependent_slop; // add'l slop for size variance due to large itable offsets
bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, index_dependent_slop);
@@ -248,7 +248,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
// dirty work.
__ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
- masm->flush();
+ masm->invalidate_icache();
slop_bytes += index_dependent_slop; // add'l slop for size variance due to large itable offsets
bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, index_dependent_slop);
diff --git a/src/hotspot/cpu/zero/bytes_zero.hpp b/src/hotspot/cpu/zero/bytes_zero.hpp
deleted file mode 100644
index 15d0fc32650f..000000000000
--- a/src/hotspot/cpu/zero/bytes_zero.hpp
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2007, 2008, 2009 Red Hat, Inc.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_ZERO_BYTES_ZERO_HPP
-#define CPU_ZERO_BYTES_ZERO_HPP
-
-#include "memory/allStatic.hpp"
-
-typedef union unaligned {
- u4 u;
- u2 us;
- u8 ul;
-} __attribute__((packed)) unaligned;
-
-class Bytes: AllStatic {
- public:
- // Efficient reading and writing of unaligned unsigned data in
- // platform-specific byte ordering.
- static inline u2 get_native_u2(address p){
- unaligned *up = (unaligned *) p;
- return up->us;
- }
-
- static inline u4 get_native_u4(address p) {
- unaligned *up = (unaligned *) p;
- return up->u;
- }
-
- static inline u8 get_native_u8(address p) {
- unaligned *up = (unaligned *) p;
- return up->ul;
- }
-
- static inline void put_native_u2(address p, u2 x) {
- unaligned *up = (unaligned *) p;
- up->us = x;
- }
-
- static inline void put_native_u4(address p, u4 x) {
- unaligned *up = (unaligned *) p;
- up->u = x;
- }
-
- static inline void put_native_u8(address p, u8 x) {
- unaligned *up = (unaligned *) p;
- up->ul = x;
- }
-
- // Efficient reading and writing of unaligned unsigned data in Java
- // byte ordering (i.e. big-endian ordering).
-#ifdef VM_LITTLE_ENDIAN
- // Byte-order reversal is needed
- static inline u2 get_Java_u2(address p) {
- return (u2(p[0]) << 8) |
- (u2(p[1]) );
- }
- static inline u4 get_Java_u4(address p) {
- return (u4(p[0]) << 24) |
- (u4(p[1]) << 16) |
- (u4(p[2]) << 8) |
- (u4(p[3]) );
- }
- static inline u8 get_Java_u8(address p) {
- u4 hi, lo;
- hi = (u4(p[0]) << 24) |
- (u4(p[1]) << 16) |
- (u4(p[2]) << 8) |
- (u4(p[3]) );
- lo = (u4(p[4]) << 24) |
- (u4(p[5]) << 16) |
- (u4(p[6]) << 8) |
- (u4(p[7]) );
- return u8(lo) | (u8(hi) << 32);
- }
-
- static inline void put_Java_u2(address p, u2 x) {
- p[0] = x >> 8;
- p[1] = x;
- }
- static inline void put_Java_u4(address p, u4 x) {
- p[0] = x >> 24;
- p[1] = x >> 16;
- p[2] = x >> 8;
- p[3] = x;
- }
- static inline void put_Java_u8(address p, u8 x) {
- u4 hi, lo;
- lo = x;
- hi = x >> 32;
- p[0] = hi >> 24;
- p[1] = hi >> 16;
- p[2] = hi >> 8;
- p[3] = hi;
- p[4] = lo >> 24;
- p[5] = lo >> 16;
- p[6] = lo >> 8;
- p[7] = lo;
- }
-#else
- // No byte-order reversal is needed
- static inline u2 get_Java_u2(address p) {
- return get_native_u2(p);
- }
- static inline u4 get_Java_u4(address p) {
- return get_native_u4(p);
- }
- static inline u8 get_Java_u8(address p) {
- return get_native_u8(p);
- }
-
- static inline void put_Java_u2(address p, u2 x) {
- put_native_u2(p, x);
- }
- static inline void put_Java_u4(address p, u4 x) {
- put_native_u4(p, x);
- }
- static inline void put_Java_u8(address p, u8 x) {
- put_native_u8(p, x);
- }
-#endif // VM_LITTLE_ENDIAN
-};
-
-#endif // CPU_ZERO_BYTES_ZERO_HPP
diff --git a/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp b/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp
index 89a021ddb294..cc2f5ff0f2e2 100644
--- a/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp
+++ b/src/hotspot/cpu/zero/zeroInterpreter_zero.cpp
@@ -41,6 +41,7 @@
#include "runtime/handles.inline.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/jniHandles.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
#include "runtime/timer.hpp"
#include "runtime/timerTrace.hpp"
#include "utilities/debug.hpp"
@@ -427,18 +428,16 @@ int ZeroInterpreter::native_entry(Method* method, intptr_t UNUSED, TRAPS) {
// ThreadStateTransition::transition_from_native() cannot be used
// here because it does not check for asynchronous exceptions.
// We have to manage the transition ourself.
- thread->set_thread_state_fence(_thread_in_vm);
+ thread->set_thread_state_fence(_thread_in_Java);
// Handle safepoint operations, pending suspend requests,
// and pending asynchronous exceptions.
if (SafepointMechanism::should_process(thread) ||
thread->has_special_condition_for_native_trans()) {
- JavaThread::check_special_condition_for_native_trans(thread);
+ SharedRuntime::check_special_condition_for_native_trans(thread);
CHECK_UNHANDLED_OOPS_ONLY(thread->clear_unhandled_oops());
}
- // Finally we can change the thread state to _thread_in_Java.
- thread->set_thread_state(_thread_in_Java);
fixup_after_potential_safepoint();
// Notify the stack watermarks machinery that we are unwinding.
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
index 1c183a9bbab5..18c1c1eb26f7 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
@@ -670,7 +670,7 @@ bool CgroupSubsystem::active_processor_count(int (*cpu_bound_func)(), double& va
*
* return:
* false if retrieving the value failed
- * true if retrieving the value was successfull and the value was
+ * true if retrieving the value was successful and the value was
* set in the 'value' reference.
*/
bool CgroupSubsystem::memory_limit_in_bytes(physical_memory_size_type upper_bound,
diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
index 3ab81697280a..5c84669776db 100644
--- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
@@ -389,7 +389,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if ((sig == SIGSEGV) || (sig == SIGBUS)) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr = JNI_FastGetField::find_slowcase_pc(pc);
if (addr != (address)-1) {
stub = addr;
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index 6f31bc284e35..90a61b7e8b63 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -374,7 +374,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if ((sig == SIGSEGV) || (sig == SIGBUS)) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr = JNI_FastGetField::find_slowcase_pc(pc);
if (addr != (address)-1) {
stub = addr;
diff --git a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
index 8668f20e371d..a380c4f316cf 100644
--- a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
+++ b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
@@ -433,7 +433,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if ((sig == SIGSEGV) || (sig == SIGBUS)) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr = JNI_FastGetField::find_slowcase_pc(pc);
if (addr != (address)-1) {
stub = addr;
diff --git a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
index a089d5981ca5..c9a537feacb5 100644
--- a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
+++ b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright 2007, 2008, 2009, 2010 Red Hat, Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -145,14 +145,6 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
ShouldNotCallThis();
}
- // jni_fast_GetField can trap at certain pc's if a GC
- // kicks in and the heap gets shrunk before the field access.
- /*if (sig == SIGSEGV || sig == SIGBUS) {
- address addr = JNI_FastGetField::find_slowcase_pc(pc);
- if (addr != (address)-1) {
- stub = addr;
- }
- }*/
}
return false;
diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
index 67e0569bf31d..216e6b729909 100644
--- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
@@ -302,7 +302,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if ((sig == SIGSEGV) || (sig == SIGBUS)) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr = JNI_FastGetField::find_slowcase_pc(pc);
if (addr != (address)-1) {
stub = addr;
diff --git a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
index 41a4dbea3842..3f158ad9cee4 100644
--- a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
+++ b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
@@ -376,7 +376,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if (sig == SIGSEGV || sig == SIGBUS) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr = JNI_FastGetField::find_slowcase_pc(pc);
if (addr != (address)-1) {
stub = addr;
diff --git a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
index 518519937306..859adcd8b4ee 100644
--- a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
+++ b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
@@ -405,7 +405,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if ((sig == SIGSEGV) || (sig == SIGBUS)) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr = JNI_FastGetField::find_slowcase_pc(pc);
if (addr != (address)-1) {
stub = addr;
diff --git a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
index 7634ac0fd379..d059046c57fb 100644
--- a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
@@ -283,7 +283,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if ((sig == SIGSEGV) || (sig == SIGBUS)) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr_slow = JNI_FastGetField::find_slowcase_pc(pc);
if (addr_slow != (address)-1) {
stub = addr_slow;
diff --git a/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp b/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
index 9276d1d744b0..cbb06d2d07d8 100644
--- a/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
+++ b/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
@@ -349,7 +349,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if ((sig == SIGSEGV) || (sig == SIGBUS)) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr = JNI_FastGetField::find_slowcase_pc(pc);
if (addr != (address)-1) {
stub = addr;
diff --git a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
index 25ee449d8b10..261739d8c35f 100644
--- a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
+++ b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
@@ -297,7 +297,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
// jni_fast_GetField can trap at certain pc's if a GC kicks in
// and the heap gets shrunk before the field access.
- if ((sig == SIGSEGV) || (sig == SIGBUS)) {
+ if (stub == nullptr && ((sig == SIGSEGV) || (sig == SIGBUS))) {
address addr = JNI_FastGetField::find_slowcase_pc(pc);
if (addr != (address)-1) {
stub = addr;
diff --git a/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp b/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
index ee9c5e2dfb23..6cad3e9fff95 100644
--- a/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
+++ b/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
* Copyright 2007, 2008, 2009, 2010 Red Hat, Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -260,14 +260,6 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
ShouldNotCallThis();
}
- // jni_fast_GetField can trap at certain pc's if a GC
- // kicks in and the heap gets shrunk before the field access.
- /*if (sig == SIGSEGV || sig == SIGBUS) {
- address addr = JNI_FastGetField::find_slowcase_pc(pc);
- if (addr != (address)-1) {
- stub = addr;
- }
- }*/
}
return false; // Fatal error
diff --git a/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp b/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
index 9238043f7a41..a92e000950fc 100644
--- a/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
+++ b/src/hotspot/os_cpu/windows_aarch64/atomicAccess_windows_aarch64.hpp
@@ -29,12 +29,6 @@
#include
#include
-// As per atomicAccess.hpp all read-modify-write operations have to provide two-way
-// barriers semantics. The memory_order parameter is ignored - we always provide
-// the strongest/most-conservative ordering
-//
-// For AARCH64 we add explicit barriers in the stubs.
-
template
struct AtomicAccess::PlatformAdd {
template
@@ -56,9 +50,26 @@ struct AtomicAccess::PlatformAdd {
I add_value, \
atomic_memory_order order) const { \
STATIC_ASSERT(sizeof(IntrinsicType) == sizeof(D)); \
- return PrimitiveConversions::cast( \
- IntrinsicName(reinterpret_cast(dest), \
- PrimitiveConversions::cast(add_value))); \
+ IntrinsicType volatile* d = \
+ reinterpret_cast(dest); \
+ IntrinsicType inc = \
+ PrimitiveConversions::cast(add_value); \
+ IntrinsicType result; \
+ switch (order) { \
+ case memory_order_relaxed: \
+ result = _##IntrinsicName##_nf(d, inc); break; \
+ case memory_order_acquire: \
+ result = _##IntrinsicName##_acq(d, inc); break; \
+ case memory_order_release: \
+ result = _##IntrinsicName##_rel(d, inc); break; \
+ case memory_order_conservative: \
+ result = _##IntrinsicName(d, inc); \
+ OrderAccess::fence(); \
+ break; \
+ default: \
+ result = _##IntrinsicName(d, inc); break; \
+ } \
+ return PrimitiveConversions::cast(result); \
}
DEFINE_INTRINSIC_ADD(InterlockedAdd, long)
@@ -78,9 +89,26 @@ struct AtomicAccess::PlatformXchg<1> : AtomicAccess::XchgUsingCmpxchg<1> {};
STATIC_ASSERT(sizeof(IntrinsicType) == sizeof(T)); \
STATIC_ASSERT(sizeof(IntrinsicType) == 4 || \
sizeof(IntrinsicType) == 8); \
- return PrimitiveConversions::cast( \
- IntrinsicName(reinterpret_cast(dest), \
- PrimitiveConversions::cast(exchange_value))); \
+ IntrinsicType volatile* d = \
+ reinterpret_cast(dest); \
+ IntrinsicType xchg = \
+ PrimitiveConversions::cast(exchange_value); \
+ IntrinsicType result; \
+ switch (order) { \
+ case memory_order_relaxed: \
+ result = _##IntrinsicName##_nf(d, xchg); break; \
+ case memory_order_acquire: \
+ result = _##IntrinsicName##_acq(d, xchg); break; \
+ case memory_order_release: \
+ result = _##IntrinsicName##_rel(d, xchg); break; \
+ case memory_order_conservative: \
+ result = _##IntrinsicName(d, xchg); \
+ OrderAccess::fence(); \
+ break; \
+ default: \
+ result = _##IntrinsicName(d, xchg); break; \
+ } \
+ return PrimitiveConversions::cast(result); \
}
DEFINE_INTRINSIC_XCHG(InterlockedExchange, long)
@@ -90,7 +118,9 @@ DEFINE_INTRINSIC_XCHG(InterlockedExchange64, __int64)
// Note: the order of the parameters is different between
// AtomicAccess::PlatformCmpxchg<*>::operator() and the
-// InterlockedCompareExchange* API.
+// _InterlockedCompareExchange* intrinsics:
+// HotSpot: (dest, compare_value, exchange_value)
+// MSVC: (dest, exchange_value, compare_value)
#define DEFINE_INTRINSIC_CMPXCHG(IntrinsicName, IntrinsicType) \
template<> \
@@ -100,16 +130,78 @@ DEFINE_INTRINSIC_XCHG(InterlockedExchange64, __int64)
T exchange_value, \
atomic_memory_order order) const { \
STATIC_ASSERT(sizeof(IntrinsicType) == sizeof(T)); \
- return PrimitiveConversions::cast( \
- IntrinsicName(reinterpret_cast(dest), \
- PrimitiveConversions::cast(exchange_value), \
- PrimitiveConversions::cast(compare_value))); \
+ IntrinsicType volatile* d = \
+ reinterpret_cast(dest); \
+ IntrinsicType xchg = \
+ PrimitiveConversions::cast(exchange_value); \
+ IntrinsicType cmp = \
+ PrimitiveConversions::cast(compare_value); \
+ IntrinsicType result; \
+ switch (order) { \
+ case memory_order_relaxed: \
+ result = _##IntrinsicName##_nf(d, xchg, cmp); break; \
+ case memory_order_acquire: \
+ result = _##IntrinsicName##_acq(d, xchg, cmp); break; \
+ case memory_order_release: \
+ result = _##IntrinsicName##_rel(d, xchg, cmp); break; \
+ case memory_order_conservative: \
+ result = _##IntrinsicName(d, xchg, cmp); \
+ OrderAccess::fence(); \
+ break; \
+ default: \
+ result = _##IntrinsicName(d, xchg, cmp); break; \
+ } \
+ return PrimitiveConversions::cast(result); \
}
-DEFINE_INTRINSIC_CMPXCHG(_InterlockedCompareExchange8, char) // Use the intrinsic as InterlockedCompareExchange8 does not exist
+DEFINE_INTRINSIC_CMPXCHG(InterlockedCompareExchange8, char) // Use the intrinsic as InterlockedCompareExchange8 does not exist
DEFINE_INTRINSIC_CMPXCHG(InterlockedCompareExchange, long)
DEFINE_INTRINSIC_CMPXCHG(InterlockedCompareExchange64, __int64)
#undef DEFINE_INTRINSIC_CMPXCHG
+#define DEFINE_ORDERED_LOAD(Size, Name, Type) \
+ template<> \
+ struct AtomicAccess::PlatformOrderedLoad { \
+ template \
+ T operator()(const volatile T* p) const { \
+ T* noconst_ptr = const_cast(p); \
+ unsigned Type value = Name(reinterpret_cast(noconst_ptr)); \
+ return PrimitiveConversions::cast(value); \
+ } \
+ };
+
+DEFINE_ORDERED_LOAD(1, __ldar8, __int8)
+DEFINE_ORDERED_LOAD(2, __ldar16, __int16)
+DEFINE_ORDERED_LOAD(4, __ldar32, __int32)
+DEFINE_ORDERED_LOAD(8, __ldar64, __int64)
+
+#undef DEFINE_ORDERED_LOAD
+
+#define DEFINE_ORDERED_STORE(Size, Name, Type) \
+ template<> \
+ struct AtomicAccess::PlatformOrderedStore { \
+ template \
+ void operator()(volatile T* p, T v) const { \
+ Name(reinterpret_cast(p), PrimitiveConversions::cast(v)); \
+ } \
+ };
+
+DEFINE_ORDERED_STORE(1, __stlr8, __int8)
+DEFINE_ORDERED_STORE(2, __stlr16, __int16)
+DEFINE_ORDERED_STORE(4, __stlr32, __int32)
+DEFINE_ORDERED_STORE(8, __stlr64, __int64)
+
+#undef DEFINE_ORDERED_STORE
+
+template
+struct AtomicAccess::PlatformOrderedStore
+{
+ template
+ void operator()(volatile T* p, T v) const {
+ PlatformOrderedStore()(p, v);
+ OrderAccess::fence();
+ }
+};
+
#endif // OS_CPU_WINDOWS_AARCH64_ATOMICACCESS_WINDOWS_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
index e3291d3a6ca9..31ce1b41f39d 100644
--- a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
+++ b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -134,7 +134,7 @@ bool os::win32::register_code_area(char *low, char *high) {
pDCD = (pDynamicCodeData) masm->pc();
masm->jump(RuntimeAddress((address)&HandleExceptionFromCodeCache), rscratch1);
- masm->flush();
+ masm->invalidate_icache();
// Create an Unwind Structure specifying no unwind info
// other than an Exception Handler
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index 8abaa62982bb..167f19a8fde7 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -3905,15 +3905,14 @@ void MatchNode::count_commutative_op(int& count) {
"MaxI","MinI","MaxHF","MinHF","MaxF","MinF","MaxD","MinD",
"MulI","MulL","MulHF","MulF","MulD",
"OrI","OrL",
- "XorI","XorL"
- "UMax","UMin"
+ "XorI","XorL",
};
static const char *commut_vector_op_list[] = {
"AddVB", "AddVS", "AddVI", "AddVL", "AddVHF", "AddVF", "AddVD",
"MulVB", "MulVS", "MulVI", "MulVL", "MulVHF", "MulVF", "MulVD",
"AndV", "OrV", "XorV", "AndVMask", "OrVMask", "XorVMask",
- "MaxVHF", "MinVHF", "MaxV", "MinV", "UMax","UMin"
+ "MaxVHF", "MinVHF", "MaxV", "MinV", "UMaxV", "UMinV",
};
if (_lChild && _rChild && (_lChild->_lChild || _rChild->_lChild)) {
diff --git a/src/hotspot/share/asm/assembler.cpp b/src/hotspot/share/asm/assembler.cpp
index 9e342d23afd8..5de8a6dfb10b 100644
--- a/src/hotspot/share/asm/assembler.cpp
+++ b/src/hotspot/share/asm/assembler.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -104,7 +104,7 @@ void AbstractAssembler::end_a_const(CodeSection* cs) {
set_code_section(cs);
}
-void AbstractAssembler::flush() {
+void AbstractAssembler::invalidate_icache() {
ICache::invalidate_range(addr_at(0), offset());
}
diff --git a/src/hotspot/share/asm/assembler.hpp b/src/hotspot/share/asm/assembler.hpp
index bfe785fb94e9..b24e65e12621 100644
--- a/src/hotspot/share/asm/assembler.hpp
+++ b/src/hotspot/share/asm/assembler.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -311,8 +311,8 @@ class AbstractAssembler : public ResourceObj {
// Creation
AbstractAssembler(CodeBuffer* code);
- // ensure buf contains all code (call this before using/copying the code)
- void flush();
+ // Invalidate ICache after writing code to its final location.
+ void invalidate_icache();
void emit_int8( int x1) { code_section()->emit_int8(narrow_cast(x1)); }
diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp
index d56ab27f0253..549b2cb20639 100644
--- a/src/hotspot/share/asm/codeBuffer.hpp
+++ b/src/hotspot/share/asm/codeBuffer.hpp
@@ -28,11 +28,11 @@
#include "code/oopRecorder.hpp"
#include "code/relocInfo.hpp"
#include "compiler/compiler_globals.hpp"
+#include "nmt/memTag.hpp"
#include "runtime/os.hpp"
#include "utilities/align.hpp"
#include "utilities/debug.hpp"
#include "utilities/growableArray.hpp"
-#include "utilities/linkedlist.hpp"
#include "utilities/macros.hpp"
#include "utilities/resizableHashTable.hpp"
@@ -546,7 +546,7 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) {
SECT_LIMIT, SECT_NONE = -1
};
- typedef LinkedListImpl Offsets;
+ typedef GrowableArrayCHeap Offsets;
typedef ResizeableHashTable SharedTrampolineRequests;
private:
diff --git a/src/hotspot/share/c1/c1_Compilation.hpp b/src/hotspot/share/c1/c1_Compilation.hpp
index 9779b3330b9a..605d3e4c353b 100644
--- a/src/hotspot/share/c1/c1_Compilation.hpp
+++ b/src/hotspot/share/c1/c1_Compilation.hpp
@@ -258,7 +258,10 @@ class Compilation: public StackObj {
}
bool profile_array_accesses() {
return env()->comp_level() == CompLevel_full_profile &&
- C1UpdateMethodData;
+ C1UpdateMethodData && MethodData::profile_array_accesses();
+ }
+ bool profile_acmp() {
+ return is_profiling() && profile_branches() && MethodData::profile_acmp();
}
// will compilation make optimistic assumptions that might lead to
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp
index 115751b129a1..a60a7619f208 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.cpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp
@@ -1390,7 +1390,7 @@ void GraphBuilder::if_node(Value x, If::Condition cond, Value y, ValueStack* sta
}
}
if ((stream()->cur_bc() == Bytecodes::_if_acmpeq || stream()->cur_bc() == Bytecodes::_if_acmpne) &&
- is_profiling() && profile_branches()) {
+ profile_acmp()) {
compilation()->set_would_profile(true);
append(new ProfileACmpTypes(method(), bci(), x, y));
}
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.hpp b/src/hotspot/share/c1/c1_GraphBuilder.hpp
index 2a8517905013..5fcde20964ba 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.hpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.hpp
@@ -430,6 +430,7 @@ class GraphBuilder {
bool profile_arguments() { return _compilation->profile_arguments(); }
bool profile_return() { return _compilation->profile_return(); }
bool profile_array_accesses(){ return _compilation->profile_array_accesses();}
+ bool profile_acmp() { return _compilation->profile_acmp(); }
Values* args_list_for_profiling(ciMethod* target, int& start, bool may_have_receiver);
Values* collect_args_for_profiling(Values* args, ciMethod* target, bool may_have_receiver);
diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp
index 19b4d9ae203c..f61daa556699 100644
--- a/src/hotspot/share/c1/c1_LIRGenerator.cpp
+++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp
@@ -1964,7 +1964,7 @@ void LIRGenerator::do_StoreIndexed(StoreIndexed* x) {
}
}
- if (GenerateArrayStoreCheck && needs_store_check) {
+ if (needs_store_check) {
CodeEmitInfo* store_check_info = new CodeEmitInfo(range_check_info);
array_store_check(value.result(), array.result(), store_check_info, x->profiled_method(), x->profiled_bci());
}
@@ -3097,7 +3097,6 @@ void LIRGenerator::do_Base(Base* x) {
__ std_entry(LIR_OprFact::illegalOpr);
// Emit moves from physical registers / stack slots to virtual registers
CallingConvention* args = compilation()->frame_map()->incoming_arguments();
- IRScope* irScope = compilation()->hir()->top_scope();
int java_index = 0;
for (int i = 0; i < args->length(); i++) {
LIR_Opr src = args->at(i);
@@ -3445,7 +3444,7 @@ void LIRGenerator::do_RuntimeCall(address routine, Intrinsic* x) {
assert(x->number_of_arguments() == 0, "wrong type");
// Enforce computation of _reserved_argument_area_size which is required on some platforms.
BasicTypeList signature;
- CallingConvention* cc = frame_map()->c_calling_convention(&signature);
+ frame_map()->c_calling_convention(&signature);
LIR_Opr reg = result_register_for(x->type());
__ call_runtime_leaf(routine, getThreadTemp(),
reg, new LIR_OprList());
diff --git a/src/hotspot/share/c1/c1_LinearScan.cpp b/src/hotspot/share/c1/c1_LinearScan.cpp
index 70dbad6d91ba..3cb30eab55db 100644
--- a/src/hotspot/share/c1/c1_LinearScan.cpp
+++ b/src/hotspot/share/c1/c1_LinearScan.cpp
@@ -744,7 +744,9 @@ void LinearScan::compute_global_live_sets() {
// Perform a backward dataflow analysis to compute live_out and live_in for each block.
// The loop is executed until a fixpoint is reached (no changes in an iteration)
// Exception handlers must be processed because not all live values are
- // present in the state array, e.g. because of global value numbering
+ // present in the state array, e.g. because of global value numbering.
+ // Exception handler live_in information is also used by build_intervals() to
+ // account for local liveness holes in exception-throwing blocks.
do {
change_occurred = false;
@@ -1363,6 +1365,55 @@ void LinearScan::build_intervals() {
add_use(opr, block_from, op_id, use_kind_of_input_operand(op, opr));
}
+ // If the visited operation 'op' may branch into an exception handler
+ // block 'handler', add all live-in registers of 'handler' as virtual uses
+ // of 'op'. This ensures that all such registers are live into 'op', which
+ // might otherwise not happen if 'op' is scheduled within a hole of their
+ // corresponding intervals, as in the following
+ // post-compute_global_live_sets() scenario:
+ //
+ // R
+ // | block:
+ // | live-in: {.., R, ..}
+ // | ..
+ // - kill R
+ // ..
+ // op: branch [BE] .. // may branch into 'handler'
+ // ..
+ // - def R
+ // | ..
+ // | branch into 'block'
+ // | live-out: {.., R, ..}
+ // |
+ // | handler:
+ // | live-in: {.., R, ..}
+ // | ..
+ //
+ // Normally, the debug information generation logic below will add
+ // registers such as R in the above scenario as uses of 'op', but this
+ // might not happen if the corresponding virtual register used within
+ // 'handler' is replaced by another one in an earlier optimization pass.
+ // An example of such a replacement is GraphBuilder::shift_op().
+ if (compilation()->has_exception_handlers() && op_id != -1 && has_info(op_id)) {
+ XHandlers* xhandlers = visitor.all_xhandler();
+ for (int k = 0; k < xhandlers->length(); k++) {
+ BlockBegin* handler = xhandlers->handler_at(k)->entry_block();
+ auto add_virtual_use_to_op = [&](BitMap::idx_t index) {
+ int reg = static_cast(index);
+ // The T_ILLEGAL type is used by add_use() as a sentinel value
+ // indicating the type is unknown (rather than illegal) so that the
+ // type of the interval corresponding to reg is not updated. The use
+ // is extended beyond 'op' (to = op_id + 1) so that liveness is
+ // preserved across possible registers killed by 'op' (e.g.
+ // caller-saved registers if 'op' is a call).
+ TRACE_LINEAR_SCAN(2, tty->print_cr(" use [R%d] from %d to %d (%d)",
+ reg, block_from, op_id + 1, noUse));
+ add_use(reg, block_from, op_id + 1, noUse, T_ILLEGAL);
+ };
+ handler->live_in().iterate(add_virtual_use_to_op);
+ }
+ }
+
// Add uses of live locals from interpreter's point of view for proper
// debug information generation
// Treat these operands as temp values (if the life range is extended
diff --git a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
index 1adf99cce61e..860a8c5470f3 100644
--- a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
+++ b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -124,7 +124,6 @@ void RangeCheckEliminator::Visitor::do_LogicOp(LogicOp *lo) {
void RangeCheckEliminator::Visitor::do_Phi(Phi *phi) {
if (!phi->type()->as_IntType() && !phi->type()->as_ObjectType()) return;
- BlockBegin *block = phi->block();
int op_count = phi->operand_count();
bool has_upper = true;
bool has_lower = true;
@@ -220,7 +219,6 @@ void RangeCheckEliminator::Visitor::do_ArithmeticOp(ArithmeticOp *ao) {
if (ao->op() == Bytecodes::_irem) {
Bound* x_bound = _rce->get_bound(x);
- Bound* y_bound = _rce->get_bound(y);
if (x_bound->lower() >= 0 && x_bound->lower_instr() == nullptr && y->as_ArrayLength() != nullptr) {
_bound = new Bound(0, nullptr, -1, y);
} else if (x_bound->has_lower() && x_bound->lower() >= 0 && y->type()->as_IntConstant() &&
@@ -872,7 +870,6 @@ void RangeCheckEliminator::process_access_indexed(BlockBegin *loop_header, Block
}
// Lower instruction
- Value index_instr = ai->index();
Value lower_instr = index_bound->lower_instr();
if (!loop_invariant(loop_header, lower_instr)) {
TRACE_RANGE_CHECK_ELIMINATION(
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 118758f3ea49..d4b1f3a111e7 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -236,8 +236,8 @@ CodeBlob* Runtime1::generate_blob(BufferBlob* buffer_blob, StubId id, const char
// align so printing shows nop's instead of random code at the end (SimpleStubs are aligned)
sasm->align(BytesPerWord);
- // make sure all code is in code buffer
- sasm->flush();
+
+ // Code will be copied. No ICache sync required.
frame_size = sasm->frame_size();
must_gc_arguments = sasm->must_gc_arguments();
diff --git a/src/hotspot/share/c1/c1_globals.hpp b/src/hotspot/share/c1/c1_globals.hpp
index fad45f747b22..e69883bc436d 100644
--- a/src/hotspot/share/c1/c1_globals.hpp
+++ b/src/hotspot/share/c1/c1_globals.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -238,9 +238,6 @@
develop(bool, UseSlowPath, false, \
"For debugging: test slow cases by always using them") \
\
- develop(bool, GenerateArrayStoreCheck, true, \
- "Generates code for array store checks") \
- \
develop(bool, PrintBailouts, false, \
"Print bailout and its reason") \
\
diff --git a/src/hotspot/share/cds/aotClassLocation.cpp b/src/hotspot/share/cds/aotClassLocation.cpp
index 48b91960ce77..e33d4b6ff24e 100644
--- a/src/hotspot/share/cds/aotClassLocation.cpp
+++ b/src/hotspot/share/cds/aotClassLocation.cpp
@@ -39,8 +39,10 @@
#include "memory/metaspaceClosure.hpp"
#include "memory/resourceArea.hpp"
#include "oops/array.hpp"
+#include "oops/klass.inline.hpp"
#include "oops/objArrayKlass.hpp"
#include "runtime/arguments.hpp"
+#include "runtime/handles.inline.hpp"
#include "utilities/classpathStream.hpp"
#include "utilities/formatBuffer.hpp"
#include "utilities/stringUtils.hpp"
@@ -279,7 +281,8 @@ AOTClassLocation* AOTClassLocation::allocate(JavaThread* current, const char* pa
}
assert(*(cs->manifest() + cs->manifest_length()) == '\0', "should be nul-terminated");
- if (strstr(cs->manifest(), "Multi-Release: true") != nullptr) {
+ const char* multi_release = cs->get_attr("Multi-Release: ");
+ if (multi_release != nullptr && strcasecmp(multi_release, "true") == 0) {
cs->_is_multi_release_jar = true;
}
@@ -321,7 +324,7 @@ char* AOTClassLocation::read_manifest(JavaThread* current, const char* path, siz
}
// The result is resource allocated.
-char* AOTClassLocation::get_cpattr() const {
+char* AOTClassLocation::get_attr(const char* tag) const {
if (_manifest_length == 0) {
return nullptr;
}
@@ -337,7 +340,6 @@ char* AOTClassLocation::get_cpattr() const {
// Remove all new-line continuation (remove all "\n " substrings)
StringUtils::replace_no_expand(buf, "\n ", "");
- const char* tag = "Class-Path: ";
size_t tag_len = strlen(tag);
char* found = nullptr;
char* line_start = buf;
@@ -351,7 +353,13 @@ char* AOTClassLocation::get_cpattr() const {
// JAR spec require the manifest file to be terminated by a new line.
break;
}
- if (strncmp(tag, line_start, tag_len) == 0) {
+
+ if (line_start == line_end) {
+ break;
+ }
+
+ // Attribute names are case insensitive
+ if (strncasecmp(tag, line_start, tag_len) == 0) {
if (found != nullptr) {
// Same behavior as jdk/src/share/classes/java/util/jar/Attributes.java
// If duplicated entries are found, the last one is used.
@@ -370,6 +378,11 @@ char* AOTClassLocation::get_cpattr() const {
return found;
}
+// The result is resource allocated.
+char* AOTClassLocation::get_cpattr() const {
+ return get_attr("Class-Path: ");
+}
+
AOTClassLocation* AOTClassLocation::write_to_archive() const {
AOTClassLocation* archived_copy = (AOTClassLocation*)ArchiveBuilder::ro_region_alloc(total_size());
memcpy((char*)archived_copy, (char*)this, total_size());
@@ -719,7 +732,9 @@ bool AOTClassLocationConfig::is_valid_classpath_index(int classpath_index, Insta
const char* const class_name = ik->name()->as_C_string();
const char* const file_name = ClassLoader::file_name_for_class_name(class_name,
ik->name()->utf8_length());
- if (!zip->has_entry(current, file_name)) {
+ Handle class_loader(current, ik->class_loader());
+ const AOTClassLocation* cl = AOTClassLocationConfig::class_location_at(classpath_index);
+ if (!zip->has_entry(current, file_name, class_loader, cl->is_multi_release_jar())) {
aot_log_warning(aot)("class %s cannot be archived because it was not defined from %s as claimed",
class_name, zip->name());
return false;
diff --git a/src/hotspot/share/cds/aotClassLocation.hpp b/src/hotspot/share/cds/aotClassLocation.hpp
index 771f4951671c..9a64e0deffae 100644
--- a/src/hotspot/share/cds/aotClassLocation.hpp
+++ b/src/hotspot/share/cds/aotClassLocation.hpp
@@ -106,6 +106,7 @@ class AOTClassLocation {
// Only boot/app classpaths can contain unnamed module
bool has_unnamed_module() const { return from_boot_classpath() || from_app_classpath(); }
+ char* get_attr(const char* tag) const;
char* get_cpattr() const;
AOTClassLocation* write_to_archive() const;
diff --git a/src/hotspot/share/cds/aotMappedHeapWriter.cpp b/src/hotspot/share/cds/aotMappedHeapWriter.cpp
index cfa6c460303e..8db7c6f3ff2d 100644
--- a/src/hotspot/share/cds/aotMappedHeapWriter.cpp
+++ b/src/hotspot/share/cds/aotMappedHeapWriter.cpp
@@ -725,39 +725,26 @@ template void AOTMappedHeapWriter::mark_oop_pointer(T* buffered_add
oopmap->set_bit(idx);
}
-void AOTMappedHeapWriter::update_header_for_requested_obj(oop requested_obj, oop src_obj, Klass* src_klass) {
+void AOTMappedHeapWriter::update_header_for_requested_obj(oop requested_obj, oop src_obj, Klass* src_klass) {
narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(src_klass);
address buffered_addr = requested_addr_to_buffered_addr(cast_from_oop(requested_obj));
- oop fake_oop = cast_to_oop(buffered_addr);
- if (UseCompactObjectHeaders) {
- markWord prototype_header = src_klass->prototype_header().set_narrow_klass(nk);
- fake_oop->set_mark(prototype_header);
- } else {
- fake_oop->set_narrow_klass(nk);
- }
+ markWord mw = Arguments::is_valhalla_enabled() ? src_klass->prototype_header() : markWord::prototype();
+ oopDesc* fake_oop = (oopDesc*)buffered_addr;
- if (src_obj == nullptr) {
- return;
- }
// We need to retain the identity_hash, because it may have been used by some hashtables
// in the shared heap.
- if (!src_obj->fast_no_hash_check() && (!(Arguments::is_valhalla_enabled() && src_obj->mark().is_inline_type()))) {
+ if (src_obj != nullptr && !src_obj->is_inline_type() && src_obj->has_identity_hash()) {
intptr_t src_hash = src_obj->identity_hash();
- if (UseCompactObjectHeaders) {
- fake_oop->set_mark(fake_oop->mark().copy_set_hash(src_hash));
- } else if (Arguments::is_valhalla_enabled()) {
- fake_oop->set_mark(src_klass->prototype_header().copy_set_hash(src_hash));
- } else {
- fake_oop->set_mark(markWord::prototype().copy_set_hash(src_hash));
- }
- assert(fake_oop->mark().is_unlocked(), "sanity");
+ mw = mw.copy_set_hash(src_hash);
+ }
- DEBUG_ONLY(intptr_t archived_hash = fake_oop->identity_hash());
- assert(src_hash == archived_hash, "Different hash codes: original " INTPTR_FORMAT ", archived " INTPTR_FORMAT, src_hash, archived_hash);
+ if (UseCompactObjectHeaders) {
+ fake_oop->set_mark(mw.set_narrow_klass(nk));
+ } else {
+ fake_oop->set_mark(mw);
+ fake_oop->set_narrow_klass(nk);
}
- // Strip age bits.
- fake_oop->set_mark(fake_oop->mark().set_age(0));
}
class AOTMappedHeapWriter::EmbeddedOopRelocator: public BasicOopIterateClosure {
diff --git a/src/hotspot/share/cds/aotStreamedHeapWriter.cpp b/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
index 8009e63e63d0..a8a5a5beb380 100644
--- a/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
+++ b/src/hotspot/share/cds/aotStreamedHeapWriter.cpp
@@ -371,7 +371,7 @@ template void AOTStreamedHeapWriter::map_oop_field_in_buffer(oop ob
void AOTStreamedHeapWriter::update_header_for_buffered_addr(address buffered_addr, oop src_obj, Klass* src_klass) {
narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(src_klass);
- markWord mw = Arguments::enable_preview() ? src_klass->prototype_header() : markWord::prototype();
+ markWord mw = Arguments::is_valhalla_enabled() ? src_klass->prototype_header() : markWord::prototype();
oopDesc* fake_oop = (oopDesc*)buffered_addr;
// We need to retain the identity_hash, because it may have been used by some hashtables
diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp
index 7261fb1a5c6c..af0ae2c7a44a 100644
--- a/src/hotspot/share/cds/archiveUtils.cpp
+++ b/src/hotspot/share/cds/archiveUtils.cpp
@@ -389,7 +389,11 @@ char* DumpRegion::allocate_metaspace_obj(size_t num_bytes, address src, Metaspac
assert(read_only == false, "only gaps in RW region are reusable");
char* gap_bottom = top();
char* gap_top = align_up(gap_bottom + RuntimeClassInfoPtrSize, alignment) - RuntimeClassInfoPtrSize;
- size_t gap_bytes = _gap_tree.add_gap(gap_bottom, gap_top);
+ size_t gap_bytes = pointer_delta(gap_top, gap_bottom, 1);
+ // A gap smaller than an allocation unit can never be reused
+ if (gap_bytes >= SharedSpaceObjectAlignment) {
+ _gap_tree.add_gap(gap_bottom, gap_top);
+ }
allocate(gap_bytes);
}
diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp
index 93d9061efb98..5402a5f9309d 100644
--- a/src/hotspot/share/cds/heapShared.cpp
+++ b/src/hotspot/share/cds/heapShared.cpp
@@ -909,7 +909,7 @@ void HeapShared::copy_and_rescan_aot_inited_mirror(InstanceKlass* ik) {
void HeapShared::copy_java_mirror(oop orig_mirror, oop scratch_m) {
// We need to retain the identity_hash, because it may have been used by some hashtables
// in the shared heap.
- if (!orig_mirror->fast_no_hash_check()) {
+ if (orig_mirror->has_identity_hash()) {
intptr_t src_hash = orig_mirror->identity_hash();
if (UseCompactObjectHeaders) {
narrowKlass nk = CompressedKlassPointers::encode(orig_mirror->klass());
@@ -918,7 +918,7 @@ void HeapShared::copy_java_mirror(oop orig_mirror, oop scratch_m) {
// For valhalla, the prototype header is the same as markWord::prototype();
scratch_m->set_mark(markWord::prototype().copy_set_hash(src_hash));
}
- assert(scratch_m->mark().is_unlocked(), "sanity");
+ assert(scratch_m->mark().is_lock_neutral(), "sanity");
DEBUG_ONLY(intptr_t archived_hash = scratch_m->identity_hash());
assert(src_hash == archived_hash, "Different hash codes: original " INTPTR_FORMAT ", archived " INTPTR_FORMAT, src_hash, archived_hash);
diff --git a/src/hotspot/share/cds/lambdaFormInvokers.cpp b/src/hotspot/share/cds/lambdaFormInvokers.cpp
index 5cfb73d2f6ce..9b87438ff118 100644
--- a/src/hotspot/share/cds/lambdaFormInvokers.cpp
+++ b/src/hotspot/share/cds/lambdaFormInvokers.cpp
@@ -225,6 +225,9 @@ void LambdaFormInvokers::regenerate_class(char* class_name, ClassFileStream& st,
cl_info,
CHECK);
+ // The result InstanceKlass* is never used during the JVM process lifetime.
+ // We create it only for writing to the CDS archive, and so it need not be monitored by JVMTI or JFR.
+
assert(result->java_mirror() != nullptr, "must be");
RegeneratedClasses::add_class(InstanceKlass::cast(klass), result);
diff --git a/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp b/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp
index 3c28bf06adf9..07b804c8ded4 100644
--- a/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp
+++ b/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp
@@ -34,6 +34,9 @@
#include "memory/metaspaceClosure.hpp"
#include "memory/resourceArea.hpp"
#include "oops/klass.inline.hpp"
+#if INCLUDE_JFR
+#include "jfr/jfr.hpp"
+#endif
DumpTimeLambdaProxyClassInfo::~DumpTimeLambdaProxyClassInfo() {
if (_proxy_klasses != nullptr) {
@@ -318,6 +321,9 @@ InstanceKlass* LambdaProxyClassDictionary::find_lambda_proxy_class(const RunTime
InstanceKlass* LambdaProxyClassDictionary::load_and_init_lambda_proxy_class(InstanceKlass* lambda_ik,
InstanceKlass* caller_ik, TRAPS) {
+
+ EventClassLoad class_load_event;
+
Handle class_loader(THREAD, caller_ik->class_loader());
Handle protection_domain;
PackageEntry* pkg_entry = caller_ik->package();
@@ -359,20 +365,22 @@ InstanceKlass* LambdaProxyClassDictionary::load_and_init_lambda_proxy_class(Inst
InstanceKlass* nest_host = caller_ik->nest_host(THREAD);
assert(nest_host == shared_nest_host, "mismatched nest host");
- EventClassLoad class_load_event;
+ JFR_ONLY(Jfr::on_definition(lambda_ik, THREAD);)
// Add to class hierarchy, and do possible deoptimizations.
lambda_ik->add_to_hierarchy(THREAD);
+ assert(lambda_ik->is_loaded(), "Must be in at least loaded state");
// But, do not add to dictionary.
+ if (class_load_event.should_commit()) {
+ JFR_ONLY(SystemDictionary::post_class_load_event(&class_load_event, lambda_ik, ClassLoaderData::class_loader_data(class_loader()));)
+ }
+
lambda_ik->link_class(CHECK_NULL);
// notify jvmti
if (JvmtiExport::should_post_class_load()) {
JvmtiExport::post_class_load(THREAD, lambda_ik);
}
- if (class_load_event.should_commit()) {
- JFR_ONLY(SystemDictionary::post_class_load_event(&class_load_event, lambda_ik, ClassLoaderData::class_loader_data(class_loader()));)
- }
lambda_ik->initialize(CHECK_NULL);
diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp
index a84a7b4fb07d..62411a09556b 100644
--- a/src/hotspot/share/classfile/classLoader.cpp
+++ b/src/hotspot/share/classfile/classLoader.cpp
@@ -25,6 +25,7 @@
#include "cds/aotClassLocation.hpp"
#include "cds/cds_globals.hpp"
#include "cds/cdsConfig.hpp"
+#include "cds/cdsProtectionDomain.hpp"
#include "cds/dynamicArchive.hpp"
#include "cds/heapShared.hpp"
#include "classfile/classFileStream.hpp"
@@ -368,18 +369,57 @@ ClassPathZipEntry::~ClassPathZipEntry() {
FREE_C_HEAP_ARRAY(_zip_name);
}
-bool ClassPathZipEntry::has_entry(JavaThread* current, const char* name) {
- ThreadToNativeFromVM ttn(current);
+bool ClassPathZipEntry::has_entry(JavaThread* current, const char* name, Handle class_loader, bool is_multi_release_jar) {
// check whether zip archive contains name
jint name_len;
jint filesize;
- jzentry* entry = ZipLibrary::find_entry(_zip, name, &filesize, &name_len);
- if (entry == nullptr) {
- return false;
- } else {
- ZipLibrary::free_entry(_zip, entry);
- return true;
+
+ {
+ ThreadToNativeFromVM ttn(current);
+ jzentry* entry = ZipLibrary::find_entry(_zip, name, &filesize, &name_len);
+ if (entry != nullptr) {
+ ZipLibrary::free_entry(_zip, entry);
+ return true;
+ }
}
+
+#if INCLUDE_CDS
+ // Make an upcall to ClassLoader.getResource() if "name" is in a multi-release JAR
+ // and was not found in the root of the JAR file. This will always be a built-in class
+ // loader but CDS.getResource() will ensure the resource is retrieved from the correct
+ // JAR file anyway.
+ if (class_loader != nullptr && is_multi_release_jar) {
+ assert(SystemDictionaryShared::is_builtin_loader(ClassLoaderData::class_loader_data(class_loader())), "must be");
+ JavaValue result(T_OBJECT);
+ oop class_name_oop = java_lang_String::create_oop_from_str(name, current);
+ oop zip_name_oop = CDSProtectionDomain::to_file_URL(_zip_name, Handle(), current);
+ Handle h_class_name(current, class_name_oop);
+ Handle h_zip_name(current, zip_name_oop);
+
+ // URL ClassLoader.getResource(String name)
+ JavaCalls::call_static(&result,
+ vmClasses::CDS_klass(),
+ vmSymbols::getResource_name(),
+ vmSymbols::getResource_cds_signature(),
+ class_loader,
+ h_zip_name,
+ h_class_name,
+ current);
+
+ // Not using CHECK, the thread must be checked manually
+ if (current->has_pending_exception()) {
+ current->clear_pending_exception();
+ return false;
+ }
+
+ assert(result.get_type() == T_OBJECT, "just checking");
+ if (result.get_oop() != nullptr) {
+ return true;
+ }
+ }
+#endif // INCLUDE_CDS
+
+ return false;
}
u1* ClassPathZipEntry::open_entry(JavaThread* current, const char* name, jint* filesize, bool nul_terminate) {
diff --git a/src/hotspot/share/classfile/classLoader.hpp b/src/hotspot/share/classfile/classLoader.hpp
index 2bdfdc0b39de..030e083374c8 100644
--- a/src/hotspot/share/classfile/classLoader.hpp
+++ b/src/hotspot/share/classfile/classLoader.hpp
@@ -93,7 +93,7 @@ class ClassPathZipEntry: public ClassPathEntry {
const char* name() const { return _zip_name; }
ClassPathZipEntry(jzfile* zip, const char* zip_name);
virtual ~ClassPathZipEntry();
- bool has_entry(JavaThread* current, const char* name);
+ bool has_entry(JavaThread* current, const char* name, Handle class_loader, bool is_multi_release_jar);
u1* open_entry(JavaThread* current, const char* name, jint* filesize, bool nul_terminate);
ClassFileStream* open_stream(JavaThread* current, const char* name);
};
diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp
index b773569ad7b8..27a899481a8f 100644
--- a/src/hotspot/share/classfile/classLoaderData.cpp
+++ b/src/hotspot/share/classfile/classLoaderData.cpp
@@ -81,6 +81,9 @@
#include "utilities/growableArray.hpp"
#include "utilities/macros.hpp"
#include "utilities/ostream.hpp"
+#if INCLUDE_JFR
+#include "jfr/jfr.hpp"
+#endif
ClassLoaderData * ClassLoaderData::_the_null_class_loader_data = nullptr;
@@ -899,6 +902,7 @@ void ClassLoaderData::free_deallocate_list() {
HeapShared::remove_scratch_resolved_references((ConstantPool*)m);
MetadataFactory::free_metadata(this, (ConstantPool*)m);
} else if (m->is_klass()) {
+ JFR_ONLY(Jfr::on_deallocation(static_cast(m));)
if (!((Klass*)m)->is_inline_klass()) {
MetadataFactory::free_metadata(this, (InstanceKlass*)m);
} else {
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
index 4a3ec7b680ff..49cc3ea337b1 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
@@ -841,6 +841,64 @@ void FieldLayoutBuilder::prologue() {
_root_group = new FieldGroup();
}
+int FieldLayoutBuilder::add_field_to_group(FieldInfo fieldinfo, int idx, FieldGroup* group) {
+ BasicType type = Signature::basic_type(fieldinfo.signature(_constant_pool));
+ switch(type) {
+ case T_BYTE:
+ case T_CHAR:
+ case T_DOUBLE:
+ case T_FLOAT:
+ case T_INT:
+ case T_LONG:
+ case T_SHORT:
+ case T_BOOLEAN:
+ group->add_primitive_field(idx, type);
+ return type2aelembytes(type); // alignment == size for primitive types
+ case T_OBJECT:
+ case T_ARRAY:
+ {
+ const bool is_inline_class = _is_inline_type || _is_abstract_value;
+ // Atomic flat fields can always be used in identity classes.
+ // Use them only for inline classes if the container is itself atomic.
+ const bool use_atomic_flat = !is_inline_class || _must_be_atomic;
+ LayoutKind lk = field_layout_selection(fieldinfo, _inline_layout_info_array, use_atomic_flat);
+ lk = adjust_with_budget(fieldinfo, _inline_layout_info_array, lk, _flattening_budget);
+ if (field_is_inlineable(fieldinfo, lk, _inline_layout_info_array)) {
+ _has_inlineable_fields = true;
+ }
+
+ if (lk == LayoutKind::REFERENCE) {
+ if (group != _static_fields) {
+ _nonstatic_oopmap_count++;
+ }
+ group->add_oop_field(idx);
+ return type2aelembytes(type); // alignment == size for oops
+ }
+
+ assert(group != _static_fields, "Static fields are not flattened");
+ assert(lk != LayoutKind::BUFFERED && lk != LayoutKind::UNKNOWN,
+ "Invalid layout kind for flat field: %s", LayoutKindHelper::layout_kind_as_string(lk));
+
+ const int field_index = (int)fieldinfo.index();
+ assert(_inline_layout_info_array != nullptr, "Array must have been created");
+ assert(_inline_layout_info_array->adr_at(field_index)->klass() != nullptr, "Klass must have been set");
+ _has_inlined_fields = true;
+ InlineKlass* vk = _inline_layout_info_array->adr_at(field_index)->klass();
+ if (is_inline_class && !vk->is_naturally_atomic(LayoutKindHelper::is_null_free_flat(lk))) {
+ _has_non_naturally_atomic_fields = true;
+ }
+ group->add_flat_field(idx, vk, lk);
+ _inline_layout_info_array->adr_at(field_index)->set_kind(lk);
+ _nonstatic_oopmap_count += vk->nonstatic_oop_map_count();
+ _field_info->adr_at(idx)->field_flags_addr()->update_flat(true);
+ _field_info->adr_at(idx)->set_layout_kind(lk);
+ return vk->layout_alignment(lk);
+ }
+ default:
+ fatal("Unexpected BasicType");
+ }
+}
+
// Field sorting for regular (non-inline) classes:
// - fields are sorted in static and non-static fields
// - non-static fields are also sorted according to their contention group
@@ -869,52 +927,7 @@ void FieldLayoutBuilder::regular_field_sorting() {
}
}
assert(group != nullptr, "invariant");
- BasicType type = Signature::basic_type(fieldinfo.signature(_constant_pool));
- switch(type) {
- case T_BYTE:
- case T_CHAR:
- case T_DOUBLE:
- case T_FLOAT:
- case T_INT:
- case T_LONG:
- case T_SHORT:
- case T_BOOLEAN:
- group->add_primitive_field(idx, type);
- break;
- case T_OBJECT:
- case T_ARRAY:
- {
- LayoutKind lk = field_layout_selection(fieldinfo, _inline_layout_info_array, true);
- lk = adjust_with_budget(fieldinfo, _inline_layout_info_array, lk, _flattening_budget);
- if (field_is_inlineable(fieldinfo, lk, _inline_layout_info_array)) {
- _has_inlineable_fields = true;
- }
-
- if (lk == LayoutKind::REFERENCE) {
- if (group != _static_fields) _nonstatic_oopmap_count++;
- group->add_oop_field(idx);
- } else {
- assert(group != _static_fields, "Static fields are not flattened");
- assert(lk != LayoutKind::BUFFERED && lk != LayoutKind::UNKNOWN,
- "Invalid layout kind for flat field: %s", LayoutKindHelper::layout_kind_as_string(lk));
-
- const int field_index = (int)fieldinfo.index();
- assert(_inline_layout_info_array != nullptr, "Array must have been created");
- assert(_inline_layout_info_array->adr_at(field_index)->klass() != nullptr, "Klass must have been set");
- _has_inlined_fields = true;
- InlineKlass* vk = _inline_layout_info_array->adr_at(field_index)->klass();
- group->add_flat_field(idx, vk, lk);
- _inline_layout_info_array->adr_at(field_index)->set_kind(lk);
- _nonstatic_oopmap_count += vk->nonstatic_oop_map_count();
- _field_info->adr_at(idx)->field_flags_addr()->update_flat(true);
- _field_info->adr_at(idx)->set_layout_kind(lk);
- // no need to update _must_be_atomic if vk->must_be_atomic() is true because current class is not an inline class
- }
- break;
- }
- default:
- fatal("Something wrong?");
- }
+ add_field_to_group(fieldinfo, idx, group);
}
_root_group->sort_by_size();
_static_fields->sort_by_size();
@@ -952,60 +965,7 @@ void FieldLayoutBuilder::inline_class_field_sorting() {
group = _root_group;
}
assert(group != nullptr, "invariant");
- BasicType type = Signature::basic_type(fieldinfo.signature(_constant_pool));
- switch(type) {
- case T_BYTE:
- case T_CHAR:
- case T_DOUBLE:
- case T_FLOAT:
- case T_INT:
- case T_LONG:
- case T_SHORT:
- case T_BOOLEAN:
- if (group != _static_fields) {
- field_alignment = type2aelembytes(type); // alignment == size for primitive types
- }
- group->add_primitive_field(idx, type);
- break;
- case T_OBJECT:
- case T_ARRAY:
- {
- bool use_atomic_flat = _must_be_atomic; // flatten atomic fields only if the container is itself atomic
- LayoutKind lk = field_layout_selection(fieldinfo, _inline_layout_info_array, use_atomic_flat);
- lk = adjust_with_budget(fieldinfo, _inline_layout_info_array, lk, _flattening_budget);
- if (field_is_inlineable(fieldinfo, lk, _inline_layout_info_array)) {
- _has_inlineable_fields = true;
- }
-
- if (lk == LayoutKind::REFERENCE) {
- if (group != _static_fields) {
- _nonstatic_oopmap_count++;
- field_alignment = type2aelembytes(type); // alignment == size for oops
- }
- group->add_oop_field(idx);
- } else {
- assert(group != _static_fields, "Static fields are not flattened");
- assert(lk != LayoutKind::BUFFERED && lk != LayoutKind::UNKNOWN,
- "Invalid layout kind for flat field: %s", LayoutKindHelper::layout_kind_as_string(lk));
-
- const int field_index = (int)fieldinfo.index();
- assert(_inline_layout_info_array != nullptr, "Array must have been created");
- assert(_inline_layout_info_array->adr_at(field_index)->klass() != nullptr, "Klass must have been set");
- _has_inlined_fields = true;
- InlineKlass* vk = _inline_layout_info_array->adr_at(field_index)->klass();
- if (!vk->is_naturally_atomic(LayoutKindHelper::is_null_free_flat(lk))) _has_non_naturally_atomic_fields = true;
- group->add_flat_field(idx, vk, lk);
- _inline_layout_info_array->adr_at(field_index)->set_kind(lk);
- _nonstatic_oopmap_count += vk->nonstatic_oop_map_count();
- field_alignment = vk->layout_alignment(lk);
- _field_info->adr_at(idx)->field_flags_addr()->update_flat(true);
- _field_info->adr_at(idx)->set_layout_kind(lk);
- }
- break;
- }
- default:
- fatal("Unexpected BasicType");
- }
+ field_alignment = add_field_to_group(fieldinfo, idx, group);
if (!fieldinfo.access_flags().is_static() && field_alignment > alignment) alignment = field_alignment;
}
_root_group->sort_by_size();
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
index 38a44e431d0d..ad148e8441d3 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
@@ -341,6 +341,7 @@ class FieldLayoutBuilder : public ResourceObj {
protected:
void prologue();
void epilogue();
+ int add_field_to_group(FieldInfo fieldinfo, int idx, FieldGroup* group);
void regular_field_sorting();
void inline_class_field_sorting();
void add_flat_field_oopmap(OopMapBlocksBuilder* nonstatic_oop_map, InlineKlass* vk, int offset);
diff --git a/src/hotspot/share/classfile/stackMapTable.cpp b/src/hotspot/share/classfile/stackMapTable.cpp
index 10f49d83b610..8954de5610b0 100644
--- a/src/hotspot/share/classfile/stackMapTable.cpp
+++ b/src/hotspot/share/classfile/stackMapTable.cpp
@@ -271,6 +271,13 @@ StackMapFrame* StackMapReader::next_helper(TRAPS) {
VerificationType* locals = nullptr;
u1 frame_type = _stream->get_u1(CHECK_NULL);
if (frame_type == EARLY_LARVAL) {
+ // early_larval frames are only supported in classes that support strict fields (preview classes)
+ if (!Verifier::supports_strict_fields(_verifier->current_class())) {
+ // reserved frame types when preview classes are disabled
+ _stream->stackmap_format_error(
+ "reserved frame type", CHECK_VERIFY_(_verifier, nullptr));
+ }
+
u2 num_unset_fields = _stream->get_u2(CHECK_NULL);
StackMapFrame::AssertUnsetFieldTable* new_fields = new StackMapFrame::AssertUnsetFieldTable();
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index 6141eeb7bd04..dbbc4196c7f7 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -837,6 +837,7 @@ InstanceKlass* SystemDictionary::resolve_hidden_class_from_stream(
cl_info,
CHECK_NULL);
assert(k != nullptr, "no klass created");
+ assert(k->class_loader_data() == loader_data, "invariant");
// Hidden classes that are not strong must update ClassLoaderData holder
// so that they can be unloaded when the mirror is no longer referenced.
@@ -844,8 +845,11 @@ InstanceKlass* SystemDictionary::resolve_hidden_class_from_stream(
k->class_loader_data()->initialize_holder(Handle(THREAD, k->java_mirror()));
}
+ JFR_ONLY(Jfr::on_definition(k, THREAD);)
+
// Add to class hierarchy, and do possible deoptimizations.
k->add_to_hierarchy(THREAD);
+ assert(k->is_loaded(), "Must be in at least loaded state");
// But, do not add to dictionary.
if (class_load_event.should_commit()) {
@@ -950,7 +954,6 @@ bool SystemDictionary::is_shared_class_visible(Symbol* class_name,
InstanceKlass* ik,
PackageEntry* pkg_entry,
Handle class_loader) {
-
assert(!ModuleEntryTable::javabase_moduleEntry()->is_patched(),
"Cannot use sharing if java.base is patched");
@@ -1334,7 +1337,11 @@ void SystemDictionary::preload_class(Handle class_loader, InstanceKlass* ik, TRA
ik->restore_unshareable_info(loader_data, pd, pkg_entry, CHECK);
load_shared_class_misc(ik, loader_data);
+
+ JFR_ONLY(Jfr::on_definition(ik, THREAD);)
+
ik->add_to_hierarchy(THREAD);
+ assert(ik->is_loaded(), "Must be in at least loaded state");
if (!ik->is_hidden()) {
update_dictionary(THREAD, ik, loader_data);
@@ -1343,8 +1350,6 @@ void SystemDictionary::preload_class(Handle class_loader, InstanceKlass* ik, TRA
if (class_load_event.should_commit()) {
JFR_ONLY(post_class_load_event(&class_load_event, ik, loader_data);)
}
-
- assert(ik->is_loaded(), "Must be in at least loaded state");
}
#endif // INCLUDE_CDS
@@ -1572,8 +1577,11 @@ void SystemDictionary::define_instance_class(InstanceKlass* k, Handle class_load
JavaCalls::call(&result, m, &args, CHECK);
}
+ JFR_ONLY(Jfr::on_definition(k, THREAD);)
+
// Add to class hierarchy, and do possible deoptimizations.
k->add_to_hierarchy(THREAD);
+ assert(k->is_loaded(), "Must be in at least loaded state");
// Add to systemDictionary - so other classes can see it.
// Grabs and releases SystemDictionary_lock
diff --git a/src/hotspot/share/classfile/systemDictionary.hpp b/src/hotspot/share/classfile/systemDictionary.hpp
index 4852ebe533a6..7a5bb1ec64ab 100644
--- a/src/hotspot/share/classfile/systemDictionary.hpp
+++ b/src/hotspot/share/classfile/systemDictionary.hpp
@@ -344,10 +344,6 @@ class SystemDictionary : AllStatic {
static InstanceKlass* find_or_define_instance_class(Symbol* class_name,
Handle class_loader,
InstanceKlass* k, TRAPS);
- JFR_ONLY(static void post_class_load_event(EventClassLoad* event,
- const InstanceKlass* k,
- const ClassLoaderData* init_cld);)
-
public:
static bool is_system_class_loader(oop class_loader);
static bool is_platform_class_loader(oop class_loader);
@@ -362,6 +358,10 @@ class SystemDictionary : AllStatic {
// Return Symbol or throw exception if name given is can not be a valid Symbol.
static Symbol* class_name_symbol(const char* name, Symbol* exception, TRAPS);
+
+ JFR_ONLY(static void post_class_load_event(EventClassLoad* event,
+ const InstanceKlass* k,
+ const ClassLoaderData* init_cld);)
};
#endif // SHARE_CLASSFILE_SYSTEMDICTIONARY_HPP
diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp
index b0ded20b6f8c..626c28c17118 100644
--- a/src/hotspot/share/classfile/verifier.cpp
+++ b/src/hotspot/share/classfile/verifier.cpp
@@ -636,10 +636,9 @@ TypeOrigin ClassVerifier::ref_ctx(const char* sig) {
return TypeOrigin::implicit(vt);
}
-static bool supports_strict_fields(InstanceKlass* klass) {
+bool Verifier::supports_strict_fields(InstanceKlass* klass) {
int ver = klass->major_version();
- return ver > Verifier::VALUE_TYPES_MAJOR_VERSION ||
- (ver == Verifier::VALUE_TYPES_MAJOR_VERSION && klass->minor_version() == Verifier::JAVA_PREVIEW_MINOR_VERSION);
+ return (ver >= Verifier::VALUE_TYPES_MAJOR_VERSION && klass->minor_version() == Verifier::JAVA_PREVIEW_MINOR_VERSION);
}
void ClassVerifier::verify_class(TRAPS) {
@@ -2428,7 +2427,7 @@ void ClassVerifier::verify_field_instructions(RawBytecodeStream* bcs,
}
}
}
- } else if (supports_strict_fields(_klass)) {
+ } else if (Verifier::supports_strict_fields(_klass)) {
// `strict` fields are not writable, but only local fields produce verification errors
if (is_local_field && fd.access_flags().is_strict() && fd.access_flags().is_final()) {
verify_error(ErrorContext::bad_code(bci),
diff --git a/src/hotspot/share/classfile/verifier.hpp b/src/hotspot/share/classfile/verifier.hpp
index 087750e4b121..97a546736a4a 100644
--- a/src/hotspot/share/classfile/verifier.hpp
+++ b/src/hotspot/share/classfile/verifier.hpp
@@ -69,6 +69,8 @@ class Verifier : AllStatic {
// Print output for class+resolve
static void trace_class_resolution(Klass* resolve_class, InstanceKlass* verify_class);
+ static bool supports_strict_fields(InstanceKlass* klass);
+
private:
static Symbol* inference_verify(
InstanceKlass* klass, char* msg, size_t msg_len, TRAPS);
diff --git a/src/hotspot/share/classfile/vmClassMacros.hpp b/src/hotspot/share/classfile/vmClassMacros.hpp
index 76071a550f94..2206da3684a5 100644
--- a/src/hotspot/share/classfile/vmClassMacros.hpp
+++ b/src/hotspot/share/classfile/vmClassMacros.hpp
@@ -138,6 +138,7 @@
do_klass(module_Modules_klass, jdk_internal_module_Modules ) \
\
/* support for CDS */ \
+ do_klass(CDS_klass, jdk_internal_misc_CDS ) \
do_klass(ByteArrayInputStream_klass, java_io_ByteArrayInputStream ) \
do_klass(URL_klass, java_net_URL ) \
do_klass(Enum_klass, java_lang_Enum ) \
diff --git a/src/hotspot/share/classfile/vmClasses.cpp b/src/hotspot/share/classfile/vmClasses.cpp
index 00d209a05ca5..1dde8b904a3c 100644
--- a/src/hotspot/share/classfile/vmClasses.cpp
+++ b/src/hotspot/share/classfile/vmClasses.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -43,6 +43,9 @@
#include "oops/instanceStackChunkKlass.hpp"
#include "prims/jvmtiExport.hpp"
#include "runtime/globals.hpp"
+#if INCLUDE_JFR
+#include "jfr/jfr.hpp"
+#endif
InstanceKlass* vmClasses::_klasses[static_cast(vmClassID::LIMIT)]
= { nullptr /*, nullptr...*/ };
@@ -260,11 +263,15 @@ void vmClasses::resolve_shared_class(InstanceKlass* klass, ClassLoaderData* load
klass->restore_unshareable_info(loader_data, domain, nullptr, THREAD);
SystemDictionary::load_shared_class_misc(klass, loader_data);
- Dictionary* dictionary = loader_data->dictionary();
- dictionary->add_klass(THREAD, klass->name(), klass);
+
+ JFR_ONLY(Jfr::on_definition(klass, THREAD));
+
klass->add_to_hierarchy(THREAD);
assert(klass->is_loaded(), "Must be in at least loaded state");
+ Dictionary* dictionary = loader_data->dictionary();
+ dictionary->add_klass(THREAD, klass->name(), klass);
+
if (class_load_event.should_commit()) {
JFR_ONLY(SystemDictionary::post_class_load_event(&class_load_event, klass, loader_data);)
}
diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp
index 1de819967e09..f2fc6704d07f 100644
--- a/src/hotspot/share/classfile/vmSymbols.hpp
+++ b/src/hotspot/share/classfile/vmSymbols.hpp
@@ -750,6 +750,7 @@ class SerializeClosure;
template(dumpSharedArchive_signature, "(ZLjava/lang/String;)Ljava/lang/String;") \
template(generateLambdaFormHolderClasses, "generateLambdaFormHolderClasses") \
template(generateLambdaFormHolderClasses_signature, "([Ljava/lang/String;)[Ljava/lang/Object;") \
+ template(getResource_name, "getResource") \
template(getResourceAsByteArray_name, "getResourceAsByteArray") \
template(getResourceAsByteArray_signature, "(Ljava/lang/String;)[B") \
template(java_lang_Enum, "java/lang/Enum") \
@@ -759,6 +760,7 @@ class SerializeClosure;
template(java_lang_invoke_DelegatingMethodHandle_Holder, "java/lang/invoke/DelegatingMethodHandle$Holder") \
template(jdk_internal_loader_ClassLoaders, "jdk/internal/loader/ClassLoaders") \
template(jdk_internal_misc_CDS, "jdk/internal/misc/CDS") \
+ template(getResource_cds_signature, "(Ljava/lang/ClassLoader;Ljava/net/URL;Ljava/lang/String;)Ljava/net/URL;")\
template(jdk_internal_vm_annotation_AOTSafeClassInitializer_signature, "Ljdk/internal/vm/annotation/AOTSafeClassInitializer;")\
template(java_util_concurrent_ConcurrentHashMap, "java/util/concurrent/ConcurrentHashMap") \
template(java_util_ArrayList, "java/util/ArrayList") \
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index c36582f73813..30cd400058b5 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -1388,7 +1388,6 @@ nmethod::nmethod(const nmethod &nm) : CodeBlob(nm._name, nm._kind, nm._size, nm.
_exception_cache = nullptr;
_gc_data = nullptr;
- _oops_do_mark_nmethods = nullptr;
_oops_do_mark_link = nullptr;
_compiled_ic_data = nullptr;
diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp
index 7ca7b57b43b7..3f1b5722f8fa 100644
--- a/src/hotspot/share/compiler/compileBroker.cpp
+++ b/src/hotspot/share/compiler/compileBroker.cpp
@@ -2169,8 +2169,12 @@ void CompileBroker::handle_full_code_cache(CodeBlobType code_blob_type) {
#ifndef PRODUCT
if (ExitOnFullCodeCache) {
codecache_print(/* detailed= */ true);
- before_exit(JavaThread::current());
- exit_globals(); // will delete tty
+ // handle_full_code_cache() can be called from a compiler thread while it
+ // is installing an nmethod, i.e. from a no-safepoint scope. before_exit()
+ // and exit_globals() acquire safepoint-checking locks (e.g. BeforeExit_lock)
+ // and would assert "Possible safepoint reached by thread that does not
+ // allow it". vm_direct_exit() terminates the VM without taking any such
+ // lock, which is sufficient for this diagnostic develop flag.
vm_direct_exit(1);
}
#endif
diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
index 774ea372bad8..cf4bacbb98b2 100644
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
@@ -575,7 +575,7 @@ void G1BarrierSetC2::emit_stubs(CodeBuffer& cb) const {
}
stubs->at(i)->emit_code(masm);
}
- masm.flush();
+ // Code will be copied. No ICache sync required.
}
#ifndef PRODUCT
diff --git a/src/hotspot/share/gc/g1/g1CardSet.cpp b/src/hotspot/share/gc/g1/g1CardSet.cpp
index f0db638a2fed..0f66e739362f 100644
--- a/src/hotspot/share/gc/g1/g1CardSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CardSet.cpp
@@ -967,7 +967,7 @@ class G1ContainerCardsClosure {
void operator()(uint card_idx, uint length) {
for (uint i = 0; i < length; i++) {
- _cl.do_card(_region_idx, card_idx);
+ _cl.do_card(_region_idx, card_idx + i);
}
}
};
diff --git a/src/hotspot/share/gc/g1/g1CodeRootSet.cpp b/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
index 7f1dec462d4f..771ab1a7fa2a 100644
--- a/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
@@ -127,15 +127,6 @@ class G1CodeRootSetHashTable : public CHeapObj {
}
}
- bool remove(nmethod* method) {
- HashTableLookUp lookup(method);
- bool removed = _table.remove(Thread::current(), lookup);
- if (removed) {
- _num_entries.sub_then_fetch(1u);
- }
- return removed;
- }
-
bool contains(nmethod* method) {
HashTableLookUp lookup(method);
HashTableIgnore ignore;
@@ -246,7 +237,9 @@ class G1CodeRootSetHashTable : public CHeapObj {
_table_scanner.set(&_table, BucketClaimSize);
}
- size_t mem_size() { return sizeof(*this) + _table.get_mem_size(Thread::current()); }
+ size_t mem_size() {
+ return sizeof(*this) - sizeof(_table) + _table.get_mem_size(Thread::current());
+ }
size_t number_of_entries() const { return _num_entries.load_relaxed(); }
};
@@ -281,11 +274,6 @@ G1CodeRootSet::~G1CodeRootSet() {
delete _table;
}
-bool G1CodeRootSet::remove(nmethod* method) {
- assert(!_is_iterating, "should not mutate while iterating the table");
- return _table->remove(method);
-}
-
void G1CodeRootSet::bulk_remove() {
assert(!_is_iterating, "should not mutate while iterating the table");
_table->bulk_remove();
diff --git a/src/hotspot/share/gc/g1/g1CodeRootSet.hpp b/src/hotspot/share/gc/g1/g1CodeRootSet.hpp
index b298bbfb9148..c01a4a8396a5 100644
--- a/src/hotspot/share/gc/g1/g1CodeRootSet.hpp
+++ b/src/hotspot/share/gc/g1/g1CodeRootSet.hpp
@@ -43,7 +43,6 @@ class G1CodeRootSet {
~G1CodeRootSet();
void add(nmethod* method);
- bool remove(nmethod* method);
void bulk_remove();
// Notify the code root set that we are about to add the given
// number of code roots. Only to be used during safepoint, not
@@ -59,8 +58,6 @@ class G1CodeRootSet {
// Remove all nmethods which no longer contain pointers into our "owner" region.
void clean(G1HeapRegion* owner);
- bool is_empty() { return length() == 0;}
-
// Length in elements
size_t length() const;
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index 6d713ff8e1ef..833d07cb6852 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -127,16 +127,6 @@ size_t G1CollectedHeap::_humongous_object_threshold_in_words = 0;
// apply to TLAB allocation, which is not part of this interface: it
// is done by clients of this interface.)
-void G1RegionMappingChangedListener::reset_from_card_cache(uint start_idx, size_t num_regions) {
- G1HeapRegionRemSet::invalidate_from_card_cache(start_idx, num_regions);
-}
-
-void G1RegionMappingChangedListener::on_commit(uint start_idx, size_t num_regions, bool zero_filled) {
- // The from card cache is not the memory that is actually committed. So we cannot
- // take advantage of the zero_filled parameter.
- reset_from_card_cache(start_idx, num_regions);
-}
-
// Collects commonly used scoped objects that are related to initial setup.
class G1GCMark : StackObj {
ResourceMark _rm;
@@ -1306,7 +1296,6 @@ G1CollectedHeap::G1CollectedHeap() :
_old_set("Old Region Set", new OldRegionSetChecker()),
_humongous_set("Humongous Region Set", new HumongousRegionSetChecker()),
_bot(nullptr),
- _listener(),
_numa(G1NUMA::create()),
_hrm(),
_allocator(nullptr),
@@ -1332,7 +1321,7 @@ G1CollectedHeap::G1CollectedHeap() :
_rem_set(nullptr),
_card_set_config(),
_card_set_freelist_pool(G1CardSetConfiguration::num_mem_object_types()),
- _young_regions_cset_group(card_set_config(), &_card_set_freelist_pool, G1CSetCandidateGroup::YoungRegionId),
+ _young_regions_cset_group(card_set_config(), &_card_set_freelist_pool, G1CSetCandidateGroup::YoungId),
_cm(nullptr),
_cr(nullptr),
_task_queues(nullptr),
@@ -1502,7 +1491,6 @@ jint G1CollectedHeap::initialize() {
heap_rs.base(),
heap_rs.size(),
page_size);
- heap_storage->set_mapping_changed_listener(&_listener);
// Create storage for the BOT, card table and the bitmap.
G1RegionToSpaceMapper* bot_storage =
@@ -1541,10 +1529,6 @@ jint G1CollectedHeap::initialize() {
const uint max_region_idx = (1U << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;
guarantee((max_num_regions() - 1) <= max_region_idx, "too many regions");
- // The G1FromCardCache reserves card with value 0 as "invalid", so the heap must not
- // start within the first card.
- guarantee((uintptr_t)(heap_rs.base()) >= G1CardTable::card_size(), "Java heap must not start within the first card.");
- G1FromCardCache::initialize(max_num_regions());
// Also create a G1 rem set.
_rem_set = new G1RemSet(this);
_rem_set->initialize(max_num_regions());
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
index 5480b5684c41..1497c61f59dd 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
@@ -111,13 +111,6 @@ class G1STWSubjectToDiscoveryClosure : public BoolObjectClosure {
bool do_object_b(oop p) override;
};
-class G1RegionMappingChangedListener : public G1MappingChangedListener {
- private:
- void reset_from_card_cache(uint start_idx, size_t num_regions);
- public:
- void on_commit(uint start_idx, size_t num_regions, bool zero_filled) override;
-};
-
// Helper to claim contiguous sets of JavaThread for processing by multiple threads.
class G1JavaThreadsListClaimer : public StackObj {
ThreadsListHandle _list;
@@ -223,9 +216,6 @@ class G1CollectedHeap : public CollectedHeap {
// free_list_only is true, it will only rebuild the free list.
void rebuild_region_sets(bool free_list_only);
- // Callback for region mapping changed events.
- G1RegionMappingChangedListener _listener;
-
// Handle G1 NUMA support.
G1NUMA* _numa;
diff --git a/src/hotspot/share/gc/g1/g1CollectionSet.cpp b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
index ca32759b54d3..efe8fe916591 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
@@ -218,7 +218,7 @@ void G1CollectionSet::add_young_region_common(G1HeapRegion* hr) {
assert(hr->is_young(), "invariant");
assert(_inc_build_state == CSetBuildType::Active, "Precondition");
- // Add to remembered set/cardset group.
+ // Add to remembered set/cset group.
_g1h->policy()->remset_tracker()->update_at_allocate(hr);
_g1h->young_regions_cset_group()->add(hr);
@@ -388,6 +388,11 @@ void G1CollectionSet::finalize_old_part(double time_remaining_ms) {
if (candidates()->retained_groups().num_regions() > 0) {
select_candidates_from_retained(time_remaining_ms);
}
+ // Optional groups are selected separately from marking and retained candidate
+ // lists; sort the combined list to maintain the GC efficiency ordering.
+ _optional_groups.sort_by_efficiency();
+ _optional_groups.verify();
+
candidates()->verify();
} else {
log_debug(gc, ergo, cset)("No candidates to reclaim.");
@@ -413,7 +418,7 @@ void G1CollectionSet::add_optional_group(G1CSetCandidateGroup* group,
double G1CollectionSet::select_candidates_from_marking(double time_remaining_ms) {
uint num_expensive_regions = 0;
- uint num_inital_regions = 0;
+ uint num_initial_regions = 0;
uint num_initial_groups = 0;
uint num_optional_regions = 0;
@@ -424,8 +429,8 @@ double G1CollectionSet::select_candidates_from_marking(double time_remaining_ms)
double optional_threshold_ms = time_remaining_ms * _policy->optional_prediction_fraction();
- uint min_old_cset_length = _policy->calc_min_old_cset_length(candidates()->last_marking_candidates_length());
- uint max_old_cset_length = MAX2(min_old_cset_length, _policy->calc_max_old_cset_length());
+ uint min_num_old_cset_regions = _policy->calc_min_old_cset_length(candidates()->last_marking_candidates_length());
+ uint max_num_old_cset_regions = MAX2(min_num_old_cset_regions, _policy->calc_max_old_cset_length());
bool check_time_remaining = _policy->use_adaptive_num_young_regions();
G1CSetCandidateGroupList* from_marking_groups = &candidates()->from_marking_groups();
@@ -435,13 +440,13 @@ double G1CollectionSet::select_candidates_from_marking(double time_remaining_ms)
log_debug(gc, ergo, cset)("Start adding marking candidates to collection set. "
"Min %u regions, max %u regions, available %u regions (%u groups), "
"time remaining %1.2fms, optional threshold %1.2fms",
- min_old_cset_length, max_old_cset_length, from_marking_groups->num_regions(), from_marking_groups->length(),
+ min_num_old_cset_regions, max_num_old_cset_regions, from_marking_groups->num_regions(), from_marking_groups->length(),
time_remaining_ms, optional_threshold_ms);
G1CSetCandidateGroupList selected_groups;
for (G1CSetCandidateGroup* group : *from_marking_groups) {
- if (num_inital_regions + num_optional_regions >= max_old_cset_length) {
+ if (num_initial_regions + num_optional_regions >= max_num_old_cset_regions) {
// Added maximum number of old regions to the CSet.
print_finish_message("Maximum number of regions reached", true);
break;
@@ -459,15 +464,15 @@ double G1CollectionSet::select_candidates_from_marking(double time_remaining_ms)
}
time_remaining_ms = MAX2(time_remaining_ms - predicted_time_ms, 0.0);
- // Add regions to old set until we reach the minimum amount
- if (num_inital_regions < min_old_cset_length) {
+ // Add regions to old set until we reach the minimum amount or reach the optional threshold.
+ if (num_initial_regions < min_num_old_cset_regions || (check_time_remaining && time_remaining_ms > optional_threshold_ms)) {
num_initial_groups++;
add_group_to_collection_set(group);
selected_groups.append(group);
- num_inital_regions += group->length();
+ num_initial_regions += group->length();
predicted_initial_time_ms += predicted_time_ms;
// Record the number of regions added with no time remaining
@@ -479,28 +484,15 @@ double G1CollectionSet::select_candidates_from_marking(double time_remaining_ms)
// to the CSet if we reach the minimum.
print_finish_message("Region amount reached min", true);
break;
+ } else if (time_remaining_ms > 0) {
+ // Keep adding optional regions until time is up.
+ add_optional_group(group,
+ num_optional_regions,
+ predicted_optional_time_ms,
+ predicted_time_ms);
} else {
- // Keep adding regions to old set until we reach the optional threshold
- if (time_remaining_ms > optional_threshold_ms) {
- num_initial_groups++;
-
- add_group_to_collection_set(group);
- selected_groups.append(group);
-
- num_inital_regions += group->length();
-
- predicted_initial_time_ms += predicted_time_ms;
-
- } else if (time_remaining_ms > 0) {
- // Keep adding optional regions until time is up.
- add_optional_group(group,
- num_optional_regions,
- predicted_optional_time_ms,
- predicted_time_ms);
- } else {
- print_finish_message("Predicted time too high", true);
- break;
- }
+ print_finish_message("Predicted time too high", true);
+ break;
}
}
@@ -523,7 +515,7 @@ double G1CollectionSet::select_candidates_from_marking(double time_remaining_ms)
selected_groups.num_regions(), selected_groups.length(), _optional_groups.num_regions(), _optional_groups.length(),
predicted_initial_time_ms, predicted_optional_time_ms, time_remaining_ms);
- assert(selected_groups.num_regions() == num_inital_regions, "must be");
+ assert(selected_groups.num_regions() == num_initial_regions, "must be");
assert(_optional_groups.num_regions() == num_optional_regions, "must be");
return time_remaining_ms;
}
@@ -538,7 +530,7 @@ void G1CollectionSet::select_candidates_from_retained(double time_remaining_ms)
double predicted_initial_time_ms = 0.0;
double predicted_optional_time_ms = 0.0;
- uint const min_regions = _policy->min_retained_old_cset_length();
+ uint const min_num_regions = _policy->min_retained_old_cset_length();
// We want to make sure that on the one hand we process the retained regions asap,
// but on the other hand do not take too many of them as optional regions.
// So we split the time budget into budget we will unconditionally take into the
@@ -552,7 +544,7 @@ void G1CollectionSet::select_candidates_from_retained(double time_remaining_ms)
log_debug(gc, ergo, cset)("Start adding retained candidates to collection set. "
"Min %u regions, available %u regions (%u groups), "
"time remaining %1.2fms, optional remaining %1.2fms",
- min_regions, retained_groups->num_regions(), retained_groups->length(),
+ min_num_regions, retained_groups->num_regions(), retained_groups->length(),
time_remaining_ms, optional_time_remaining_ms);
G1CSetCandidateGroupList remove_from_retained;
@@ -585,10 +577,10 @@ void G1CollectionSet::select_candidates_from_retained(double time_remaining_ms)
continue;
}
- if (fits_in_remaining_time || (num_expensive_regions < min_regions)) {
+ if (num_initial_regions < min_num_regions || fits_in_remaining_time) {
predicted_initial_time_ms += predicted_time_ms;
if (!fits_in_remaining_time) {
- num_expensive_regions++;
+ num_expensive_regions += group->length();
}
add_group_to_collection_set(group);
@@ -622,7 +614,7 @@ void G1CollectionSet::select_candidates_from_retained(double time_remaining_ms)
// for the regions in these groups.
candidates()->remove(&remove_from_retained);
- groups_to_abandon.clear(true /* uninstall_group_cardset */);
+ groups_to_abandon.clear(true /* uninstall_cset_group */);
assert(num_optional_regions >= prev_num_optional_regions, "Sanity");
uint selected_optional_regions = num_optional_regions - prev_num_optional_regions;
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
index ac1b29a6bd79..84af28726a48 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
@@ -24,10 +24,9 @@
#include "gc/g1/g1CollectionSetCandidates.inline.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "utilities/growableArray.hpp"
-uint G1CSetCandidateGroup::_next_group_id = G1CSetCandidateGroup::InitialId;
-
G1CSetCandidateGroup::G1CSetCandidateGroup(G1CardSetConfiguration* config, G1MonotonicArenaFreePool* card_set_freelist_pool, uint group_id) :
_candidates(4, mtGCCardSet),
_card_set_mm(config, card_set_freelist_pool),
@@ -38,10 +37,16 @@ G1CSetCandidateGroup::G1CSetCandidateGroup(G1CardSetConfiguration* config, G1Mon
{ }
G1CSetCandidateGroup::G1CSetCandidateGroup() :
- G1CSetCandidateGroup(G1CollectedHeap::heap()->card_set_config(), G1CollectedHeap::heap()->card_set_freelist_pool(), _next_group_id++)
+ G1CSetCandidateGroup(G1CollectedHeap::heap()->card_set_config(), G1CollectedHeap::heap()->card_set_freelist_pool(), InvalidId)
{ }
void G1CSetCandidateGroup::add(G1HeapRegion* hr) {
+ precond(hr->is_young() == (_group_id == YoungId));
+
+ if (_candidates.is_empty() && _group_id != YoungId) {
+ precond(_group_id == InvalidId);
+ _group_id = FirstNonYoungId + hr->hrm_index();
+ }
G1CollectionSetCandidateInfo c(hr);
_candidates.append(c);
hr->install_cset_group(this);
@@ -63,16 +68,23 @@ double G1CSetCandidateGroup::liveness_percent() const {
return ((capacity - _reclaimable_bytes) * 100.0) / capacity;
}
-void G1CSetCandidateGroup::clear(bool uninstall_group_cardset) {
- if (uninstall_group_cardset) {
+void G1CSetCandidateGroup::clear(bool uninstall_cset_group) {
+ clear_card_set();
+ if (uninstall_cset_group) {
for (G1CollectionSetCandidateInfo ci : _candidates) {
G1HeapRegion* r = ci._r;
r->uninstall_cset_group();
- r->rem_set()->clear(true /* only_cardset */);
+ r->rem_set()->set_state_untracked();
}
}
- _card_set.clear();
_candidates.clear();
+ if (_group_id != YoungId) {
+ _group_id = InvalidId;
+ }
+}
+
+void G1CSetCandidateGroup::clear_card_set() {
+ _card_set.clear();
}
double G1CSetCandidateGroup::predict_group_total_time_ms() const {
@@ -116,16 +128,24 @@ double G1CSetCandidateGroup::predict_group_total_time_ms() const {
}
int G1CSetCandidateGroup::compare_gc_efficiency(G1CSetCandidateGroup** gr1, G1CSetCandidateGroup** gr2) {
- double gc_eff1 = (*gr1)->gc_efficiency();
- double gc_eff2 = (*gr2)->gc_efficiency();
+ G1CSetCandidateGroup* group_1 = *gr1;
+ G1CSetCandidateGroup* group_2 = *gr2;
+ double gc_eff1 = group_1->gc_efficiency();
+ double gc_eff2 = group_2->gc_efficiency();
if (gc_eff1 > gc_eff2) {
return -1;
} else if (gc_eff1 < gc_eff2) {
return 1;
- } else {
- return 0;
}
+
+ // Make ordering deterministic by breaking ties with group ids.
+ if (group_1->group_id() < group_2->group_id()) {
+ return -1;
+ } else if (group_1->group_id() > group_2->group_id()) {
+ return 1;
+ }
+ return 0;
}
G1CSetCandidateGroupList::G1CSetCandidateGroupList() : _groups(8, mtGC), _num_regions(0) { }
@@ -141,9 +161,9 @@ G1CSetCandidateGroup* G1CSetCandidateGroupList::at(uint index) {
return _groups.at(index);
}
-void G1CSetCandidateGroupList::clear(bool uninstall_group_cardset) {
+void G1CSetCandidateGroupList::clear(bool uninstall_cset_group) {
for (G1CSetCandidateGroup* gr : _groups) {
- gr->clear(uninstall_group_cardset);
+ gr->clear(uninstall_cset_group);
delete gr;
}
_groups.clear();
@@ -232,8 +252,8 @@ void G1CollectionSetCandidates::initialize(uint max_regions) {
}
void G1CollectionSetCandidates::clear() {
- _retained_groups.clear(true /* uninstall_group_cardset */);
- _from_marking_groups.clear(true /* uninstall_group_cardset */);
+ _retained_groups.clear(true /* uninstall_cset_group */);
+ _from_marking_groups.clear(true /* uninstall_cset_group */);
for (uint i = 0; i < _max_regions; i++) {
_contains_map[i] = CandidateOrigin::Invalid;
}
@@ -267,7 +287,6 @@ void G1CollectionSetCandidates::set_candidates_from_marking(GrowableArrayCHeapcalc_min_old_cset_length(num_candidates);
- G1CSetCandidateGroup::reset_next_group_id();
G1CSetCandidateGroup* current = nullptr;
current = new G1CSetCandidateGroup();
@@ -345,6 +364,7 @@ void G1CollectionSetCandidates::add_retained_region_unsorted(G1HeapRegion* r) {
G1CSetCandidateGroup* gr = new G1CSetCandidateGroup();
gr->add(r);
+ gr->calculate_efficiency();
_retained_groups.append(gr);
}
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp
index a70f9e395b64..368022a586c1 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.hpp
@@ -73,23 +73,17 @@ class G1CSetCandidateGroup : public CHeapObj{
size_t _reclaimable_bytes;
double _gc_efficiency;
+ // The _group_id identifies a candidate group in logging and in the
+ // FromCardCache. A group id must be assigned to at most one cset group
+ // at any time.
+ uint _group_id;
public:
- // The _group_id uniquely identifies a candidate group when printing, making it
- // easier to associate regions with their assigned G1CSetCandidateGroup, if any.
- // Special values for the id:
- // * id 0 is reserved for regions that do not have a remembered set.
- // * id 1 is reserved for the G1CollectionSetCandidate that contains all young regions.
- // * other ids are handed out incrementally, starting from InitialId.
- static const uint NoRemSetId = 0;
- static const uint YoungRegionId = 1;
- static const uint InitialId = 2;
+ static constexpr uint NoGroupId = 0;
+ static constexpr uint YoungId = NoGroupId + 1;
+ static constexpr uint FirstNonYoungId = YoungId + 1;
+ static constexpr uint InvalidId = UINT_MAX;
-private:
- const uint _group_id;
- static uint _next_group_id;
-
-public:
G1CSetCandidateGroup();
G1CSetCandidateGroup(G1CardSetConfiguration* config, G1MonotonicArenaFreePool* card_set_freelist_pool, uint group_id);
~G1CSetCandidateGroup() {
@@ -127,7 +121,10 @@ class G1CSetCandidateGroup : public CHeapObj{
return _card_set.occupied();
}
- void clear(bool uninstall_group_cardset = false);
+ // Clear the group-owned card set.
+ void clear_card_set();
+
+ void clear(bool uninstall_cset_group = false);
G1CSetCandidateGroupIterator begin() const {
return _candidates.begin();
@@ -137,10 +134,9 @@ class G1CSetCandidateGroup : public CHeapObj{
return _candidates.end();
}
- uint group_id() const { return _group_id; }
-
- static void reset_next_group_id() {
- _next_group_id = InitialId;
+ uint group_id() const {
+ assert(_group_id != InvalidId, "group must have an assigned id");
+ return _group_id;
}
};
@@ -154,11 +150,11 @@ class G1CSetCandidateGroupList {
G1CSetCandidateGroupList();
void append(G1CSetCandidateGroup* group);
- // Delete all groups from the list. The cardset cleanup for regions within the
- // groups could have been done elsewhere (e.g. when adding groups to the
- // collection set or to retained regions). The uninstall_group_cardset is set to
+ // Delete all groups from the list. The card set cleanup for regions within
+ // the groups could have been done elsewhere (e.g. when adding groups to the
+ // collection set or to retained regions). The uninstall_cset_group is set to
// true if cleanup needs to happen as we clear the groups from the list.
- void clear(bool uninstall_group_cardset = false);
+ void clear(bool uninstall_cset_group = false);
G1CSetCandidateGroup* at(uint index);
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
index 2246ffc12e7f..ccac7e01db07 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
@@ -36,7 +36,6 @@
#include "gc/g1/g1ConcurrentMarkRemarkTasks.hpp"
#include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
#include "gc/g1/g1ConcurrentRebuildAndScrub.hpp"
-#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1HeapRegionPrinter.hpp"
@@ -448,7 +447,6 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
_finger(nullptr), // _finger set in set_non_marking_state
- _worker_id_offset(G1ConcRefinementThreads), // The refinement control thread does not refine cards, so it's just the worker threads.
_max_num_tasks(MAX2(ConcGCThreads, ParallelGCThreads)),
_num_active_tasks(0), // _num_active_tasks set in set_non_marking_state()
_tasks(nullptr),
@@ -502,7 +500,7 @@ void G1ConcurrentMark::fully_initialize() {
vm_shutdown_during_initialization("Could not create ConcurrentMarkThread");
}
- log_debug(gc)("ConcGCThreads: %u offset %u", ConcGCThreads, _worker_id_offset);
+ log_debug(gc)("ConcGCThreads: %u", ConcGCThreads);
log_debug(gc)("ParallelGCThreads: %u", ParallelGCThreads);
_max_concurrent_workers = ConcGCThreads;
@@ -3161,7 +3159,7 @@ bool G1PrintRegionLivenessInfoClosure::do_heap_region(G1HeapRegion* r) {
const char* remset_type = r->rem_set()->get_short_state_str();
uint cset_group_id = r->rem_set()->has_cset_group()
? r->rem_set()->cset_group_id()
- : G1CSetCandidateGroup::NoRemSetId;
+ : G1CSetCandidateGroup::NoGroupId;
_total_used_bytes += used_bytes;
_total_capacity_bytes += capacity_bytes;
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
index 925d250ab0ad..ca36b48de5b4 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
@@ -354,7 +354,6 @@ class G1ConcurrentMark : public CHeapObj {
// always pointing to the end of the
// last claimed region
- uint _worker_id_offset;
uint _max_num_tasks; // Maximum number of marking tasks
uint _num_active_tasks; // Number of tasks currently active
G1CMTask** _tasks; // Task queue array (max_worker_id length)
@@ -567,8 +566,6 @@ class G1ConcurrentMark : public CHeapObj {
// TARS for the given region during remembered set rebuilding.
inline HeapWord* top_at_rebuild_start(G1HeapRegion* r) const;
- uint worker_id_offset() const { return _worker_id_offset; }
-
// Fully allocates and initializes data structures for the concurrent cycle.
// Methods that use concurrent cycle state such as the concurrent mark threads,
// tasks, marking stack, statistics, TAMS or TARS require this initialization.
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp
index a7fcc566d618..61ec542c71f0 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkRemarkTasks.cpp
@@ -28,6 +28,7 @@
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1HeapRegionPrinter.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1RemSetTrackingPolicy.hpp"
#include "logging/log.hpp"
#include "runtime/mutexLocker.hpp"
@@ -208,7 +209,8 @@ void G1UpdateRegionLivenessAndSelectForRebuildTask::prune(GrowableArrayCHeap allowed_waste) {
break;
}
- r->rem_set()->clear(true /* cardset_only */);
+ assert(!r->rem_set()->has_cset_group(), "must not have a cset group");
+ r->rem_set()->set_state_untracked();
wasted_bytes += reclaimable;
num_pruned++;
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
index 5b652f096a76..cf3e07592a09 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
@@ -93,8 +93,11 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
reset_processed_words();
// If a yield occurs (potential young-gc pause), must recheck for
// potential regions reclamation.
- if (_cm->do_yield_check() && !should_rebuild_or_scrub(hr)) {
- return true;
+ if (_cm->do_yield_check()) {
+ _rebuild_closure.reset_from_card_cache();
+ if (!should_rebuild_or_scrub(hr)) {
+ return true;
+ }
}
}
return _cm->has_aborted() || !should_rebuild_or_scrub(hr);
@@ -245,16 +248,18 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
}
public:
- G1RebuildRSAndScrubRegionClosure(G1ConcurrentMark* cm, bool should_rebuild_remset, uint worker_id) :
+ G1RebuildRSAndScrubRegionClosure(G1ConcurrentMark* cm, bool should_rebuild_remset) :
_cm(cm),
_bitmap(_cm->mark_bitmap()),
- _rebuild_closure(G1CollectedHeap::heap(), worker_id + cm->worker_id_offset()),
+ _rebuild_closure(G1CollectedHeap::heap()),
_should_rebuild_remset(should_rebuild_remset),
_processed_words(0) { }
bool do_heap_region(G1HeapRegion* hr) {
// Avoid stalling safepoints and stop iteration if mark cycle has been aborted.
- _cm->do_yield_check();
+ if (_cm->do_yield_check()) {
+ _rebuild_closure.reset_from_card_cache();
+ }
if (_cm->has_aborted()) {
return true;
}
@@ -294,7 +299,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
SuspendibleThreadSetJoiner sts_join;
G1CollectedHeap* g1h = G1CollectedHeap::heap();
- G1RebuildRSAndScrubRegionClosure cl(_cm, _should_rebuild_remset, worker_id);
+ G1RebuildRSAndScrubRegionClosure cl(_cm, _should_rebuild_remset);
g1h->heap_region_par_iterate_from_worker_offset(&cl, &_hr_claimer, worker_id);
}
};
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineSweepTask.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineSweepTask.cpp
index b0cf8353dfb8..d50c31f3da48 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineSweepTask.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineSweepTask.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,8 +33,6 @@ class G1RefineRegionClosure : public G1HeapRegionClosure {
G1RemSet* _rem_set;
G1CardTableClaimTable* _scan_state;
- uint _worker_id;
-
bool has_work(G1HeapRegion* r) {
return _scan_state->has_unclaimed_cards(r->hrm_index());
}
@@ -55,7 +53,7 @@ class G1RefineRegionClosure : public G1HeapRegionClosure {
void do_dirty_card(CardValue* source_card, CardValue* dest_card) {
verify_card_pair_refers_to_same_card(source_card, dest_card);
- G1RemSet::RefineResult res = _rem_set->refine_card_concurrently(source_card, _worker_id);
+ G1RemSet::RefineResult res = _rem_set->refine_card_concurrently(source_card);
// Gather statistics based on the result.
switch (res) {
case G1RemSet::HasRefToCSet: {
@@ -94,11 +92,10 @@ class G1RefineRegionClosure : public G1HeapRegionClosure {
bool _completed;
G1LocalRefineStats _per_worker_refine_data;
- G1RefineRegionClosure(uint worker_id, G1CardTableClaimTable* scan_state) :
+ G1RefineRegionClosure(G1CardTableClaimTable* scan_state) :
G1HeapRegionClosure(),
_rem_set(G1CollectedHeap::heap()->rem_set()),
_scan_state(scan_state),
- _worker_id(worker_id),
_completed(true),
_per_worker_refine_data() { }
@@ -164,8 +161,8 @@ class G1RefineRegionClosure : public G1HeapRegionClosure {
};
G1ConcurrentRefineSweepTask::G1ConcurrentRefineSweepTask(G1CardTableClaimTable* scan_state,
- G1ConcurrentRefineStats* stats,
- uint max_workers) :
+ G1ConcurrentRefineStats* stats,
+ uint max_workers) :
WorkerTask("G1 Refine Task"),
_scan_state(scan_state),
_stats(stats),
@@ -176,7 +173,7 @@ G1ConcurrentRefineSweepTask::G1ConcurrentRefineSweepTask(G1CardTableClaimTable*
void G1ConcurrentRefineSweepTask::work(uint worker_id) {
jlong start = os::elapsed_counter();
- G1RefineRegionClosure sweep_cl(worker_id, _scan_state);
+ G1RefineRegionClosure sweep_cl(_scan_state);
_scan_state->heap_region_iterate_from_worker_offset(&sweep_cl, worker_id, _max_workers);
if (!sweep_cl._completed) {
diff --git a/src/hotspot/share/gc/g1/g1FromCardCache.cpp b/src/hotspot/share/gc/g1/g1FromCardCache.cpp
deleted file mode 100644
index 8f5c84da0e3e..000000000000
--- a/src/hotspot/share/gc/g1/g1FromCardCache.cpp
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "gc/g1/g1FromCardCache.hpp"
-#include "gc/shared/gc_globals.hpp"
-#include "memory/padded.inline.hpp"
-#include "runtime/globals.hpp"
-#include "utilities/debug.hpp"
-
-uintptr_t** G1FromCardCache::_cache = nullptr;
-uint G1FromCardCache::_max_reserved_regions = 0;
-size_t G1FromCardCache::_static_mem_size = 0;
-#ifdef ASSERT
-uint G1FromCardCache::_max_workers = 0;
-#endif
-
-void G1FromCardCache::initialize(uint max_reserved_regions) {
- guarantee(max_reserved_regions > 0, "Heap size must be valid");
- guarantee(_cache == nullptr, "Should not call this multiple times");
-
- _max_reserved_regions = max_reserved_regions;
-#ifdef ASSERT
- _max_workers = num_par_rem_sets();
-#endif
- _cache = Padded2DArray::create_unfreeable(_max_reserved_regions,
- num_par_rem_sets(),
- &_static_mem_size);
-
- if (AlwaysPreTouch) {
- invalidate(0, _max_reserved_regions);
- }
-}
-
-void G1FromCardCache::invalidate(uint start_idx, size_t new_num_regions) {
- guarantee((size_t)start_idx + new_num_regions <= max_uintx,
- "Trying to invalidate beyond maximum region, from %u size %zu",
- start_idx, new_num_regions);
- uint end_idx = (start_idx + (uint)new_num_regions);
- assert(end_idx <= _max_reserved_regions, "Must be within max.");
-
- for (uint i = 0; i < num_par_rem_sets(); i++) {
- for (uint j = start_idx; j < end_idx; j++) {
- set(i, j, InvalidCard);
- }
- }
-}
-
-#ifndef PRODUCT
-void G1FromCardCache::print(outputStream* out) {
- for (uint i = 0; i < num_par_rem_sets(); i++) {
- for (uint j = 0; j < _max_reserved_regions; j++) {
- out->print_cr("_from_card_cache[%u][%u] = %zu.",
- i, j, at(i, j));
- }
- }
-}
-#endif
-
-uint G1FromCardCache::num_par_rem_sets() {
- return G1ConcRefinementThreads + ConcGCThreads;
-}
-
-void G1FromCardCache::clear(uint region_idx) {
- uint num_par_remsets = num_par_rem_sets();
- for (uint i = 0; i < num_par_remsets; i++) {
- set(i, region_idx, InvalidCard);
- }
-}
diff --git a/src/hotspot/share/gc/g1/g1FromCardCache.hpp b/src/hotspot/share/gc/g1/g1FromCardCache.hpp
index 0a01e0102aed..5f759be0c4fb 100644
--- a/src/hotspot/share/gc/g1/g1FromCardCache.hpp
+++ b/src/hotspot/share/gc/g1/g1FromCardCache.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,76 +25,44 @@
#ifndef SHARE_GC_G1_G1FROMCARDCACHE_HPP
#define SHARE_GC_G1_G1FROMCARDCACHE_HPP
-#include "memory/allStatic.hpp"
-#include "utilities/ostream.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/globalDefinitions.hpp"
-// G1FromCardCache remembers the most recently processed card on the heap on
-// a per-region and per-thread basis.
-class G1FromCardCache : public AllStatic {
-private:
- // Array of card indices. Indexed by heap region (rows) and thread (columns) to minimize
- // thread contention.
- // This order minimizes the time to clear all entries for a given region during region
- // freeing. I.e. a single clear of a single memory area instead of multiple separate
- // accesses with a large stride per region.
- static uintptr_t** _cache;
- static uint _max_reserved_regions;
- static size_t _static_mem_size;
-#ifdef ASSERT
- static uint _max_workers;
+// G1FromCardCache remembers which destination cset groups have been
+// encountered while a worker scans the current from_card.
+//
+// Refinement and remembered set rebuild scan the heap linearly, visiting
+// references from a card consecutively. Therefore, the cache only tracks
+// the destination cset groups found while scanning the current card. The
+// cache state is discarded when advancing to the next card.
+//
+// A scan can be suspended at a yield point. A GC may run while it is
+// suspended and change the cset group assignments. Therefore, the cache
+// must be reset before the scan resumes after every yield.
+class G1FromCardCache {
+ // Worst case: each reference in a card targets a different cset group.
+ static constexpr uint MaxGroupsPerCard = MaxGCCardSizeInBytes / sizeof(narrowOop);
- static void check_bounds(uint worker_id, uint region_idx) {
- assert(worker_id < _max_workers, "Worker_id %u is larger than maximum %u", worker_id, _max_workers);
- assert(region_idx < _max_reserved_regions, "Region_idx %u is larger than maximum %u", region_idx, _max_reserved_regions);
- }
-#endif
-
- // This card index indicates "no card for that entry" yet. This allows us to use the OS
- // lazy backing of memory with zero-filled pages to avoid initial actual memory use.
- // This means that the heap must not contain card zero.
- static const uintptr_t InvalidCard = 0;
+ uintptr_t _from_card;
+ uint _num_cset_groups;
+ uint _cset_group_ids[MaxGroupsPerCard];
- // Gives an approximation on how many threads can be expected to add records to
- // a remembered set in parallel. This is used for sizing the G1FromCardCache to
- // decrease performance losses due to data structure sharing.
- // Examples for quantities that influence this value are the maximum number of
- // mutator threads, maximum number of concurrent refinement or GC threads.
- static uint num_par_rem_sets();
+ NONCOPYABLE(G1FromCardCache);
public:
- static void clear(uint region_idx);
-
- // Returns true if the given card is in the cache at the given location, or
- // replaces the card at that location and returns false.
- static bool contains_or_replace(uint worker_id, uint region_idx, uintptr_t card) {
- uintptr_t card_in_cache = at(worker_id, region_idx);
- if (card_in_cache == card) {
- return true;
- } else {
- set(worker_id, region_idx, card);
- return false;
- }
- }
-
- static uintptr_t at(uint worker_id, uint region_idx) {
- DEBUG_ONLY(check_bounds(worker_id, region_idx);)
- return _cache[region_idx][worker_id];
- }
+ G1FromCardCache()
+ : _from_card(0),
+ _num_cset_groups(0) {}
- static void set(uint worker_id, uint region_idx, uintptr_t val) {
- DEBUG_ONLY(check_bounds(worker_id, region_idx);)
- _cache[region_idx][worker_id] = val;
+ // Discard the state associated with the _from_card.
+ void reset() {
+ _num_cset_groups = 0;
}
- static void initialize(uint max_reserved_regions);
-
- static void invalidate(uint start_idx, size_t num_regions);
-
- static void print(outputStream* out = tty) PRODUCT_RETURN;
-
- static size_t static_mem_size() {
- return _static_mem_size;
- }
+ // Returns true if cset_group_id has already been encountered while
+ // scanning from_card. Otherwise, records the id and returns false.
+ inline bool contains_or_add(uintptr_t from_card, uint cset_group_id);
};
#endif // SHARE_GC_G1_G1FROMCARDCACHE_HPP
diff --git a/test/hotspot/jtreg/vmTestbase/metaspace/gc/firstGC_50m/TestDescription.java b/src/hotspot/share/gc/g1/g1FromCardCache.inline.hpp
similarity index 58%
rename from test/hotspot/jtreg/vmTestbase/metaspace/gc/firstGC_50m/TestDescription.java
rename to src/hotspot/share/gc/g1/g1FromCardCache.inline.hpp
index 6df788cf3548..9a4abac3bc81 100644
--- a/test/hotspot/jtreg/vmTestbase/metaspace/gc/firstGC_50m/TestDescription.java
+++ b/src/hotspot/share/gc/g1/g1FromCardCache.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -19,26 +19,30 @@
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
+ *
*/
+#ifndef SHARE_GC_G1_G1FROMCARDCACHE_INLINE_HPP
+#define SHARE_GC_G1_G1FROMCARDCACHE_INLINE_HPP
-/*
- * @test
- * @modules java.base/jdk.internal.misc:+open
- *
- * @summary converted from VM Testbase metaspace/gc/firstGC_50m.
- * VM Testbase keywords: [nonconcurrent, quarantine]
- * VM Testbase comments: 8208250
- *
- * @library /vmTestbase /test/lib
- * @run main/othervm
- * -Xms200m
- * -Xlog:gc+heap=trace,gc:gc.log
- * -XX:MetaspaceSize=50m
- * -XX:+IgnoreUnrecognizedVMOptions
- * -XX:+UnlockDiagnosticVMOptions
- * -XX:-VerifyBeforeExit
- * -XX:-UseCompressedOops
- * metaspace.gc.FirstGCTest
- */
+#include "gc/g1/g1FromCardCache.hpp"
+
+bool G1FromCardCache::contains_or_add(uintptr_t from_card, uint cset_group_id) {
+ if (_from_card != from_card) {
+ _from_card = from_card;
+ _num_cset_groups = 0;
+ }
+
+ for (uint i = 0; i < _num_cset_groups; i++) {
+ if (_cset_group_ids[i] == cset_group_id) {
+ return true;
+ }
+ }
+
+ assert(_num_cset_groups < MaxGroupsPerCard, "from_card has too many destination cset groups");
+
+ _cset_group_ids[_num_cset_groups++] = cset_group_id;
+ return false;
+}
+#endif // SHARE_GC_G1_G1FROMCARDCACHE_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.cpp b/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.cpp
index 310cc4297c67..d6d39bafb34e 100644
--- a/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.cpp
@@ -31,8 +31,14 @@ G1FullGCResetMetadataTask::G1ResetMetadataClosure::G1ResetMetadataClosure(G1Full
_collector(collector) { }
void G1FullGCResetMetadataTask::G1ResetMetadataClosure::reset_region_metadata(G1HeapRegion* hr) {
- assert(hr->is_humongous() || !hr->rem_set()->has_cset_group(),
- "Non-humongous regions must not have cset group");
+ if (hr->rem_set()->has_cset_group()) {
+ assert(hr->is_starts_humongous(), "Only humongous regions can retain a cset group");
+ assert(hr->rem_set()->cset_group()->length() == 1,
+ "Humongous region cset group must contain exactly one region");
+
+ hr->rem_set()->cset_group()->clear_card_set();
+ }
+
hr->rem_set()->clear();
hr->clear_both_card_tables();
_g1h->concurrent_mark()->reset_region_marking_state(hr);
diff --git a/src/hotspot/share/gc/g1/g1HeapRegion.cpp b/src/hotspot/share/gc/g1/g1HeapRegion.cpp
index 2c85e2fcc0d5..a9a76eee634f 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegion.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.cpp
@@ -109,7 +109,12 @@ void G1HeapRegion::handle_evacuation_failure(bool retain) {
move_to_old();
_rem_set->clean_code_roots(this);
- _rem_set->clear(true /* only_cardset */, retain /* keep_tracked */);
+ assert(!_rem_set->has_cset_group(), "must not have a cset group");
+ if (retain) {
+ assert(_rem_set->is_tracked(), "must be");
+ } else {
+ _rem_set->set_state_untracked();
+ }
}
void G1HeapRegion::unlink_from_list() {
@@ -263,7 +268,7 @@ G1HeapRegion::G1HeapRegion(uint hrm_index,
assert(Universe::on_page_boundary(mr.start()) && Universe::on_page_boundary(mr.end()),
"invalid space boundaries");
- _rem_set = new G1HeapRegionRemSet(this);
+ _rem_set = new G1HeapRegionRemSet();
initialize();
}
@@ -391,7 +396,7 @@ bool G1HeapRegion::verify_code_roots(VerifyOption vo) const {
}
G1HeapRegionRemSet* hrrs = rem_set();
- size_t code_roots_length = hrrs->code_roots_list_length();
+ size_t code_roots_length = hrrs->code_roots_length();
// if this region is empty then there should be no entries
// on its code root list
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
index e2009b0e77d4..a965859a4146 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
@@ -22,24 +22,8 @@
*
*/
-#include "gc/g1/g1BlockOffsetTable.inline.hpp"
-#include "gc/g1/g1CardSetContainers.inline.hpp"
-#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/g1ConcurrentRefine.hpp"
-#include "gc/g1/g1HeapRegionManager.inline.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
-#include "memory/allocation.hpp"
-#include "memory/padded.inline.hpp"
-#include "oops/oop.inline.hpp"
-#include "runtime/globals_extension.hpp"
-#include "runtime/java.hpp"
-#include "runtime/mutexLocker.hpp"
-#include "utilities/bitMap.inline.hpp"
-#include "utilities/debug.hpp"
-#include "utilities/formatBuffer.hpp"
-#include "utilities/globalDefinitions.hpp"
-#include "utilities/growableArray.hpp"
-#include "utilities/powerOfTwo.hpp"
+#include "utilities/ostream.hpp"
HeapWord* G1HeapRegionRemSet::_heap_base_address = nullptr;
@@ -55,36 +39,19 @@ void G1HeapRegionRemSet::uninstall_cset_group() {
_cset_group = nullptr;
}
-G1HeapRegionRemSet::G1HeapRegionRemSet(G1HeapRegion* hr) :
+G1HeapRegionRemSet::G1HeapRegionRemSet() :
_code_roots(),
_cset_group(nullptr),
- _hr(hr),
_state(Untracked) { }
G1HeapRegionRemSet::~G1HeapRegionRemSet() {
assert(!has_cset_group(), "Still assigned to a CSet group");
}
-void G1HeapRegionRemSet::clear_fcc() {
- G1FromCardCache::clear(_hr->hrm_index());
-}
-
-void G1HeapRegionRemSet::clear(bool only_cardset, bool keep_tracked) {
- if (!only_cardset) {
- _code_roots.clear();
- }
- clear_fcc();
-
- if (has_cset_group()) {
- card_set()->clear();
- assert(card_set()->occupied() == 0, "Should be clear.");
- }
-
- if (!keep_tracked) {
- set_state_untracked();
- } else {
- assert(is_tracked(), "must be");
- }
+void G1HeapRegionRemSet::clear() {
+ assert(card_set_is_empty(), "Card set must be empty");
+ _code_roots.clear();
+ set_state_untracked();
}
void G1HeapRegionRemSet::reset_code_root_table_scanner() {
@@ -108,26 +75,12 @@ void G1HeapRegionRemSet::print_static_mem_size(outputStream* out) {
}
// Code roots support
-//
-// The code root set is protected by two separate locking schemes
-// When at safepoint the per-hrrs lock must be held during modifications
-// except when doing a full gc.
-// When not at safepoint the CodeCache_lock must be held during modifications.
void G1HeapRegionRemSet::add_code_root(nmethod* nm) {
assert(nm != nullptr, "sanity");
_code_roots.add(nm);
}
-void G1HeapRegionRemSet::remove_code_root(nmethod* nm) {
- assert(nm != nullptr, "sanity");
-
- _code_roots.remove(nm);
-
- // Check that there were no duplicates
- guarantee(!_code_roots.contains(nm), "duplicate entry found");
-}
-
void G1HeapRegionRemSet::bulk_remove_code_roots() {
_code_roots.bulk_remove();
}
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
index b185aa3151c2..2552df58b3a3 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
@@ -29,13 +29,8 @@
#include "gc/g1/g1CardSetMemory.hpp"
#include "gc/g1/g1CodeRootSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
-#include "gc/g1/g1FromCardCache.hpp"
-#include "runtime/mutexLocker.hpp"
-#include "runtime/safepoint.hpp"
-#include "utilities/bitMap.hpp"
-class G1CardSetMemoryManager;
-class G1CSetCandidateGroup;
+class G1FromCardCache;
class outputStream;
class G1HeapRegionRemSet : public CHeapObj {
@@ -43,16 +38,11 @@ class G1HeapRegionRemSet : public CHeapObj {
// the region that owns this RSet.
G1CodeRootSet _code_roots;
- // The collection set groups to which the region owning this RSet is assigned.
G1CSetCandidateGroup* _cset_group;
- G1HeapRegion* _hr;
-
// Cached value of heap base address.
static HeapWord* _heap_base_address;
- void clear_fcc();
-
G1CardSet* card_set() {
assert(has_cset_group(), "pre-condition");
return cset_group()->card_set();
@@ -63,14 +53,14 @@ class G1HeapRegionRemSet : public CHeapObj {
return cset_group()->card_set();
}
-public:
- G1HeapRegionRemSet(G1HeapRegion* hr);
- ~G1HeapRegionRemSet();
-
- bool cardset_is_empty() const {
+ bool card_set_is_empty() const {
return !has_cset_group() || card_set()->is_empty();
}
+public:
+ G1HeapRegionRemSet();
+ ~G1HeapRegionRemSet();
+
void install_cset_group(G1CSetCandidateGroup* cset_group) {
assert(cset_group != nullptr, "pre-condition");
assert(_cset_group == nullptr, "pre-condition");
@@ -98,14 +88,14 @@ class G1HeapRegionRemSet : public CHeapObj {
}
bool is_empty() const {
- return (code_roots_list_length() == 0) && cardset_is_empty();
+ return (code_roots_length() == 0) && card_set_is_empty();
}
bool occupancy_less_or_equal_than(size_t occ) const {
- return (code_roots_list_length() == 0) && card_set()->occupancy_less_or_equal_to(occ);
+ return (code_roots_length() == 0) && card_set()->occupancy_less_or_equal_to(occ);
}
- // Iterate the card based remembered set for merging them into the card table.
+ // Iterate the cards in this remembered set for merging them into the card table.
// The passed closure must be a CardOrRangeVisitor; we use a template parameter
// to pass it in to facilitate inlining as much as possible.
template
@@ -119,7 +109,6 @@ class G1HeapRegionRemSet : public CHeapObj {
return card_set()->occupied();
}
-
static void initialize(MemRegion reserved);
inline uintptr_t to_card(OopOrNarrowOopStar from) const;
@@ -148,11 +137,10 @@ class G1HeapRegionRemSet : public CHeapObj {
inline void set_state_updating();
inline void set_state_complete();
- inline void add_reference(OopOrNarrowOopStar from, uint tid);
+ inline void add_reference(OopOrNarrowOopStar from, G1FromCardCache& from_card_cache);
- // The region is being reclaimed; clear its remset, and any mention of
- // entries for this region in other remsets.
- void clear(bool only_cardset = false, bool keep_tracked = false);
+ // Clear the region-specific remset state.
+ void clear();
void reset_code_root_table_scanner();
void reset_table_scanner();
@@ -162,13 +150,13 @@ class G1HeapRegionRemSet : public CHeapObj {
// The actual # of bytes this hr_remset takes up. Also includes the code
// root set.
size_t mem_size() {
- return sizeof(G1HeapRegionRemSet) + code_roots_mem_size();
+ return sizeof(G1HeapRegionRemSet) - sizeof(G1CodeRootSet) + code_roots_mem_size();
}
// Returns the memory occupancy of all static data structures associated
// with remembered sets.
static size_t static_mem_size() {
- return G1CardSet::static_mem_size() + G1FromCardCache::static_mem_size();
+ return G1CardSet::static_mem_size();
}
static void print_static_mem_size(outputStream* out);
@@ -177,10 +165,9 @@ class G1HeapRegionRemSet : public CHeapObj {
inline void print_info(outputStream* st, OopOrNarrowOopStar from);
- // Routines for managing the list of code roots that point into
- // the heap region that owns this RSet.
+ // Routines for managing the code roots that point into the heap region
+ // that owns this RSet.
void add_code_root(nmethod* nm);
- void remove_code_root(nmethod* nm);
void bulk_remove_code_roots();
void prepare_for_adding_code_roots(size_t num_code_roots);
@@ -190,13 +177,13 @@ class G1HeapRegionRemSet : public CHeapObj {
void clean_code_roots(G1HeapRegion* hr);
// Returns the number of elements in _code_roots
- size_t code_roots_list_length() const {
+ size_t code_roots_length() const {
return _code_roots.length();
}
// Returns true if the code roots contains the given
// nmethod.
- bool code_roots_list_contains(nmethod* nm) {
+ bool code_roots_contains(nmethod* nm) {
return _code_roots.contains(nm);
}
@@ -204,15 +191,7 @@ class G1HeapRegionRemSet : public CHeapObj {
// consumed by the code roots.
size_t code_roots_mem_size();
- static void invalidate_from_card_cache(uint start_idx, size_t num_regions) {
- G1FromCardCache::invalidate(start_idx, num_regions);
- }
-
#ifndef PRODUCT
- static void print_from_card_cache() {
- G1FromCardCache::print();
- }
-
static void test();
#endif
};
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
index f621b1318c1e..25b1fbebfff5 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
@@ -22,15 +22,16 @@
*
*/
-#ifndef SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
-#define SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1CardSet.inline.hpp"
-#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/g1HeapRegion.inline.hpp"
-#include "utilities/bitMap.inline.hpp"
+#include "gc/g1/g1CollectionSetCandidates.hpp"
+#include "gc/g1/g1FromCardCache.inline.hpp"
+#include "gc/shared/cardTable.hpp"
+#include "runtime/safepoint.hpp"
void G1HeapRegionRemSet::set_state_untracked() {
guarantee(SafepointSynchronize::is_at_safepoint() || !is_tracked(),
@@ -38,19 +39,16 @@ void G1HeapRegionRemSet::set_state_untracked() {
if (_state == Untracked) {
return;
}
- clear_fcc();
_state = Untracked;
}
void G1HeapRegionRemSet::set_state_updating() {
guarantee(SafepointSynchronize::is_at_safepoint() && !is_tracked(),
"Should only set to Updating from Untracked during safepoint but is %s", get_state_str());
- clear_fcc();
_state = Updating;
}
void G1HeapRegionRemSet::set_state_complete() {
- clear_fcc();
_state = Complete;
}
@@ -123,18 +121,15 @@ uintptr_t G1HeapRegionRemSet::to_card(OopOrNarrowOopStar from) const {
return pointer_delta(from, _heap_base_address, 1) >> CardTable::card_shift();
}
-void G1HeapRegionRemSet::add_reference(OopOrNarrowOopStar from, uint tid) {
- assert(has_cset_group(), "pre-condition");
+void G1HeapRegionRemSet::add_reference(OopOrNarrowOopStar from, G1FromCardCache& from_card_cache) {
+ precond(has_cset_group());
+ precond(_state != Untracked);
- assert(_state != Untracked, "must be");
-
- uint cur_idx = _hr->hrm_index();
uintptr_t from_card = uintptr_t(from) >> CardTable::card_shift();
- if (G1FromCardCache::contains_or_replace(tid, cur_idx, from_card)) {
+ if (from_card_cache.contains_or_add(from_card, cset_group()->group_id())) {
// We can't check whether the card is in the remembered set - the card container
// may be coarsened just now.
- //assert(contains_reference(from), "We just found " PTR_FORMAT " in the FromCardCache", p2i(from));
return;
}
@@ -149,4 +144,4 @@ void G1HeapRegionRemSet::print_info(outputStream* st, OopOrNarrowOopStar from) {
card_set()->print_info(st, to_card(from));
}
-#endif // SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
index da8953f5a7df..b477138873a7 100644
--- a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
@@ -115,7 +115,7 @@ class G1VerifyCodeRootOopClosure: public OopClosure {
G1HeapRegionRemSet* hrrs = hr->rem_set();
// Verify that the code root list for this region
// contains the nmethod
- if (!hrrs->code_roots_list_contains(_nm)) {
+ if (!hrrs->code_roots_contains(_nm)) {
log_error(gc, verify)("Code root location " PTR_FORMAT " "
"from nmethod " PTR_FORMAT " not in strong "
"code roots for region [" PTR_FORMAT "," PTR_FORMAT ")",
diff --git a/src/hotspot/share/gc/g1/g1OopClosures.hpp b/src/hotspot/share/gc/g1/g1OopClosures.hpp
index a61c9d17f70c..b6cbb765280d 100644
--- a/src/hotspot/share/gc/g1/g1OopClosures.hpp
+++ b/src/hotspot/share/gc/g1/g1OopClosures.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
#define SHARE_GC_G1_G1OOPCLOSURES_HPP
#include "classfile/classLoaderData.hpp"
+#include "gc/g1/g1FromCardCache.hpp"
#include "gc/g1/g1HeapRegionAttr.hpp"
#include "memory/iterator.hpp"
#include "oops/markWord.hpp"
@@ -205,17 +206,16 @@ class G1RootRegionScanClosure : public ClaimMetadataVisitingOopIterateClosure {
class G1ConcurrentRefineOopClosure: public BasicOopIterateClosure {
G1CollectedHeap* _g1h;
- uint _worker_id;
+ G1FromCardCache _from_card_cache;
bool _has_ref_to_cset;
bool _has_ref_to_old;
public:
- G1ConcurrentRefineOopClosure(G1CollectedHeap* g1h, uint worker_id) :
+ G1ConcurrentRefineOopClosure(G1CollectedHeap* g1h) :
_g1h(g1h),
- _worker_id(worker_id),
+ _from_card_cache(),
_has_ref_to_cset(false),
- _has_ref_to_old(false) {
- }
+ _has_ref_to_old(false) {}
bool has_ref_to_cset() const { return _has_ref_to_cset; }
bool has_ref_to_old() const { return _has_ref_to_old; }
@@ -229,11 +229,14 @@ class G1ConcurrentRefineOopClosure: public BasicOopIterateClosure {
class G1RebuildRemSetClosure : public BasicOopIterateClosure {
G1CollectedHeap* _g1h;
- uint _worker_id;
+ G1FromCardCache _from_card_cache;
public:
- G1RebuildRemSetClosure(G1CollectedHeap* g1h, uint worker_id) : _g1h(g1h), _worker_id(worker_id) {
- }
+ G1RebuildRemSetClosure(G1CollectedHeap* g1h)
+ : _g1h(g1h),
+ _from_card_cache() {}
+
+ void reset_from_card_cache() { _from_card_cache.reset(); }
template void do_oop_work(T* p);
virtual void do_oop(oop* p) { do_oop_work(p); }
diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
index 80fb1be14ddb..aed36f8738fd 100644
--- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -164,7 +164,7 @@ inline void G1ConcurrentRefineOopClosure::do_oop_work(T* p) {
G1HeapRegion* from = _g1h->heap_region_containing(p);
if (from->rem_set()->cset_group() != to_rem_set->cset_group()) {
- to_rem_set->add_reference(p, _worker_id);
+ to_rem_set->add_reference(p, _from_card_cache);
_has_ref_to_old = true;
}
}
@@ -291,7 +291,7 @@ template void G1RebuildRemSetClosure::do_oop_work(T* p) {
G1HeapRegion* from = _g1h->heap_region_containing(p);
if (from->rem_set()->cset_group() != rem_set->cset_group()) {
- rem_set->add_reference(p, _worker_id);
+ rem_set->add_reference(p, _from_card_cache);
}
}
}
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index 3a7ff7adc4f1..f120ac320357 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -351,8 +351,7 @@ G1HeapRegionAttr G1ParScanThreadState::next_region_attr(G1HeapRegionAttr const r
assert(region_attr.is_young() || region_attr.is_old(), "must be either Young or Old");
if (region_attr.is_young()) {
- age = !m.has_displaced_mark_helper() ? m.age()
- : m.displaced_mark_helper().age();
+ age = m.age();
if (age < _tenuring_threshold) {
return region_attr;
}
diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp
index fc63a6e212a9..66dd3967e387 100644
--- a/src/hotspot/share/gc/g1/g1Policy.cpp
+++ b/src/hotspot/share/gc/g1/g1Policy.cpp
@@ -69,6 +69,7 @@ G1Policy::G1Policy(STWGCTimer* gc_timer) :
_young_gen_sizer(),
_free_regions_at_end_of_collection(0),
_pending_cards_from_gc(0),
+ _to_collection_set_cards(0),
_collection_set(nullptr),
_g1h(nullptr),
_phase_times_timer(gc_timer),
@@ -1183,7 +1184,7 @@ double G1Policy::predict_merge_scan_time(size_t card_rs_length) const {
}
double G1Policy::predict_region_code_root_scan_time(G1HeapRegion* hr, bool for_young_only_phase) const {
- size_t code_root_length = hr->rem_set()->code_roots_list_length();
+ size_t code_root_length = hr->rem_set()->code_roots_length();
return
_analytics->predict_code_root_scan_time_ms(code_root_length, for_young_only_phase);
diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp
index 149f1da1a8bf..5261d39e715e 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp
@@ -34,7 +34,6 @@
#include "gc/g1/g1CollectorState.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineSweepTask.hpp"
-#include "gc/g1/g1FromCardCache.hpp"
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
@@ -201,8 +200,8 @@ class G1ClearCardTableTask : public G1AbstractSubTask {
return AlmostNoWork;
}
- double num_cards = num_regions << G1HeapRegion::LogCardsPerRegion;
- return ceil(num_cards / num_cards_per_worker);
+ size_t num_cards = (size_t)num_regions << G1HeapRegion::LogCardsPerRegion;
+ return align_up(num_cards, num_cards_per_worker) / num_cards_per_worker;
}
virtual ~G1ClearCardTableTask() {
@@ -1078,7 +1077,10 @@ class G1MergeHeapRootsTask : public WorkerTask {
// remembered sets for this region.
// We want to continue collecting remembered set entries for humongous regions
// that were not reclaimed.
- r->rem_set()->clear(true /* only_cardset */, true /* keep_tracked */);
+ G1CSetCandidateGroup* group = r->rem_set()->cset_group();
+ assert(group != nullptr, "must have a cset group");
+ assert(group->length() == 1, "humongous regions cset group must have a single entry");
+ group->clear_card_set();
}
// Postcondition
@@ -1264,8 +1266,7 @@ inline void check_card_ptr(CardTable::CardValue* card_ptr, G1CardTable* ct) {
#endif
}
-G1RemSet::RefineResult G1RemSet::refine_card_concurrently(CardValue* const card_ptr,
- const uint worker_id) {
+G1RemSet::RefineResult G1RemSet::refine_card_concurrently(CardValue* const card_ptr) {
assert(!_g1h->is_stw_gc_active(), "Only call concurrently");
G1CardTable* ct = _g1h->refinement_table();
check_card_ptr(card_ptr, ct);
@@ -1295,7 +1296,7 @@ G1RemSet::RefineResult G1RemSet::refine_card_concurrently(CardValue* const card_
MemRegion dirty_region(start, MIN2(scan_limit, end));
assert(!dirty_region.is_empty(), "sanity");
- G1ConcurrentRefineOopClosure conc_refine_cl(_g1h, worker_id);
+ G1ConcurrentRefineOopClosure conc_refine_cl(_g1h);
if (r->oops_on_memregion_seq_iterate_careful(dirty_region, &conc_refine_cl) != nullptr) {
if (conc_refine_cl.has_ref_to_cset()) {
return HasRefToCSet;
diff --git a/src/hotspot/share/gc/g1/g1RemSet.hpp b/src/hotspot/share/gc/g1/g1RemSet.hpp
index 4893e0839d00..4b079ae297b3 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.hpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.hpp
@@ -126,8 +126,7 @@ class G1RemSet: public CHeapObj {
// Refine the region corresponding to "card_ptr". Must be called after
// being filtered by clean_card_before_refine(), and after proper
// fence/synchronization.
- RefineResult refine_card_concurrently(CardValue* const card_ptr,
- const uint worker_id);
+ RefineResult refine_card_concurrently(CardValue* const card_ptr);
// Print accumulated summary info from the start of the VM.
void print_summary_info();
diff --git a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
index 1c0e15757cc3..00682df647d2 100644
--- a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
@@ -269,7 +269,7 @@ class G1HeapRegionStatsClosure: public G1HeapRegionClosure {
_max_code_root_mem_sz = code_root_mem_sz;
_max_code_root_mem_sz_region = r;
}
- size_t code_root_elems = hrrs->code_roots_list_length();
+ size_t code_root_elems = hrrs->code_roots_length();
G1PerRegionTypeRemSetCounters* current = nullptr;
if (r->is_free()) {
@@ -392,7 +392,7 @@ class G1HeapRegionStatsClosure: public G1HeapRegionClosure {
HR_FORMAT_PARAMS(max_code_root_mem_sz_region()),
byte_size_in_proper_unit(max_code_root_rem_set->code_roots_mem_size()),
proper_unit_for_byte_size(max_code_root_rem_set->code_roots_mem_size()),
- max_code_root_rem_set->code_roots_list_length());
+ max_code_root_rem_set->code_roots_length());
}
};
diff --git a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
index 94f5466b8e0f..587ef2b6a12a 100644
--- a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
@@ -102,11 +102,17 @@ void G1RemSetTrackingPolicy::update_after_rebuild(G1HeapRegion* r) {
// cycle as e.g. remembered set entries will always be added.
if (r->is_starts_humongous() && !g1h->is_potential_eager_reclaim_candidate(r)) {
// Handle HC regions with the HS region.
+ G1CSetCandidateGroup* group = r->rem_set()->cset_group();
+
+ assert(group != nullptr, "humongous start must have a cset group");
+ assert(group->length() == 1, "humongous group must have only one region");
+
+ group->clear_card_set();
g1h->humongous_obj_regions_iterate(r,
[&] (G1HeapRegion* r) {
assert(!r->is_continues_humongous() || r->rem_set()->is_empty(),
"Continues humongous region %u remset should be empty", r->hrm_index());
- r->rem_set()->clear(true /* only_cardset */);
+ r->rem_set()->set_state_untracked();
});
}
diff --git a/src/hotspot/share/gc/g1/g1ReviseNumYoungRegionsTask.cpp b/src/hotspot/share/gc/g1/g1ReviseNumYoungRegionsTask.cpp
index 71c8d7bf772c..22cee3bb457f 100644
--- a/src/hotspot/share/gc/g1/g1ReviseNumYoungRegionsTask.cpp
+++ b/src/hotspot/share/gc/g1/g1ReviseNumYoungRegionsTask.cpp
@@ -55,7 +55,7 @@ class G1ReviseNumYoungRegionsTask::RemSetSamplingClosure : public G1HeapRegionCl
bool do_heap_region(G1HeapRegion* r) override {
G1HeapRegionRemSet* rem_set = r->rem_set();
- _sampled_code_root_rs_length += rem_set->code_roots_list_length();
+ _sampled_code_root_rs_length += rem_set->code_roots_length();
return false;
}
diff --git a/src/hotspot/share/gc/g1/g1YoungCollector.cpp b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
index 359ed4586c17..f8137a162016 100644
--- a/src/hotspot/share/gc/g1/g1YoungCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
@@ -436,7 +436,7 @@ class G1PrepareEvacuationTask : public WorkerTask {
cast_to_oop(hr->bottom())->size() * HeapWordSize,
p2i(hr->bottom()),
hr->rem_set()->occupied(),
- hr->rem_set()->code_roots_list_length(),
+ hr->rem_set()->code_roots_length(),
_g1h->concurrent_mark()->mark_bitmap()->is_marked(hr->bottom()),
hr->pinned_count(),
_g1h->is_humongous_reclaim_candidate(index),
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
index 1d8e358c72b7..2f7fb7cd970e 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2002, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -249,8 +249,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
size_t new_obj_size = o->size_given_klass(klass);
// Find the objects age, MT safe.
- uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
- test_mark.displaced_mark_helper().age() : test_mark.age();
+ uint age = test_mark.age();
if (!promote_immediately) {
// Try allocating obj in to-space (unless too old)
diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
index a0876a8842c9..8ff93029e832 100644
--- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
+++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp
@@ -316,8 +316,6 @@ class BarrierSetC2: public CHeapObj {
Node*& fast_oop_ctrl, Node*& fast_oop_rawmem,
intx prefetch_lines) const;
- virtual Node* ideal_node(PhaseGVN* phase, Node* n, bool can_reshape) const { return nullptr; }
-
// These are general helper methods used by C2
enum ArrayCopyPhase {
Parsing,
@@ -328,19 +326,9 @@ class BarrierSetC2: public CHeapObj {
virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const { return false; }
virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const;
- // Support for GC barriers emitted during parsing
- virtual bool has_load_barrier_nodes() const { return false; }
- virtual bool is_gc_pre_barrier_node(Node* node) const { return false; }
- virtual bool is_gc_barrier_node(Node* node) const { return false; }
- virtual Node* step_over_gc_barrier(Node* c) const { return c; }
-
// Support for macro expanded GC barriers
- virtual void register_potential_barrier_node(Node* node) const { }
- virtual void unregister_potential_barrier_node(Node* node) const { }
virtual void eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const { }
virtual void eliminate_gc_barrier_data(Node* node) const { }
- virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {}
- virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {}
// Allow barrier sets to have shared state that is preserved across a compilation unit.
// This could for example comprise macro nodes to be expanded during macro expansion.
@@ -349,9 +337,6 @@ class BarrierSetC2: public CHeapObj {
// unit state to be expanded later, then now is the time to do so.
virtual bool expand_barriers(Compile* C, PhaseIterGVN& igvn) const { return false; }
virtual void final_refinement(Compile* C) const { }
- virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { return false; }
- virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return false; }
- virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return false; }
// Estimated size of the node barrier in number of C2 Ideal nodes.
// This is used to guide heuristics in C2, e.g. whether to unroll a loop.
virtual uint estimated_barrier_size(const Node* node) const { return 0; }
@@ -368,15 +353,6 @@ class BarrierSetC2: public CHeapObj {
virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const {}
#endif
- virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode, Unique_Node_List& dead_nodes) const { return false; }
-
- virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const { return false; }
- virtual bool escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const { return false; }
- virtual bool escape_has_out_with_unsafe_object(Node* n) const { return false; }
-
- virtual bool matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { return false; };
- virtual bool matcher_is_store_load_barrier(Node* x, uint xop) const { return false; }
-
// Whether the given phi node joins OOPs from fast and slow allocation paths.
static bool is_allocation(const Node* node);
// Elide GC barriers from a Mach node according to elide_dominated_barriers().
diff --git a/src/hotspot/share/gc/shared/gc_globals.hpp b/src/hotspot/share/gc/shared/gc_globals.hpp
index 336f4bd59a12..2eeee2b0cac6 100644
--- a/src/hotspot/share/gc/shared/gc_globals.hpp
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp
@@ -46,6 +46,8 @@
#include "gc/z/z_globals.hpp"
#endif
+constexpr uint MaxGCCardSizeInBytes = NOT_LP64(512) LP64_ONLY(1024);
+
#define GC_FLAGS(develop, \
develop_pd, \
product, \
@@ -523,7 +525,7 @@
\
product(uint, GCCardSizeInBytes, 512, \
"Card table entry size (in bytes) for card based collectors") \
- range(128, NOT_LP64(512) LP64_ONLY(1024)) \
+ range(128, MaxGCCardSizeInBytes) \
constraint(GCCardSizeInBytesConstraintFunc,AtParse)
// end of GC_FLAGS
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
index 1bac056a2253..298c756ef2b3 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
@@ -40,10 +40,6 @@
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
-ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
- return reinterpret_cast(BarrierSet::barrier_set()->barrier_set_c2());
-}
-
ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) :
BarrierSetC2State(comp_arena),
_stubs(new (comp_arena) GrowableArray(comp_arena, 8, 0, nullptr)),
@@ -667,10 +663,6 @@ void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
}
-ShenandoahBarrierSetC2State* ShenandoahBarrierSetC2::state() const {
- return reinterpret_cast(Compile::current()->barrier_set_state());
-}
-
void ShenandoahBarrierSetC2::print_barrier_data(outputStream* os, uint8_t data) {
os->print(" Node barriers: ");
if ((data & ShenandoahBitStrong) != 0) {
@@ -888,7 +880,7 @@ void ShenandoahBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
skipped_after, skipped_before, skipped_after - skipped_before);
#endif
- masm.flush();
+ // Code will be copied. No ICache sync required.
}
void ShenandoahBarrierStubC2::register_stub(ShenandoahBarrierStubC2* stub) {
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
index 097e28a562ef..065c746d93ce 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp
@@ -110,10 +110,6 @@ class ShenandoahBarrierSetC2 : public BarrierSetC2 {
virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const;
public:
- static ShenandoahBarrierSetC2* bsc2();
-
- ShenandoahBarrierSetC2State* state() const;
-
// This is the entry-point for the backend to perform accesses through the Access API.
virtual void clone(GraphKit* kit, Node* src_base, Node* dst_base, Node* size, bool is_array) const;
virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const;
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
index dd259497d341..55a582cd54c2 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp
@@ -228,7 +228,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
PROPERFMTARGS(available), PROPERFMTARGS(capacity));
if (_start_gc_is_pending) {
- log_trigger("GC start is already pending");
+ log_info(gc, ergo)("GC start is already pending");
return true;
}
@@ -263,7 +263,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() {
bool ShenandoahAdaptiveHeuristics::trigger_min_free_threshold(size_t available, size_t capacity) {
const size_t min_threshold = min_free_threshold(capacity);
if (available < min_threshold) {
- log_trigger("Free (Soft) (" PROPERFMT ") is below minimum threshold (" PROPERFMT ")",
+ log_trigger("Occupancy. " PROPERFMT " free, below " PROPERFMT " threshold",
PROPERFMTARGS(available), PROPERFMTARGS(min_threshold));
accept_trigger_with_type(OTHER);
return true;
@@ -276,8 +276,9 @@ bool ShenandoahAdaptiveHeuristics::trigger_learning(size_t available, size_t cap
if (_gc_times_learned < ShenandoahLearningSteps) {
const size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold;
if (available < init_threshold) {
- log_trigger("Learning %zu of %zu. Free (" PROPERFMT ") is below initial threshold (" PROPERFMT ")",
- _gc_times_learned + 1, ShenandoahLearningSteps, PROPERFMTARGS(available), PROPERFMTARGS(init_threshold));
+ log_trigger("Learning. Step %zu of %zu, " PROPERFMT " free, below " PROPERFMT " threshold",
+ _gc_times_learned + 1, ShenandoahLearningSteps,
+ PROPERFMTARGS(available), PROPERFMTARGS(init_threshold));
accept_trigger_with_type(OTHER);
return true;
}
@@ -287,10 +288,11 @@ bool ShenandoahAdaptiveHeuristics::trigger_learning(size_t available, size_t cap
bool ShenandoahAdaptiveHeuristics::trigger_average_allocation_rate(const ShenandoahAnticipatedConsumption& rate, const size_t allocatable_bytes) {
if (rate.baseline_consumption() > allocatable_bytes) {
- log_trigger("Anticipated GC duration (%.2f ms) is above the time for average allocation rate (" PROPERFMT_F "/s)"
- " to deplete free headroom (" PROPERFMT ") (margin of error = %.2f)",
- rate.duration_seconds() * 1000,
- PROPERFMT_F_ARGS(rate.baseline_rate()), PROPERFMTARGS(allocatable_bytes), _margin_of_error_sd);
+ const ShenandoahSignedSize baseline_rate = ShenandoahSignedSize::get(rate.baseline_rate());
+ log_trigger("Allocation Rate. %.2fms GC predicted, " PROPERFMT " free, "
+ PROPERFMT_F "/s average allocation rate",
+ rate.duration_seconds() * 1000, PROPERFMTARGS(allocatable_bytes),
+ PROPERFMTARGS_SIGNED(baseline_rate));
accept_trigger_with_type(RATE);
return true;
}
@@ -383,10 +385,10 @@ bool ShenandoahAdaptiveHeuristics::trigger_accelerating_allocation_rate(const Sh
if (rate.momentary_consumption() > allocatable_bytes) {
const ShenandoahSignedSize momentary_rate = ShenandoahSignedSize::get(rate.momentary_rate());
assert(rate.accelerated_consumption() == 0, "Momentary trigger is meant to exclude acceleration trigger");
- log_trigger("Momentary spike consumption (" PROPERFMT ") exceeds free headroom (" PROPERFMT ") at "
- "current rate (" PROPERFMT_F "/s) for anticipated GC duration (%.2f ms)",
- PROPERFMTARGS(rate.momentary_consumption()), PROPERFMTARGS(allocatable_bytes),
- PROPERFMTARGS_SIGNED(momentary_rate), rate.duration_seconds() * 1000);
+ log_trigger("Allocation Rate. %.2fms GC predicted, " PROPERFMT " free, "
+ PROPERFMT_F "/s momentary allocation rate",
+ rate.duration_seconds() * 1000, PROPERFMTARGS(allocatable_bytes),
+ PROPERFMTARGS_SIGNED(momentary_rate));
accept_trigger_with_type(RATE);
return true;
}
@@ -395,10 +397,10 @@ bool ShenandoahAdaptiveHeuristics::trigger_accelerating_allocation_rate(const Sh
const ShenandoahSignedSize predicted_rate = ShenandoahSignedSize::get(rate.predicted_rate());
const ShenandoahSignedSize acceleration = ShenandoahSignedSize::get(rate.acceleration());
assert(rate.momentary_consumption() == 0, "Acceleration trigger is meant to exclude momentary trigger");
- log_trigger("Accelerated consumption (" PROPERFMT ") exceeds free headroom (" PROPERFMT ") at "
- "current rate (" PROPERFMT_F "/s) with acceleration (" PROPERFMT_F "/s/s) for anticipated GC duration (%.2f ms)",
- PROPERFMTARGS(rate.accelerated_consumption()), PROPERFMTARGS(allocatable_bytes),
- PROPERFMTARGS_SIGNED(predicted_rate), PROPERFMTARGS_SIGNED(acceleration), rate.duration_seconds() * 1000);
+ log_trigger("Allocation Rate. %.2fms GC predicted, " PROPERFMT " free, "
+ PROPERFMT_F "/s predicted allocation rate, " PROPERFMT_F "/s^2 acceleration",
+ rate.duration_seconds() * 1000, PROPERFMTARGS(allocatable_bytes),
+ PROPERFMTARGS_SIGNED(predicted_rate), PROPERFMTARGS_SIGNED(acceleration));
accept_trigger_with_type(RATE);
return true;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
index 26a2363d4d50..a11820e5a809 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -60,14 +60,14 @@ bool ShenandoahCompactHeuristics::should_start_gc() {
const size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold;
if (available < min_threshold) {
- log_trigger("Free (Soft) (" PROPERFMT ") is below minimum threshold (" PROPERFMT ")",
+ log_trigger("Occupancy. " PROPERFMT " free, below " PROPERFMT " threshold",
PROPERFMTARGS(available), PROPERFMTARGS(min_threshold));
accept_trigger();
return true;
}
if (bytes_allocated > threshold_bytes_allocated) {
- log_trigger("Allocated since last cycle started (" PROPERFMT ") is larger than allocation threshold (" PROPERFMT ")",
+ log_trigger("Allocation. " PROPERFMT " allocated, above " PROPERFMT " threshold",
PROPERFMTARGS(bytes_allocated), PROPERFMTARGS(threshold_bytes_allocated));
accept_trigger();
return true;
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
index dbc795651f22..31f32836ceb3 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGenerationalHeuristics.cpp
@@ -29,6 +29,7 @@
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahGenerationalHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahInPlacePromoter.hpp"
#include "gc/shenandoah/shenandoahOldGeneration.hpp"
@@ -107,9 +108,14 @@ size_t ShenandoahGenerationalHeuristics::prepare_regions_for_promotion(Shenandoa
assert_no_in_place_promotions();
size_t candidates = 0;
for (size_t i = 0, num_regions = heap->num_regions(); i < num_regions; i++) {
+ if (!heap->is_region_young(i)) {
+ // Skip regions that aren't young
+ continue;
+ }
+
ShenandoahHeapRegion* const r = heap->get_region(i);
- if (r->is_empty() || !r->has_live() || !r->is_young()) {
- // skip over regions that aren't young with some live data
+ if (r->is_empty() || !r->has_live()) {
+ // Skip over regions that don't have live data
continue;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
index a0aec3c70a29..d0a5d0e0cc10 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
@@ -1,7 +1,7 @@
/*
* Copyright (c) 2018, 2026, Red Hat, Inc. All rights reserved.
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
#include "gc/shenandoah/shenandoahAllocRate.inline.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahOldGeneration.hpp"
@@ -100,12 +101,11 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec
size_t free_regions = 0;
for (size_t i = 0; i < num_regions; i++) {
- ShenandoahHeapRegion* region = heap->get_region(i);
-
- if (!_space_info->contains(region)) {
+ if (!_space_info->contains(heap->region_affiliation(i))) {
continue;
}
+ ShenandoahHeapRegion* region = heap->get_region(i);
size_t garbage = region->garbage();
total_garbage += garbage;
@@ -189,7 +189,7 @@ void ShenandoahHeuristics::record_cycle_end() {
bool ShenandoahHeuristics::should_start_gc() {
if (_start_gc_is_pending) {
- log_trigger("GC start is already pending");
+ log_info(gc, ergo)("GC start is already pending");
return true;
}
// Perform GC to cleanup metaspace
@@ -203,8 +203,8 @@ bool ShenandoahHeuristics::should_start_gc() {
if (_guaranteed_gc_interval > 0) {
double last_time_ms = (os::elapsedTime() - _last_cycle_end) * 1000;
if (last_time_ms > _guaranteed_gc_interval) {
- log_trigger("Time since last GC (%.0f ms) is larger than guaranteed interval (%zu ms)",
- last_time_ms, _guaranteed_gc_interval);
+ log_trigger("Guaranteed Interval. %.0f ms since last GC, above %zu ms guaranteed interval",
+ last_time_ms, _guaranteed_gc_interval);
accept_trigger();
return true;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
index 44bc683cbd6c..1b956b6994a3 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp
@@ -1,6 +1,6 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahOldGeneration.hpp"
#include "gc/shenandoah/shenandoahYoungGeneration.hpp"
@@ -413,11 +414,11 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() {
size_t live_data = 0;
RegionData* candidates = _region_data;
for (size_t i = 0; i < num_regions; i++) {
- ShenandoahHeapRegion* region = heap->get_region(i);
- if (!region->is_old()) {
+ if (!heap->is_region_old(i)) {
continue;
}
+ ShenandoahHeapRegion* region = heap->get_region(i);
size_t garbage = region->garbage();
size_t live_bytes = region->get_live_data_bytes();
if (!region->was_promoted_in_place()) {
@@ -724,13 +725,13 @@ bool ShenandoahOldHeuristics::should_resume_old_cycle() {
// If we are preparing to mark old, or if we are already marking old, then try to continue that work.
if (_old_generation->is_concurrent_mark_in_progress()) {
assert(_old_generation->state() == ShenandoahOldGeneration::MARKING, "Unexpected old gen state: %s", _old_generation->state_name());
- log_trigger("Resume marking old");
+ log_trigger("Resume Marking");
return true;
}
if (_old_generation->is_preparing_for_mark()) {
assert(_old_generation->state() == ShenandoahOldGeneration::FILLING, "Unexpected old gen state: %s", _old_generation->state_name());
- log_trigger("Resume preparing to mark old");
+ log_trigger("Resume Prepare Marking");
return true;
}
@@ -750,7 +751,7 @@ bool ShenandoahOldHeuristics::should_start_gc() {
const size_t old_gen_capacity = _old_generation->max_capacity();
const size_t heap_capacity = heap->capacity();
const double percent = percent_of(old_gen_capacity, heap_capacity);
- log_trigger("Expansion failure, current size: %zu%s which is %.1f%% of total heap size",
+ log_trigger("Handle Expansion Failure. %zu%s (%.1f%%) old generation",
byte_size_in_proper_unit(old_gen_capacity), proper_unit_for_byte_size(old_gen_capacity), percent);
adjust_old_garbage_threshold();
return true;
@@ -770,9 +771,7 @@ bool ShenandoahOldHeuristics::should_start_gc() {
const size_t span_of_old_regions = (last_old_region >= first_old_region)? last_old_region + 1 - first_old_region: 0;
const size_t fragmented_free = used_regions_size - used;
- log_trigger("Old has become fragmented: "
- "%zu%s available bytes spread between range spanned from "
- "%zu to %zu (%zu), density: %.1f%%",
+ log_trigger("Fragmentation. %zu%s available in old, [%zu, %zu] (%zu) regions, density: %.1f%%",
byte_size_in_proper_unit(fragmented_free), proper_unit_for_byte_size(fragmented_free),
first_old_region, last_old_region, span_of_old_regions, density * 100);
adjust_old_garbage_threshold();
@@ -800,8 +799,7 @@ bool ShenandoahOldHeuristics::should_start_gc() {
} else if (current_usage > trigger_threshold) {
const size_t live_at_previous_old = _old_generation->get_live_bytes_at_last_mark();
const double percent_growth = percent_of(current_usage - live_at_previous_old, live_at_previous_old);
- log_trigger("Old has overgrown, live at end of previous OLD marking: "
- "%zu%s, current usage: %zu%s, percent growth: %.1f%%",
+ log_trigger("Occupancy. %zu%s live at old mark end, %zu%s used, %.1f%% growth",
byte_size_in_proper_unit(live_at_previous_old), proper_unit_for_byte_size(live_at_previous_old),
byte_size_in_proper_unit(current_usage), proper_unit_for_byte_size(current_usage), percent_growth);
adjust_old_garbage_threshold();
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp
index 85c5d9fb2fb2..68e297a8273a 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp
@@ -25,6 +25,7 @@
#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSPACEINFO_HPP
#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSPACEINFO_HPP
+#include "gc/shenandoah/shenandoahAffiliation.hpp"
#include "utilities/globalDefinitions.hpp"
class ShenandoahHeapRegion;
@@ -46,6 +47,7 @@ class ShenandoahSpaceInfo {
// Return true if this region belongs to this space.
virtual bool contains(ShenandoahHeapRegion* region) const = 0;
+ virtual bool contains(ShenandoahAffiliation affiliation) const = 0;
};
#endif //SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSPACEINFO_HPP
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
index 98d679f86d9c..0ceab0460cfd 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,8 +46,8 @@ bool ShenandoahStaticHeuristics::should_start_gc() {
size_t threshold_available = capacity / 100 * ShenandoahMinFreeThreshold;
if (available < threshold_available) {
- log_trigger("Free (Soft) (" PROPERFMT ") is below minimum threshold (" PROPERFMT ")",
- PROPERFMTARGS(available), PROPERFMTARGS(threshold_available));
+ log_trigger("Occupancy. " PROPERFMT " free, below " PROPERFMT " threshold",
+ PROPERFMTARGS(available), PROPERFMTARGS(threshold_available));
accept_trigger();
return true;
}
diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp
index 280076377594..49be36aaf949 100644
--- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp
+++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahYoungHeuristics.cpp
@@ -138,7 +138,7 @@ bool ShenandoahYoungHeuristics::trigger_expedite_promotions(ShenandoahGeneration
if (promo_potential > promo_expedite_threshold) {
// Detect unsigned arithmetic underflow
assert(promo_potential < heap->capacity(), "Sanity");
- log_trigger("Expedite promotion of " PROPERFMT, PROPERFMTARGS(promo_potential));
+ log_trigger("Expedite Promotion. " PROPERFMT " promotion potential", PROPERFMTARGS(promo_potential));
accept_trigger();
return true;
}
@@ -152,7 +152,7 @@ bool ShenandoahYoungHeuristics::trigger_expedite_mixed(ShenandoahGenerationalHea
// If concurrent weak root processing is in progress, it means the old cycle has chosen mixed collection
// candidates, but has not completed. There is no point in trying to start the young cycle before the old
// cycle completes.
- log_trigger("Expedite mixed evacuation of %zu regions", mixed_candidates);
+ log_trigger("Expedite Mixed. %zu region candidates for mixed evacuation", mixed_candidates);
accept_trigger();
return true;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp
index 8fa497802fde..3c23dd0bf233 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.cpp
@@ -1,6 +1,6 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
- * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2025, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,7 +24,7 @@
*/
#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
-#include "gc/shenandoah/shenandoahAgeCensus.hpp"
+#include "gc/shenandoah/shenandoahAgeCensus.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
ShenandoahAgeCensus::ShenandoahAgeCensus()
@@ -82,38 +82,6 @@ ShenandoahAgeCensus::~ShenandoahAgeCensus() {
}
}
-CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id) {)
-NO_CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, size_t size, uint worker_id) {)
- if (obj_age <= markWord::max_age) {
- assert(obj_age < MAX_COHORTS && region_age < MAX_COHORTS, "Should have been tenured");
-#ifdef SHENANDOAH_CENSUS_NOISE
- // Region ageing is stochastic and non-monotonic; this vitiates mortality
- // demographics in ways that might defeat our algorithms. Marking may be a
- // time when we might be able to correct this, but we currently do not do
- // this. Like skipped statistics further below, we want to track the
- // impact of this noise to see if this may be worthwhile. JDK-.
- uint age = obj_age;
- if (region_age > 0) {
- add_aged(size, worker_id); // this tracking is coarse for now
- age += region_age;
- if (age >= MAX_COHORTS) {
- age = (uint)(MAX_COHORTS - 1); // clamp
- add_clamped(size, worker_id);
- }
- }
- if (region_youth > 0) { // track object volume with retrograde age
- add_young(size, worker_id);
- }
-#else // SHENANDOAH_CENSUS_NOISE
- uint age = MIN2(obj_age + region_age, (uint)(MAX_COHORTS - 1)); // clamp
-#endif // SHENANDOAH_CENSUS_NOISE
- get_local_age_table(worker_id)->add(age, size);
- } else {
- // update skipped statistics
- CENSUS_NOISE(add_skipped(size, worker_id);)
- }
-}
-
#ifdef SHENANDOAH_CENSUS_NOISE
void ShenandoahAgeCensus::add_skipped(size_t size, uint worker_id) {
_local_noise[worker_id].skipped += size;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp
index 8cc8e31cf291..2d79eb559609 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.hpp
@@ -205,8 +205,8 @@ class ShenandoahAgeCensus: public CHeapObj {
// Update the local age table for worker_id by size for
// given obj_age, region_age, and region_youth
- CENSUS_NOISE(void add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id);)
- NO_CENSUS_NOISE(void add(uint obj_age, uint region_age, size_t size, uint worker_id);)
+ CENSUS_NOISE(inline void add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id);)
+ NO_CENSUS_NOISE(inline void add(uint obj_age, uint region_age, size_t size, uint worker_id);)
#ifdef SHENANDOAH_CENSUS_NOISE
// Update the local skip table for worker_id by size
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.inline.hpp
new file mode 100644
index 000000000000..299b9a3f8b7a
--- /dev/null
+++ b/src/hotspot/share/gc/shenandoah/shenandoahAgeCensus.inline.hpp
@@ -0,0 +1,62 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHAGECENSUS_INLINE_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHAGECENSUS_INLINE_HPP
+
+#include "gc/shenandoah/shenandoahAgeCensus.hpp"
+
+CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, uint region_youth, size_t size, uint worker_id) {)
+NO_CENSUS_NOISE(void ShenandoahAgeCensus::add(uint obj_age, uint region_age, size_t size, uint worker_id) {)
+ if (obj_age <= markWord::max_age) {
+ assert(obj_age < MAX_COHORTS && region_age < MAX_COHORTS, "Should have been tenured");
+#ifdef SHENANDOAH_CENSUS_NOISE
+ // Region ageing is stochastic and non-monotonic; this vitiates mortality
+ // demographics in ways that might defeat our algorithms. Marking may be a
+ // time when we might be able to correct this, but we currently do not do
+ // this. Like skipped statistics further below, we want to track the
+ // impact of this noise to see if this may be worthwhile. JDK-.
+ uint age = obj_age;
+ if (region_age > 0) {
+ add_aged(size, worker_id); // this tracking is coarse for now
+ age += region_age;
+ if (age >= MAX_COHORTS) {
+ age = (uint)(MAX_COHORTS - 1); // clamp
+ add_clamped(size, worker_id);
+ }
+ }
+ if (region_youth > 0) { // track object volume with retrograde age
+ add_young(size, worker_id);
+ }
+#else // SHENANDOAH_CENSUS_NOISE
+ uint age = MIN2(obj_age + region_age, (uint)(MAX_COHORTS - 1)); // clamp
+#endif // SHENANDOAH_CENSUS_NOISE
+ get_local_age_table(worker_id)->add(age, size);
+ } else {
+ // update skipped statistics
+ CENSUS_NOISE(add_skipped(size, worker_id);)
+ }
+}
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHAGECENSUS_INLINE_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
index 1e839fab6554..107f126b44d5 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
@@ -29,9 +29,14 @@
#include "gc/shenandoah/shenandoahBarrierSetStackChunk.hpp"
#include "gc/shenandoah/shenandoahCardTable.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
+#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
+#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
#include "gc/shenandoah/shenandoahStackWatermark.hpp"
+#include "memory/iterator.inline.hpp"
+#include "oops/compressedOops.inline.hpp"
#ifdef COMPILER1
#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
#endif
@@ -94,8 +99,7 @@ bool ShenandoahBarrierSet::need_satb_barrier(DecoratorSet decorators, BasicType
bool ShenandoahBarrierSet::need_card_barrier(DecoratorSet decorators, BasicType type) {
if (!ShenandoahCardBarrier) return false;
if (!is_reference_type(type)) return false;
- bool in_heap = (decorators & IN_HEAP) != 0;
- return in_heap;
+ return is_heap_access(decorators);
}
void ShenandoahBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
@@ -181,8 +185,51 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
}
}
-void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) {
- assert(ShenandoahCardBarrier, "Should have been checked by caller");
+void ShenandoahBarrierSet::keepalive_barrier_slow(oop obj, Filter filter) {
+ if (!ShenandoahSATBBarrier) {
+ return;
+ }
+ assert(obj != nullptr, "Filtered by caller");
+ assert(_heap->is_concurrent_mark_in_progress(), "Filtered by caller");
+
+ // Filter marked objects before hitting the SATB queues. The same predicate would
+ // be used by SATBMQ::filter to eliminate already marked objects downstream, but
+ // filtering here helps to avoid wasteful SATB queueing work to begin with.
+ if (((filter & FILTER_MARKED) != 0) && !_heap->requires_marking(obj)) {
+ return;
+ }
+
+ shenandoah_assert_correct(nullptr, obj);
+ assert(_satb_mark_queue_set.is_active(), "only get here when SATB active");
+
+ SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(Thread::current());
+ _satb_mark_queue_set.enqueue_known_active(queue, obj);
+}
+
+template
+oop ShenandoahBarrierSet::load_reference_barrier_slow(oop obj, T* load_addr) {
+ if (!ShenandoahLoadRefBarrier) {
+ return obj;
+ }
+ assert(_heap->has_forwarded_objects(), "Filtered by caller");
+ assert(_heap->in_collection_set(obj), "Filtered by caller");
+ oop fwd = ShenandoahForwarding::get_forwardee(obj);
+ if (obj == fwd && _heap->is_evacuation_in_progress()) {
+ Thread* t = Thread::current();
+ fwd = _heap->evacuate_object(obj, t);
+ }
+ if (load_addr != nullptr && fwd != obj) {
+ // Since we are here and we know the load address, update the reference.
+ ShenandoahHeap::atomic_update_oop(fwd, load_addr, obj);
+ }
+ return fwd;
+}
+
+template oop ShenandoahBarrierSet::load_reference_barrier_slow(oop obj, oop* load_addr);
+template oop ShenandoahBarrierSet::load_reference_barrier_slow(oop obj, narrowOop* load_addr);
+
+void ShenandoahBarrierSet::card_barrier_array_slow(HeapWord* start, size_t count) {
+ assert(ShenandoahCardBarrier, "Filtered by caller");
HeapWord* end = (HeapWord*)((char*) start + (count * heapOopSize));
// In the case of compressed oops, start and end may potentially be misaligned;
@@ -199,3 +246,164 @@ void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) {
_heap->old_generation()->card_scan()->mark_range_as_dirty(aligned_start, (aligned_end - aligned_start));
}
+// Clone barrier support
+template
+class ShenandoahUpdateEvacForCloneOopClosure : public BasicOopIterateClosure {
+private:
+ ShenandoahHeap* const _heap;
+ const ShenandoahCollectionSet* const _cset;
+ Thread* const _thread;
+
+ template
+ inline void do_oop_work(T* p) {
+ T o = RawAccess<>::oop_load(p);
+ if (!CompressedOops::is_null(o)) {
+ oop obj = CompressedOops::decode_not_null(o);
+ if (_cset->is_in(obj)) {
+ oop fwd = ShenandoahForwarding::get_forwardee(obj);
+ if (EVAC && obj == fwd) {
+ fwd = _heap->evacuate_object(obj, _thread);
+ }
+ shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc());
+ ShenandoahHeap::atomic_update_oop(fwd, p, o);
+ obj = fwd;
+ }
+ }
+ }
+
+public:
+ ShenandoahUpdateEvacForCloneOopClosure() :
+ _heap(ShenandoahHeap::heap()),
+ _cset(_heap->collection_set()),
+ _thread(Thread::current()) {}
+
+ virtual void do_oop(oop* p) { do_oop_work(p); }
+ virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+};
+
+void ShenandoahBarrierSet::clone_evacuation(oop obj) {
+ if (!ShenandoahCloneBarrier) {
+ return;
+ }
+ if (!need_bulk_update(cast_from_oop(obj))) {
+ return;
+ }
+
+ ShenandoahUpdateEvacForCloneOopClosure cl;
+ obj->oop_iterate(&cl);
+}
+
+void ShenandoahBarrierSet::clone_update(oop obj) {
+ if (!ShenandoahCloneBarrier) {
+ return;
+ }
+ if (!need_bulk_update(cast_from_oop(obj))) {
+ return;
+ }
+
+ ShenandoahUpdateEvacForCloneOopClosure cl;
+ obj->oop_iterate(&cl);
+}
+
+template
+bool ShenandoahBarrierSet::is_above_tams(const ShenandoahMarkingContext* ctx, T* dst) const {
+ // TAMS for an old region is unreliable during a young-only mark, so overwritten pointers in old dst arrays must
+ // be enqueued to preserve old->young referents copied in and overwritten after init mark. See JDK-8373116.
+ return ctx->allocated_after_mark_start(reinterpret_cast(dst))
+ && !(IS_GENERATIONAL
+ && _heap->heap_region_containing(dst)->is_old()
+ && _heap->is_concurrent_young_mark_in_progress());
+}
+
+inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) const {
+ return ary < _heap->heap_region_containing(ary)->get_update_watermark();
+}
+
+template
+void ShenandoahBarrierSet::arraycopy_marking(T* dst, size_t count) {
+ assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
+ if (!ShenandoahSATBBarrier) {
+ return;
+ }
+
+ const ShenandoahMarkingContext* ctx = _heap->marking_context();
+ // Everything allocated above TAMS is alive and doesn't need the barrier to keep it that way
+ if (is_above_tams(ctx, dst)) {
+ return;
+ }
+
+ assert(!_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded object status is sane");
+ Thread* thread = Thread::current();
+ SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread);
+ T* end = dst + count;
+ for (T* elem_ptr = dst; elem_ptr < end; ++elem_ptr) {
+ T o = RawAccess<>::oop_load(elem_ptr);
+ if (!CompressedOops::is_null(o)) {
+ oop obj = CompressedOops::decode_not_null(o);
+ if (!ctx->is_marked_strong(obj)) {
+ _satb_mark_queue_set.enqueue_known_active(queue, obj);
+ }
+ }
+ }
+}
+
+template void ShenandoahBarrierSet::arraycopy_marking(oop* dst, size_t count);
+template void ShenandoahBarrierSet::arraycopy_marking(narrowOop* dst, size_t count);
+template void ShenandoahBarrierSet::arraycopy_marking(oop* dst, size_t count);
+template void ShenandoahBarrierSet::arraycopy_marking(narrowOop* dst, size_t count);
+
+template
+void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) {
+ assert(_heap->is_evacuation_in_progress(), "only during evacuation");
+ if (!need_bulk_update(reinterpret_cast(src))) {
+ return;
+ }
+
+ assert(_heap->has_forwarded_objects(), "Forwarded object status is sane");
+ Thread* thread = Thread::current();
+ const ShenandoahCollectionSet* const cset = _heap->collection_set();
+ T* end = src + count;
+ for (T* elem_ptr = src; elem_ptr < end; ++elem_ptr) {
+ T o = RawAccess<>::oop_load(elem_ptr);
+ if (!CompressedOops::is_null(o)) {
+ oop obj = CompressedOops::decode_not_null(o);
+ if (cset->is_in(obj)) {
+ oop fwd = ShenandoahForwarding::get_forwardee(obj);
+ if (obj == fwd) {
+ fwd = _heap->evacuate_object(obj, thread);
+ }
+ shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc());
+ ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o);
+ }
+ }
+ }
+}
+
+template void ShenandoahBarrierSet::arraycopy_evacuation(oop* src, size_t count);
+template void ShenandoahBarrierSet::arraycopy_evacuation(narrowOop* src, size_t count);
+
+template
+void ShenandoahBarrierSet::arraycopy_update(T* src, size_t count) {
+ assert(_heap->is_update_refs_in_progress(), "only during update-refs");
+ if (!need_bulk_update(reinterpret_cast(src))) {
+ return;
+ }
+
+ assert(_heap->has_forwarded_objects(), "Forwarded object status is sane");
+ const ShenandoahCollectionSet* const cset = _heap->collection_set();
+ T* end = src + count;
+ for (T* elem_ptr = src; elem_ptr < end; ++elem_ptr) {
+ T o = RawAccess<>::oop_load(elem_ptr);
+ if (!CompressedOops::is_null(o)) {
+ oop obj = CompressedOops::decode_not_null(o);
+ if (cset->is_in(obj)) {
+ oop fwd = ShenandoahForwarding::get_forwardee(obj);
+ shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc());
+ ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o);
+ }
+ }
+ }
+}
+
+template void ShenandoahBarrierSet::arraycopy_update(oop* src, size_t count);
+template void ShenandoahBarrierSet::arraycopy_update(narrowOop* src, size_t count);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
index 4ae1f03a08da..83f0d42e2781 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
@@ -80,10 +80,11 @@ class ShenandoahBarrierSet: public BarrierSet {
return (decorators & IN_NATIVE) != 0;
}
- void print_on(outputStream* st) const override;
+ static bool is_heap_access(DecoratorSet decorators) {
+ return (decorators & IN_HEAP) != 0;
+ }
- template
- inline void arraycopy_barrier(T* src, T* dst, size_t count, bool dest_uninit);
+ void print_on(outputStream* st) const override;
// Support for optimizing compilers to call the barrier set on slow path allocations
// that did not enter a TLAB. Used for e.g. ReduceInitialCardMarks to take any
@@ -94,49 +95,68 @@ class ShenandoahBarrierSet: public BarrierSet {
void on_thread_attach(Thread* thread) override;
void on_thread_detach(Thread* thread) override;
- template
- inline void satb_barrier(T* field);
- inline void satb_enqueue(oop value);
-
- inline void keep_alive_if_weak(DecoratorSet decorators, oop value);
+ enum Filter {
+ FILTER_NONE = 0,
+ FILTER_WEAK = (1 << 0),
+ FILTER_MARKED = (1 << 1),
+ FILTER_WEAK_AND_MARKED = FILTER_WEAK | FILTER_MARKED,
+ };
- inline void enqueue(oop obj, bool filter = true);
+ template
+ inline oop oop_load_post(DecoratorSet decorators, oop value, T* addr);
- inline oop load_reference_barrier(oop obj);
+ template
+ inline void oop_store_pre(DecoratorSet decorators, T* addr, oop new_value);
- template
- inline oop load_reference_barrier_mutator(oop obj, T* load_addr);
+ template
+ inline void oop_cmpxchg_pre(DecoratorSet decorators, T* addr, oop compare_value, oop new_value);
- template
- inline oop load_reference_barrier(DecoratorSet decorators, oop obj, T* load_addr);
+ template
+ inline void oop_xchg_pre(DecoratorSet decorators, T* addr, oop new_value);
template
- inline oop oop_cmpxchg(DecoratorSet decorators, T* addr, oop compare_value, oop new_value);
+ inline void oop_store_post(DecoratorSet decorators, T* addr, oop new_value);
template
- inline oop oop_xchg(DecoratorSet decorators, T* addr, oop new_value);
+ inline void keepalive_barrier(DecoratorSet decorators, T* addr, oop obj, Filter filter);
- template
- void write_ref_field_post(T* field, oop new_value);
+ template
+ inline oop load_reference_barrier(DecoratorSet decorators, oop obj, T* load_addr);
- void write_ref_array(HeapWord* start, size_t count);
+ template
+ inline void arraycopy_barrier(T* src, T* dst, size_t count, bool dest_uninit);
private:
- template
- void arraycopy_marking(T* dst, size_t count);
+ void keepalive_barrier_slow(oop obj, Filter filter);
- template
+ template
+ oop load_reference_barrier_slow(oop obj, T* load_addr);
+
+ template
bool is_above_tams(const ShenandoahMarkingContext* ctx, T* dst) const;
- template
- inline void arraycopy_evacuation(T* src, size_t count);
- template
- inline void arraycopy_update(T* src, size_t count);
+ template
+ void arraycopy_marking(T* dst, size_t count);
+
+ template
+ void arraycopy_evacuation(T* src, size_t count);
+
+ template
+ void arraycopy_update(T* src, size_t count);
+
+ void clone_evacuation(oop src);
- template
- inline void clone_work(oop src);
+ void clone_update(oop src);
+
+ template
+ inline void card_barrier(T* field, oop new_value);
+
+ inline void card_barrier_array(HeapWord* start, size_t count);
+
+ void card_barrier_array_slow(HeapWord* start, size_t count);
+
+ bool need_bulk_update(HeapWord* dst) const;
- inline bool need_bulk_update(HeapWord* dst) const;
public:
// Callbacks for runtime accesses.
template
@@ -144,11 +164,8 @@ class ShenandoahBarrierSet: public BarrierSet {
typedef BarrierSet::AccessBarrier Raw;
private:
- template
- static oop oop_load_common(DecoratorSet resolved_decorators, T* addr);
-
- template
- static void oop_store_common(T* addr, oop value);
+ static DecoratorSet resolve_unknown(oop base, ptrdiff_t offset);
+ static DecoratorSet resolve_unknown_to_strong(oop base, ptrdiff_t offset);
public:
// Heap oop accesses. These accessors get resolved when
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
index b2f5fbad5cf0..a27516f97378 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
@@ -34,7 +34,6 @@
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahCardTable.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
-#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahGeneration.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
@@ -43,86 +42,14 @@
#include "memory/iterator.inline.hpp"
#include "oops/oop.inline.hpp"
-template
-inline oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj, T* load_addr) {
- assert(ShenandoahLoadRefBarrier, "Should be enabled");
-
- constexpr bool on_weak = HasDecorator::value;
- constexpr bool on_phantom = HasDecorator::value;
-
- // Handle nulls. Strong loads filtered nulls with cset checks.
- // Weak/phantom loads need to check for nulls here.
- if (on_weak || on_phantom) {
- if (obj == nullptr) {
- return nullptr;
- }
- } else {
- assert(obj != nullptr, "Should have been filtered before");
- }
-
- // Prevent resurrection of unreachable phantom (i.e. weak-native) references.
- if (on_phantom &&
- _heap->is_concurrent_weak_root_in_progress() &&
- _heap->is_in_active_generation(obj) &&
- !_heap->marking_context()->is_marked(obj)) {
- return nullptr;
- }
-
- // Prevent resurrection of unreachable weak references.
- if (on_weak &&
- _heap->is_concurrent_weak_root_in_progress() &&
- _heap->is_in_active_generation(obj) &&
- !_heap->marking_context()->is_marked_strong(obj)) {
- return nullptr;
- }
-
- // Weak/phantom loads need additional cset check.
- if (on_phantom || on_weak) {
- if (!_heap->has_forwarded_objects() || !_heap->in_collection_set(obj)) {
- return obj;
- }
- } else {
- shenandoah_assert_in_cset(load_addr, obj);
- }
-
- oop fwd = ShenandoahForwarding::get_forwardee_mutator(obj);
- if (obj == fwd) {
- assert(_heap->is_evacuation_in_progress(), "evac should be in progress");
- Thread* const t = Thread::current();
- fwd = _heap->evacuate_object(obj, t);
- }
-
- if (load_addr != nullptr && fwd != obj) {
- // Since we are here and we know the load address, update the reference.
- ShenandoahHeap::atomic_update_oop(fwd, load_addr, obj);
- }
-
- return fwd;
-}
-
-inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj) {
- if (!ShenandoahLoadRefBarrier) {
- return obj;
- }
- if (_heap->has_forwarded_objects() && _heap->in_collection_set(obj)) {
- // Subsumes null-check
- assert(obj != nullptr, "cset check must have subsumed null-check");
- oop fwd = ShenandoahForwarding::get_forwardee(obj);
- if (obj == fwd && _heap->is_evacuation_in_progress()) {
- Thread* t = Thread::current();
- return _heap->evacuate_object(obj, t);
- }
- return fwd;
- }
- return obj;
-}
-
-template
+template
inline oop ShenandoahBarrierSet::load_reference_barrier(DecoratorSet decorators, oop obj, T* load_addr) {
if (obj == nullptr) {
return nullptr;
}
+ assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
+
// Prevent resurrection of unreachable phantom (i.e. weak-native) references.
if ((decorators & ON_PHANTOM_OOP_REF) != 0 &&
_heap->is_concurrent_weak_root_in_progress() &&
@@ -146,70 +73,54 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(DecoratorSet decorators,
return obj;
}
- oop fwd = load_reference_barrier(obj);
- if (load_addr != nullptr && fwd != obj) {
- // Since we are here and we know the load address, update the reference.
- ShenandoahHeap::atomic_update_oop(fwd, load_addr, obj);
+ // No need for the barrier if object is not forwarded.
+ if (!_heap->has_forwarded_objects() || !_heap->in_collection_set(obj)) {
+ return obj;
}
- return fwd;
+ return load_reference_barrier_slow(obj, load_addr);
}
-inline void ShenandoahBarrierSet::enqueue(oop obj, bool filter) {
- assert(obj != nullptr, "checked by caller");
- shenandoah_assert_correct(nullptr, obj);
- assert(_satb_mark_queue_set.is_active(), "only get here when SATB active");
-
- // Filter marked objects before hitting the SATB queues. The same predicate would
- // be used by SATBMQ::filter to eliminate already marked objects downstream, but
- // filtering here helps to avoid wasteful SATB queueing work to begin with.
- if (filter && !_heap->requires_marking(obj)) return;
+template
+inline void ShenandoahBarrierSet::keepalive_barrier(DecoratorSet decorators, T* addr, oop obj, Filter filter) {
+ // Uninitialized and no-keepalive loads/stores do not need barrier.
+ if (((decorators & IS_DEST_UNINITIALIZED) != 0) ||
+ ((decorators & AS_NO_KEEPALIVE) != 0)) {
+ return;
+ }
- SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(Thread::current());
- _satb_mark_queue_set.enqueue_known_active(queue, obj);
-}
+ assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
-template
-inline void ShenandoahBarrierSet::satb_barrier(T *field) {
- // Uninitialized and no-keepalive stores do not need barrier.
- if (HasDecorator::value ||
- HasDecorator::value) {
+ // No need for barriers on weaks, if requested. Normally filtered for stores, accepted for loads.
+ if (((filter & FILTER_WEAK) != 0) &&
+ (((decorators & ON_WEAK_OOP_REF) != 0) ||
+ ((decorators & ON_PHANTOM_OOP_REF) != 0))) {
return;
}
- // Stores to weak/phantom require no barrier. The original references would
- // have been enqueued in the SATB buffer by the load barrier if they were needed.
- if (HasDecorator::value ||
- HasDecorator::value) {
+ // No need for the barrier if marking is not in progress.
+ if (!_heap->is_concurrent_mark_in_progress()) {
return;
}
- if (ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) {
- T heap_oop = RawAccess<>::oop_load(field);
- if (!CompressedOops::is_null(heap_oop)) {
- enqueue(CompressedOops::decode_not_null(heap_oop));
- }
+ if (addr != nullptr) {
+ assert(obj == nullptr, "Ambiguity: use addr or obj?");
+ obj = RawAccess<>::oop_load(addr);
}
-}
-inline void ShenandoahBarrierSet::satb_enqueue(oop value) {
- if (value != nullptr && ShenandoahSATBBarrier && _heap->is_concurrent_mark_in_progress()) {
- enqueue(value);
+ // Null objects require no barriers.
+ if (obj == nullptr) {
+ return;
}
-}
-inline void ShenandoahBarrierSet::keep_alive_if_weak(DecoratorSet decorators, oop value) {
- assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
- const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0;
- const bool peek = (decorators & AS_NO_KEEPALIVE) != 0;
- if (!peek && !on_strong_oop_ref) {
- satb_enqueue(value);
- }
+ keepalive_barrier_slow(obj, filter);
}
-template
-inline void ShenandoahBarrierSet::write_ref_field_post(T* field, oop new_value) {
- assert(ShenandoahCardBarrier, "Should have been checked by caller");
+template
+inline void ShenandoahBarrierSet::card_barrier(T* field, oop new_value) {
+ if (!ShenandoahCardBarrier) {
+ return;
+ }
if (new_value == nullptr) {
// Null reference stores do not require card mark.
@@ -233,241 +144,232 @@ inline void ShenandoahBarrierSet::write_ref_field_post(T* field, oop new_value)
*byte = CardTable::dirty_card_val();
}
+inline void ShenandoahBarrierSet::card_barrier_array(HeapWord* start, size_t count) {
+ if (!ShenandoahCardBarrier) {
+ return;
+ }
+ card_barrier_array_slow(start, count);
+}
+
+template
+inline oop ShenandoahBarrierSet::oop_load_post(DecoratorSet decorators, oop value, T* addr) {
+ assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
+
+ shenandoah_assert_not_in_cset_loc_except(addr, !is_heap_access(decorators) || _heap->cancelled_gc());
+
+ // Perform LRB to handle evacuation and possibly weak loads.
+ value = load_reference_barrier(decorators, value, addr);
+
+ // If weak load survived the LRB, we need to keep-alive the value.
+ if (!is_strong_access(decorators)) {
+ keepalive_barrier(decorators, (T*)nullptr, value, FILTER_MARKED);
+ }
+
+ return value;
+}
+
+template
+inline void ShenandoahBarrierSet::oop_store_pre(DecoratorSet decorators, T* addr, oop new_value) {
+ assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
+
+ shenandoah_assert_not_in_cset_loc_except(addr, !is_heap_access(decorators) || _heap->cancelled_gc());
+ shenandoah_assert_not_in_cset_except(nullptr, new_value, new_value == nullptr || _heap->cancelled_gc());
+ shenandoah_assert_not_forwarded_except(nullptr, new_value, new_value == nullptr || _heap->cancelled_gc());
+
+ shenandoah_assert_marked_if(nullptr, new_value,
+ !CompressedOops::is_null(new_value) &&
+ _heap->is_evacuation_in_progress() &&
+ !(_heap->active_generation()->is_young() && _heap->heap_region_containing(new_value)->is_old()));
+
+ // Handle the previous value through SATB, as we are about to perform the store.
+ keepalive_barrier(decorators, addr, nullptr, FILTER_WEAK_AND_MARKED);
+}
+
template
-inline oop ShenandoahBarrierSet::oop_cmpxchg(DecoratorSet decorators, T* addr, oop compare_value, oop new_value) {
- shenandoah_assert_not_in_cset_except(nullptr, compare_value, (compare_value == nullptr || ShenandoahHeap::heap()->cancelled_gc()));
- shenandoah_assert_not_in_cset_except(nullptr, new_value, (new_value == nullptr || ShenandoahHeap::heap()->cancelled_gc()));
+inline void ShenandoahBarrierSet::oop_store_post(DecoratorSet decorators, T* addr, oop new_value) {
+ // Handle card table updates if needed.
+ if (is_heap_access(decorators)) {
+ card_barrier(addr, new_value);
+ }
+}
+
+template
+inline void ShenandoahBarrierSet::oop_cmpxchg_pre(DecoratorSet decorators, T* addr, oop compare_value, oop new_value) {
+ assert((decorators & AS_NO_KEEPALIVE) == 0, "CAS only with keep-alive");
+ assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "CAS should have resolved ref strength");
+ assert((decorators & ON_STRONG_OOP_REF) != 0, "CAS only for strong refs");
+
+ shenandoah_assert_not_in_cset_loc_except(addr, !is_heap_access(decorators) || _heap->cancelled_gc());
+ shenandoah_assert_not_in_cset_except(nullptr, compare_value, compare_value == nullptr || _heap->cancelled_gc());
+ shenandoah_assert_not_in_cset_except(nullptr, new_value, new_value == nullptr || _heap->cancelled_gc());
+ shenandoah_assert_not_forwarded_except(addr, compare_value, compare_value == nullptr || _heap->cancelled_gc());
+ shenandoah_assert_not_forwarded_except(addr, new_value, new_value == nullptr || _heap->cancelled_gc());
// Handle the previous value through SATB, as we are about to perform the store.
oop prev = RawAccess<>::oop_load(addr);
- satb_enqueue(prev);
+ keepalive_barrier(decorators, (T*)nullptr, prev, FILTER_MARKED);
// Perform LRB on location to fix it up for this and all following accesses.
// This guarantees there are no false negatives due to concurrent evacuation,
// and the value loaded later by CAS is sanitized by some LRB, or is null.
load_reference_barrier(decorators, prev, addr);
-
- return RawAccess<>::oop_atomic_cmpxchg(addr, compare_value, new_value);
}
template
-inline oop ShenandoahBarrierSet::oop_xchg(DecoratorSet decorators, T* addr, oop new_value) {
- shenandoah_assert_not_in_cset_except(nullptr, new_value, (new_value == nullptr || ShenandoahHeap::heap()->cancelled_gc()));
+inline void ShenandoahBarrierSet::oop_xchg_pre(DecoratorSet decorators, T* addr, oop new_value) {
+ assert((decorators & AS_NO_KEEPALIVE) == 0, "XCHG only with keep-alive");
+ assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "XCHG should have resolved ref strength");
+ assert((decorators & ON_STRONG_OOP_REF) != 0, "XCHG only for strong refs");
+
+ shenandoah_assert_not_in_cset_loc_except(addr, !is_heap_access(decorators) || _heap->cancelled_gc());
+ shenandoah_assert_not_in_cset_except(nullptr, new_value, new_value == nullptr || _heap->cancelled_gc());
+ shenandoah_assert_not_forwarded_except(addr, new_value, new_value == nullptr || _heap->cancelled_gc());
// Handle the previous value through SATB, as we are about to perform the store.
oop prev = RawAccess<>::oop_load(addr);
- satb_enqueue(prev);
+ keepalive_barrier(decorators, (T*)nullptr, prev, FILTER_MARKED);
// Perform LRB on location to fix it up for this and all following accesses.
// This is purely opportunistic: we would not have any false negatives here.
// This guarantees the value loaded later by XCHG is sanitized by some LRB, or is null.
load_reference_barrier(decorators, prev, addr);
+}
- return RawAccess<>::oop_atomic_xchg(addr, new_value);
+template
+inline DecoratorSet ShenandoahBarrierSet::AccessBarrier::resolve_unknown(oop base, ptrdiff_t offset) {
+ return AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
}
template
-template
-inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_common(DecoratorSet resolved_decorators, T* addr) {
- // This raw access inherits decorators that are needed for proper memory ordering.
- oop value = Raw::template oop_load(addr);
- ShenandoahBarrierSet* bs = barrier_set();
- value = bs->load_reference_barrier(resolved_decorators, value, addr);
- bs->keep_alive_if_weak(resolved_decorators, value);
- return value;
+inline DecoratorSet ShenandoahBarrierSet::AccessBarrier::resolve_unknown_to_strong(oop base, ptrdiff_t offset) {
+ // Unsafe operations come to this barrier set with ON_UNKNOWN_OOP_REF set.
+ // These are normally strong refs, but one can use Unsafe on Reference.referent.
+ // We cannot deal with that case. If application does Unsafe operations on
+ // Reference.referent field, this likely breaks weak reference semantics already.
+ // We upgrade the access to strong in (sometimes futile) attempt to maintain heap
+ // integrity, and assert in debug builds for better diagnostics.
+ assert((decorators & (ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0, "Only strong or unknown expected here");
+ DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
+ assert((resolved_decorators & ON_STRONG_OOP_REF) != 0, "Application error: Unsupported operation on weak location");
+ return (resolved_decorators & ~ON_DECORATOR_MASK) | ON_STRONG_OOP_REF;
}
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_not_in_heap(T* addr) {
- assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "must be absent");
- return oop_load_common(decorators, addr);
+ oop value = Raw::oop_load_not_in_heap(addr);
+ return barrier_set()->oop_load_post(decorators, value, addr);
}
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_in_heap(T* addr) {
- assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "must be absent");
- return oop_load_common(decorators, addr);
+ oop value = Raw::oop_load_in_heap(addr);
+ return barrier_set()->oop_load_post(decorators, value, addr);
}
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
- DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
- return oop_load_common(resolved_decorators, AccessInternal::oop_field_addr(base, offset));
-}
-
-template
-template
-inline void ShenandoahBarrierSet::AccessBarrier::oop_store_common(T* addr, oop value) {
- shenandoah_assert_marked_if(nullptr, value,
- !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()
- && !(ShenandoahHeap::heap()->active_generation()->is_young()
- && ShenandoahHeap::heap()->heap_region_containing(value)->is_old()));
- shenandoah_assert_not_in_cset_if(addr, value, value != nullptr && !ShenandoahHeap::heap()->cancelled_gc());
- ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set();
- bs->satb_barrier(addr);
- Raw::oop_store(addr, value);
+ DecoratorSet resolved_decorators = resolve_unknown(base, offset);
+ auto addr = AccessInternal::oop_field_addr(base, offset);
+ oop value = Raw::oop_load_in_heap(addr);
+ return barrier_set()->oop_load_post(resolved_decorators, value, addr);
}
template
template
inline void ShenandoahBarrierSet::AccessBarrier::oop_store_not_in_heap(T* addr, oop value) {
- assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Reference strength must be known");
- oop_store_common(addr, value);
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_store_pre(decorators, addr, value);
+ Raw::oop_store_not_in_heap(addr, value);
+ bs->oop_store_post(decorators, addr, value);
}
template
template
inline void ShenandoahBarrierSet::AccessBarrier::oop_store_in_heap(T* addr, oop value) {
- shenandoah_assert_not_in_cset_loc_except(addr, ShenandoahHeap::heap()->cancelled_gc());
- shenandoah_assert_not_forwarded_except (addr, value, value == nullptr || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress());
-
- oop_store_common(addr, value);
- if (ShenandoahCardBarrier) {
- ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
- bs->write_ref_field_post(addr, value);
- }
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_store_pre(decorators, addr, value);
+ Raw::oop_store_in_heap(addr, value);
+ bs->oop_store_post(decorators, addr, value);
}
template
inline void ShenandoahBarrierSet::AccessBarrier::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
- oop_store_in_heap(AccessInternal::oop_field_addr(base, offset), value);
+ auto addr = AccessInternal::oop_field_addr(base, offset);
+
+ // In contrast to CASes, we resolve unknown to weak/phantom access, because some code
+ // legitimately enters here, e.g. for clearing Reference.referent.
+ DecoratorSet resolved_decorators = resolve_unknown(base, offset);
+
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_store_pre(resolved_decorators, addr, value);
+ Raw::oop_store_in_heap(addr, value);
+ bs->oop_store_post(resolved_decorators, addr, value);
}
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value) {
- assert((decorators & AS_NO_KEEPALIVE) == 0, "CAS only with keep-alive");
- assert((decorators & ON_STRONG_OOP_REF) != 0, "CAS only for strong refs");
- ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
- return bs->oop_cmpxchg(decorators, addr, compare_value, new_value);
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_cmpxchg_pre(decorators, addr, compare_value, new_value);
+ oop result = Raw::oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value);
+ bs->oop_store_post(decorators, addr, new_value);
+ return result;
}
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) {
- assert((decorators & AS_NO_KEEPALIVE) == 0, "CAS only with keep-alive");
- assert((decorators & ON_STRONG_OOP_REF) != 0, "CAS only for strong refs");
- ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
- oop result = bs->oop_cmpxchg(decorators, addr, compare_value, new_value);
- if (ShenandoahCardBarrier) {
- bs->write_ref_field_post(addr, new_value);
- }
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_cmpxchg_pre(decorators, addr, compare_value, new_value);
+ oop result = Raw::oop_atomic_cmpxchg_in_heap(addr, compare_value, new_value);
+ bs->oop_store_post(decorators, addr, new_value);
return result;
}
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
- assert((decorators & AS_NO_KEEPALIVE) == 0, "CAS only with keep-alive");
- assert((decorators & (ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0, "CAS only for strong refs OR unknown refs (Unsafe)");
- ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
-
- // Unsafe.compareAndExchange/Set come here with ON_UNKNOWN_OOP_REF set.
- // These are normally strong refs, but one can use Unsafe on Reference.referent.
- // We cannot deal with that case. If application does Unsafe operations on
- // Reference.referent field, this likely breaks weak reference semantics already.
- // We upgrade the access to strong in (sometimes futile) attempt to maintain heap
- // integrity, and assert in debug builds for better diagnostics.
- DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
- assert((resolved_decorators & ON_STRONG_OOP_REF) != 0, "Application error: CAS on weak location");
- resolved_decorators = (resolved_decorators & ~ON_DECORATOR_MASK) | ON_STRONG_OOP_REF;
-
auto addr = AccessInternal::oop_field_addr(base, offset);
- oop result = bs->oop_cmpxchg(resolved_decorators, addr, compare_value, new_value);
- if (ShenandoahCardBarrier) {
- bs->write_ref_field_post(addr, new_value);
- }
+ DecoratorSet resolved_decorators = resolve_unknown_to_strong(base, offset);
+
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_cmpxchg_pre(resolved_decorators, addr, compare_value, new_value);
+ oop result = Raw::oop_atomic_cmpxchg_in_heap(addr, compare_value, new_value);
+ bs->oop_store_post(resolved_decorators, addr, new_value);
return result;
}
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_not_in_heap(T* addr, oop new_value) {
- assert((decorators & AS_NO_KEEPALIVE) == 0, "XCHG only with keep-alive");
- assert((decorators & ON_STRONG_OOP_REF) != 0, "XCHG only for strong refs");
- ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
- return bs->oop_xchg(decorators, addr, new_value);
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_xchg_pre(decorators, addr, new_value);
+ oop result = Raw::oop_atomic_xchg_not_in_heap(addr, new_value);
+ bs->oop_store_post(decorators, addr, new_value);
+ return result;
}
template
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap(T* addr, oop new_value) {
- assert((decorators & AS_NO_KEEPALIVE) == 0, "XCHG only with keep-alive");
- assert((decorators & ON_STRONG_OOP_REF) != 0, "XCHG only for strong refs");
- ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
- oop result = bs->oop_xchg(decorators, addr, new_value);
- if (ShenandoahCardBarrier) {
- bs->write_ref_field_post(addr, new_value);
- }
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_xchg_pre(decorators, addr, new_value);
+ oop result = Raw::oop_atomic_xchg_in_heap(addr, new_value);
+ bs->oop_store_post(decorators, addr, new_value);
return result;
}
template
inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
- assert((decorators & AS_NO_KEEPALIVE) == 0, "XCHG only with keep-alive");
- assert((decorators & (ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0, "XCHG only for strong refs OR unknown refs (Unsafe)");
- ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
-
- // Unsafe.getAndSet comes here with ON_UNKNOWN_OOP_REF set.
- // These are normally strong refs, but one can use Unsafe on Reference.referent.
- // We cannot deal with that case. If application does Unsafe operations on
- // Reference.referent field, this likely breaks weak reference semantics already.
- // We upgrade the access to strong in (sometimes futile) attempt to maintain heap
- // integrity, and assert in debug builds for better diagnostics.
- DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset);
- assert((resolved_decorators & ON_STRONG_OOP_REF) != 0, "Application error: XCHG on weak location");
- resolved_decorators = (resolved_decorators & ~ON_DECORATOR_MASK) | ON_STRONG_OOP_REF;
-
auto addr = AccessInternal::oop_field_addr(base, offset);
- oop result = bs->oop_xchg(resolved_decorators, addr, new_value);
- if (ShenandoahCardBarrier) {
- bs->write_ref_field_post(addr, new_value);
- }
- return result;
-}
+ DecoratorSet resolved_decorators = resolve_unknown_to_strong(base, offset);
-// Clone barrier support
-template
-class ShenandoahUpdateEvacForCloneOopClosure : public BasicOopIterateClosure {
-private:
- ShenandoahHeap* const _heap;
- const ShenandoahCollectionSet* const _cset;
- Thread* const _thread;
-
- template
- inline void do_oop_work(T* p) {
- T o = RawAccess<>::oop_load(p);
- if (!CompressedOops::is_null(o)) {
- oop obj = CompressedOops::decode_not_null(o);
- if (_cset->is_in(obj)) {
- oop fwd = ShenandoahForwarding::get_forwardee(obj);
- if (EVAC && obj == fwd) {
- fwd = _heap->evacuate_object(obj, _thread);
- }
- shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc());
- ShenandoahHeap::atomic_update_oop(fwd, p, o);
- obj = fwd;
- }
- }
- }
-
-public:
- ShenandoahUpdateEvacForCloneOopClosure() :
- _heap(ShenandoahHeap::heap()),
- _cset(_heap->collection_set()),
- _thread(Thread::current()) {}
-
- virtual void do_oop(oop* p) { do_oop_work(p); }
- virtual void do_oop(narrowOop* p) { do_oop_work(p); }
-};
-
-template
-void ShenandoahBarrierSet::clone_work(oop obj) {
- if (need_bulk_update(cast_from_oop(obj))) {
- ShenandoahUpdateEvacForCloneOopClosure cl;
- obj->oop_iterate(&cl);
- }
+ ShenandoahBarrierSet* bs = barrier_set();
+ bs->oop_xchg_pre(resolved_decorators, addr, new_value);
+ oop result = Raw::oop_atomic_xchg_in_heap(addr, new_value);
+ bs->oop_store_post(resolved_decorators, addr, new_value);
+ return result;
}
template
@@ -476,12 +378,12 @@ void ShenandoahBarrierSet::AccessBarrier::clone_in_heap
// Fix up src before doing the copy, if needed.
const char gc_state = ShenandoahThreadLocalData::gc_state(Thread::current());
- if (gc_state != 0 && ShenandoahCloneBarrier) {
- ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+ if (gc_state != 0) {
+ ShenandoahBarrierSet* bs = barrier_set();
if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
- bs->clone_work(src);
+ bs->clone_evacuation(src);
} else if ((gc_state & ShenandoahHeap::UPDATE_REFS) != 0) {
- bs->clone_work(src);
+ bs->clone_update(src);
}
}
@@ -501,7 +403,7 @@ void ShenandoahBarrierSet::AccessBarrier