From 997b4858a0a4c4e302c45cea252a05313ee5858a Mon Sep 17 00:00:00 2001 From: Igor Veresov Date: Wed, 20 Aug 2025 11:13:51 -0700 Subject: [PATCH 1/7] Fix TD verification --- .../share/compiler/compilationPolicy.cpp | 8 +++ .../share/compiler/compilationPolicy.hpp | 21 ++++++-- src/hotspot/share/oops/trainingData.cpp | 53 ++++++++++++------- src/hotspot/share/oops/trainingData.hpp | 6 +-- src/hotspot/share/runtime/init.cpp | 5 +- src/hotspot/share/runtime/java.cpp | 7 +++ 6 files changed, 68 insertions(+), 32 deletions(-) diff --git a/src/hotspot/share/compiler/compilationPolicy.cpp b/src/hotspot/share/compiler/compilationPolicy.cpp index 9c49d941bbca0..aab1a5d64a089 100644 --- a/src/hotspot/share/compiler/compilationPolicy.cpp +++ b/src/hotspot/share/compiler/compilationPolicy.cpp @@ -138,6 +138,13 @@ void CompilationPolicy::compile_if_required(const methodHandle& m, TRAPS) { } } + void CompilationPolicy::flush_replay_training_at_init(TRAPS) { + MonitorLocker locker(THREAD, TrainingReplayQueue_lock); + while (!_training_replay_queue.is_empty_unlocked() || _training_replay_queue.is_processing_unlocked()) { + locker.wait(); // let the replay training thread drain the queue + } + } + void CompilationPolicy::replay_training_at_init_impl(InstanceKlass* klass, TRAPS) { if (!klass->has_init_deps_processed()) { ResourceMark rm; @@ -186,6 +193,7 @@ void CompilationPolicy::replay_training_at_init_loop(TRAPS) { InstanceKlass* ik = _training_replay_queue.pop(TrainingReplayQueue_lock, THREAD); if (ik != nullptr) { replay_training_at_init_impl(ik, THREAD); + _training_replay_queue.processing_done(TrainingReplayQueue_lock, THREAD); } } } diff --git a/src/hotspot/share/compiler/compilationPolicy.hpp b/src/hotspot/share/compiler/compilationPolicy.hpp index f4a7c4c249b41..cb49ccee4f647 100644 --- a/src/hotspot/share/compiler/compilationPolicy.hpp +++ b/src/hotspot/share/compiler/compilationPolicy.hpp @@ -46,6 +46,7 @@ class Queue { QueueNode* _head; QueueNode* _tail; + int _processing; void push_unlocked(T* value) { QueueNode* n = new QueueNode(value, nullptr); @@ -68,12 +69,17 @@ class Queue { T* value = nullptr; if (n != nullptr) { value = n->value(); + ++_processing; delete n; } return value; } + void processing_done_unlocked() { + precond(_processing > 0); + --_processing; + } public: - Queue() : _head(nullptr), _tail(nullptr) { } + Queue() : _head(nullptr), _tail(nullptr), _processing(0) { } void push(T* value, Monitor* lock, TRAPS) { MonitorLocker locker(THREAD, lock); push_unlocked(value); @@ -81,6 +87,7 @@ class Queue { } bool is_empty_unlocked() const { return _head == nullptr; } + bool is_processing_unlocked() const { return _processing > 0; } T* pop(Monitor* lock, TRAPS) { MonitorLocker locker(THREAD, lock); @@ -93,13 +100,16 @@ class Queue { T* try_pop(Monitor* lock, TRAPS) { MonitorLocker locker(THREAD, lock); - T* value = nullptr; - if (!is_empty_unlocked()) { - value = pop_unlocked(); - } + T* value = pop_unlocked(); return value; } + void processing_done(Monitor* lock, TRAPS) { + MonitorLocker locker(THREAD, lock); + processing_done_unlocked(); + locker.notify_all(); + } + void print_on(outputStream* st); }; } // namespace CompilationPolicyUtils @@ -352,6 +362,7 @@ class CompilationPolicy : AllStatic { // This supports the -Xcomp option. static void compile_if_required(const methodHandle& m, TRAPS); + static void flush_replay_training_at_init(TRAPS); static void replay_training_at_init(InstanceKlass* klass, TRAPS); static void replay_training_at_init_loop(TRAPS); diff --git a/src/hotspot/share/oops/trainingData.cpp b/src/hotspot/share/oops/trainingData.cpp index 70e8f2437c1b6..c9da16a1222ee 100644 --- a/src/hotspot/share/oops/trainingData.cpp +++ b/src/hotspot/share/oops/trainingData.cpp @@ -83,7 +83,7 @@ static void verify_archived_entry(TrainingData* td, const TrainingData::Key* k) } void TrainingData::verify() { - if (TrainingData::have_data()) { + if (TrainingData::have_data() && !TrainingData::assembling_data()) { archived_training_data_dictionary()->iterate([&](TrainingData* td) { if (td->is_KlassTrainingData()) { KlassTrainingData* ktd = td->as_KlassTrainingData(); @@ -98,9 +98,19 @@ void TrainingData::verify() { Key k(mtd->holder()); verify_archived_entry(td, &k); } - mtd->verify(); - } else if (td->is_CompileTrainingData()) { - td->as_CompileTrainingData()->verify(); + mtd->verify(true); + } + }); + } + if (TrainingData::need_data()) { + TrainingDataLocker l; + training_data_set()->iterate([&](TrainingData* td) { + if (td->is_KlassTrainingData()) { + KlassTrainingData* ktd = td->as_KlassTrainingData(); + ktd->verify(); + } else if (td->is_MethodTrainingData()) { + MethodTrainingData* mtd = td->as_MethodTrainingData(); + mtd->verify(false); } }); } @@ -476,10 +486,10 @@ void TrainingData::init_dumptime_table(TRAPS) { _dumptime_training_data_dictionary->append(td); } }); + } - if (AOTVerifyTrainingData) { - training_data_set()->verify(); - } + if (AOTVerifyTrainingData) { + TrainingData::verify(); } } @@ -592,22 +602,13 @@ void KlassTrainingData::verify() { } } -void MethodTrainingData::verify() { - iterate_compiles([](CompileTrainingData* ctd) { - ctd->verify(); - - int init_deps_left1 = ctd->init_deps_left(); - int init_deps_left2 = ctd->compute_init_deps_left(); - - if (init_deps_left1 != init_deps_left2) { - ctd->print_on(tty); tty->cr(); - } - guarantee(init_deps_left1 == init_deps_left2, "mismatch: %d %d %d", - init_deps_left1, init_deps_left2, ctd->init_deps_left()); +void MethodTrainingData::verify(bool verify_dep_counter) { + iterate_compiles([&](CompileTrainingData* ctd) { + ctd->verify(verify_dep_counter); }); } -void CompileTrainingData::verify() { +void CompileTrainingData::verify(bool verify_dep_counter) { for (int i = 0; i < init_dep_count(); i++) { KlassTrainingData* ktd = init_dep(i); if (ktd->has_holder() && ktd->holder()->defined_by_other_loaders()) { @@ -624,6 +625,18 @@ void CompileTrainingData::verify() { } guarantee(ktd->_comp_deps.contains(this), ""); } + + if (verify_dep_counter) { + int init_deps_left1 = init_deps_left(); + int init_deps_left2 = compute_init_deps_left(); + + if (init_deps_left1 != init_deps_left2) { + print_on(tty); + tty->cr(); + } + guarantee(init_deps_left1 == init_deps_left2, + "init deps invariant violation: %d == %d", init_deps_left1, init_deps_left2); + } } void CompileTrainingData::cleanup(Visitor& visitor) { diff --git a/src/hotspot/share/oops/trainingData.hpp b/src/hotspot/share/oops/trainingData.hpp index baf19773a7b6a..502fa87fbe3e4 100644 --- a/src/hotspot/share/oops/trainingData.hpp +++ b/src/hotspot/share/oops/trainingData.hpp @@ -675,7 +675,7 @@ class CompileTrainingData : public TrainingData { } void dec_init_deps_left(KlassTrainingData* ktd); int init_deps_left() const { - return Atomic::load(&_init_deps_left); + return Atomic::load_acquire(&_init_deps_left); } uint compute_init_deps_left(bool count_initialized = false); @@ -707,7 +707,7 @@ class CompileTrainingData : public TrainingData { return (int)align_metadata_size(align_up(sizeof(CompileTrainingData), BytesPerWord)/BytesPerWord); } - void verify(); + void verify(bool verify_dep_counter); static CompileTrainingData* allocate(MethodTrainingData* mtd, int level, int compile_id) { return TrainingData::allocate(mtd, level, compile_id); @@ -828,7 +828,7 @@ class MethodTrainingData : public TrainingData { return "{ method training data }"; }; - void verify(); + void verify(bool verify_dep_counter); static MethodTrainingData* allocate(Method* m, KlassTrainingData* ktd) { return TrainingData::allocate(m, ktd); diff --git a/src/hotspot/share/runtime/init.cpp b/src/hotspot/share/runtime/init.cpp index 73292a7b6d91c..56e6ea30c0a9b 100644 --- a/src/hotspot/share/runtime/init.cpp +++ b/src/hotspot/share/runtime/init.cpp @@ -195,10 +195,7 @@ jint init_globals2() { } #endif - // Initialize TrainingData only we're recording/replaying - if (TrainingData::have_data() || TrainingData::need_data()) { - TrainingData::initialize(); - } + TrainingData::initialize(); if (!universe_post_init()) { return JNI_ERR; diff --git a/src/hotspot/share/runtime/java.cpp b/src/hotspot/share/runtime/java.cpp index 3361ecf15581d..84a42b3bc2eff 100644 --- a/src/hotspot/share/runtime/java.cpp +++ b/src/hotspot/share/runtime/java.cpp @@ -34,6 +34,7 @@ #include "classfile/systemDictionary.hpp" #include "code/codeCache.hpp" #include "compiler/compilationMemoryStatistic.hpp" +#include "compiler/compilationPolicy.hpp" #include "compiler/compileBroker.hpp" #include "compiler/compilerOracle.hpp" #include "gc/shared/collectedHeap.hpp" @@ -515,6 +516,12 @@ void before_exit(JavaThread* thread, bool halt) { // Note: we don't wait until it actually dies. os::terminate_signal_thread(); + if (AOTVerifyTrainingData) { + EXCEPTION_MARK; + CompilationPolicy::flush_replay_training_at_init(THREAD); + TrainingData::verify(); + } + print_statistics(); { MutexLocker ml(BeforeExit_lock); From b9dde1394ec34ce51335a6051388193ea05feaad Mon Sep 17 00:00:00 2001 From: Igor Veresov Date: Wed, 20 Aug 2025 11:28:45 -0700 Subject: [PATCH 2/7] Fix minimal build --- src/hotspot/share/runtime/java.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/share/runtime/java.cpp b/src/hotspot/share/runtime/java.cpp index 84a42b3bc2eff..86d99982bb126 100644 --- a/src/hotspot/share/runtime/java.cpp +++ b/src/hotspot/share/runtime/java.cpp @@ -516,11 +516,13 @@ void before_exit(JavaThread* thread, bool halt) { // Note: we don't wait until it actually dies. os::terminate_signal_thread(); + #if INCLUDE_CDS if (AOTVerifyTrainingData) { EXCEPTION_MARK; CompilationPolicy::flush_replay_training_at_init(THREAD); TrainingData::verify(); } + #endif print_statistics(); From 3a4690fc04db4c235e2a4a1470a0331776d4b769 Mon Sep 17 00:00:00 2001 From: Igor Veresov Date: Wed, 20 Aug 2025 19:41:14 -0700 Subject: [PATCH 3/7] Rephrase arguments of the flush() --- src/hotspot/share/compiler/compilationPolicy.cpp | 2 +- src/hotspot/share/compiler/compilationPolicy.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/compiler/compilationPolicy.cpp b/src/hotspot/share/compiler/compilationPolicy.cpp index aab1a5d64a089..db3fefe6489fb 100644 --- a/src/hotspot/share/compiler/compilationPolicy.cpp +++ b/src/hotspot/share/compiler/compilationPolicy.cpp @@ -138,7 +138,7 @@ void CompilationPolicy::compile_if_required(const methodHandle& m, TRAPS) { } } - void CompilationPolicy::flush_replay_training_at_init(TRAPS) { + void CompilationPolicy::flush_replay_training_at_init(JavaThread* THREAD) { MonitorLocker locker(THREAD, TrainingReplayQueue_lock); while (!_training_replay_queue.is_empty_unlocked() || _training_replay_queue.is_processing_unlocked()) { locker.wait(); // let the replay training thread drain the queue diff --git a/src/hotspot/share/compiler/compilationPolicy.hpp b/src/hotspot/share/compiler/compilationPolicy.hpp index cb49ccee4f647..4f23cfb201762 100644 --- a/src/hotspot/share/compiler/compilationPolicy.hpp +++ b/src/hotspot/share/compiler/compilationPolicy.hpp @@ -362,7 +362,7 @@ class CompilationPolicy : AllStatic { // This supports the -Xcomp option. static void compile_if_required(const methodHandle& m, TRAPS); - static void flush_replay_training_at_init(TRAPS); + static void flush_replay_training_at_init(JavaThread* THREAD); static void replay_training_at_init(InstanceKlass* klass, TRAPS); static void replay_training_at_init_loop(TRAPS); From 289fb74c7ef3c8a477eb39a79c17433f671fb77b Mon Sep 17 00:00:00 2001 From: Igor Veresov Date: Wed, 20 Aug 2025 19:54:29 -0700 Subject: [PATCH 4/7] More cleanup --- src/hotspot/share/compiler/compilationPolicy.cpp | 6 +++--- src/hotspot/share/compiler/compilationPolicy.hpp | 14 +++++++------- src/hotspot/share/runtime/java.cpp | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/compiler/compilationPolicy.cpp b/src/hotspot/share/compiler/compilationPolicy.cpp index db3fefe6489fb..ec74f0cdffc61 100644 --- a/src/hotspot/share/compiler/compilationPolicy.cpp +++ b/src/hotspot/share/compiler/compilationPolicy.cpp @@ -138,7 +138,7 @@ void CompilationPolicy::compile_if_required(const methodHandle& m, TRAPS) { } } - void CompilationPolicy::flush_replay_training_at_init(JavaThread* THREAD) { + void CompilationPolicy::wait_replay_training_at_init(JavaThread* THREAD) { MonitorLocker locker(THREAD, TrainingReplayQueue_lock); while (!_training_replay_queue.is_empty_unlocked() || _training_replay_queue.is_processing_unlocked()) { locker.wait(); // let the replay training thread drain the queue @@ -170,7 +170,7 @@ void CompilationPolicy::replay_training_at_init_impl(InstanceKlass* klass, TRAPS } } -void CompilationPolicy::replay_training_at_init(InstanceKlass* klass, TRAPS) { +void CompilationPolicy::replay_training_at_init(InstanceKlass* klass, JavaThread* THREAD) { assert(klass->is_initialized(), ""); if (TrainingData::have_data() && klass->is_shared()) { _training_replay_queue.push(klass, TrainingReplayQueue_lock, THREAD); @@ -188,7 +188,7 @@ void CompilationPolicyUtils::Queue::print_on(outputStream* st) { } } -void CompilationPolicy::replay_training_at_init_loop(TRAPS) { +void CompilationPolicy::replay_training_at_init_loop(JavaThread* THREAD) { while (!CompileBroker::is_compilation_disabled_forever()) { InstanceKlass* ik = _training_replay_queue.pop(TrainingReplayQueue_lock, THREAD); if (ik != nullptr) { diff --git a/src/hotspot/share/compiler/compilationPolicy.hpp b/src/hotspot/share/compiler/compilationPolicy.hpp index 4f23cfb201762..bbc33c6df09dc 100644 --- a/src/hotspot/share/compiler/compilationPolicy.hpp +++ b/src/hotspot/share/compiler/compilationPolicy.hpp @@ -80,7 +80,7 @@ class Queue { } public: Queue() : _head(nullptr), _tail(nullptr), _processing(0) { } - void push(T* value, Monitor* lock, TRAPS) { + void push(T* value, Monitor* lock, JavaThread* THREAD) { MonitorLocker locker(THREAD, lock); push_unlocked(value); locker.notify_all(); @@ -89,7 +89,7 @@ class Queue { bool is_empty_unlocked() const { return _head == nullptr; } bool is_processing_unlocked() const { return _processing > 0; } - T* pop(Monitor* lock, TRAPS) { + T* pop(Monitor* lock, JavaThread* THREAD) { MonitorLocker locker(THREAD, lock); while(is_empty_unlocked() && !CompileBroker::is_compilation_disabled_forever()) { locker.wait(); @@ -98,13 +98,13 @@ class Queue { return value; } - T* try_pop(Monitor* lock, TRAPS) { + T* try_pop(Monitor* lock, JavaThread* THREAD) { MonitorLocker locker(THREAD, lock); T* value = pop_unlocked(); return value; } - void processing_done(Monitor* lock, TRAPS) { + void processing_done(Monitor* lock, JavaThread* THREAD) { MonitorLocker locker(THREAD, lock); processing_done_unlocked(); locker.notify_all(); @@ -362,9 +362,9 @@ class CompilationPolicy : AllStatic { // This supports the -Xcomp option. static void compile_if_required(const methodHandle& m, TRAPS); - static void flush_replay_training_at_init(JavaThread* THREAD); - static void replay_training_at_init(InstanceKlass* klass, TRAPS); - static void replay_training_at_init_loop(TRAPS); + static void wait_replay_training_at_init(JavaThread* THREAD); + static void replay_training_at_init(InstanceKlass* klass, JavaThread* THREAD); + static void replay_training_at_init_loop(JavaThread* THREAD); // m is allowed to be compiled static bool can_be_compiled(const methodHandle& m, int comp_level = CompLevel_any); diff --git a/src/hotspot/share/runtime/java.cpp b/src/hotspot/share/runtime/java.cpp index 86d99982bb126..7cc4a42de856d 100644 --- a/src/hotspot/share/runtime/java.cpp +++ b/src/hotspot/share/runtime/java.cpp @@ -519,7 +519,7 @@ void before_exit(JavaThread* thread, bool halt) { #if INCLUDE_CDS if (AOTVerifyTrainingData) { EXCEPTION_MARK; - CompilationPolicy::flush_replay_training_at_init(THREAD); + CompilationPolicy::wait_replay_training_at_init(THREAD); TrainingData::verify(); } #endif From edc8591d43ade28a093f49d65839792d51546435 Mon Sep 17 00:00:00 2001 From: Igor Veresov Date: Thu, 21 Aug 2025 10:29:17 -0700 Subject: [PATCH 5/7] More cleanup and renames --- .../share/compiler/compilationPolicy.cpp | 22 ++++++++--------- .../share/compiler/compilationPolicy.hpp | 24 +++++++++---------- src/hotspot/share/oops/trainingData.cpp | 6 +++-- src/hotspot/share/oops/trainingData.hpp | 4 +--- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/hotspot/share/compiler/compilationPolicy.cpp b/src/hotspot/share/compiler/compilationPolicy.cpp index ec74f0cdffc61..5d4ab3a245d11 100644 --- a/src/hotspot/share/compiler/compilationPolicy.cpp +++ b/src/hotspot/share/compiler/compilationPolicy.cpp @@ -138,14 +138,14 @@ void CompilationPolicy::compile_if_required(const methodHandle& m, TRAPS) { } } - void CompilationPolicy::wait_replay_training_at_init(JavaThread* THREAD) { - MonitorLocker locker(THREAD, TrainingReplayQueue_lock); + void CompilationPolicy::wait_replay_training_at_init(JavaThread* current) { + MonitorLocker locker(current, TrainingReplayQueue_lock); while (!_training_replay_queue.is_empty_unlocked() || _training_replay_queue.is_processing_unlocked()) { locker.wait(); // let the replay training thread drain the queue } } -void CompilationPolicy::replay_training_at_init_impl(InstanceKlass* klass, TRAPS) { +void CompilationPolicy::replay_training_at_init_impl(InstanceKlass* klass, JavaThread* current) { if (!klass->has_init_deps_processed()) { ResourceMark rm; log_debug(training)("Replay training: %s", klass->external_name()); @@ -160,8 +160,8 @@ void CompilationPolicy::replay_training_at_init_impl(InstanceKlass* klass, TRAPS if (ctd->init_deps_left() == 0) { MethodTrainingData* mtd = ctd->method(); if (mtd->has_holder()) { - const methodHandle mh(THREAD, const_cast(mtd->holder())); - CompilationPolicy::maybe_compile_early(mh, THREAD); + const methodHandle mh(current, const_cast(mtd->holder())); + CompilationPolicy::maybe_compile_early(mh, current); } } }); @@ -170,10 +170,10 @@ void CompilationPolicy::replay_training_at_init_impl(InstanceKlass* klass, TRAPS } } -void CompilationPolicy::replay_training_at_init(InstanceKlass* klass, JavaThread* THREAD) { +void CompilationPolicy::replay_training_at_init(InstanceKlass* klass, JavaThread* current) { assert(klass->is_initialized(), ""); if (TrainingData::have_data() && klass->is_shared()) { - _training_replay_queue.push(klass, TrainingReplayQueue_lock, THREAD); + _training_replay_queue.push(klass, TrainingReplayQueue_lock, current); } } @@ -188,12 +188,12 @@ void CompilationPolicyUtils::Queue::print_on(outputStream* st) { } } -void CompilationPolicy::replay_training_at_init_loop(JavaThread* THREAD) { +void CompilationPolicy::replay_training_at_init_loop(JavaThread* current) { while (!CompileBroker::is_compilation_disabled_forever()) { - InstanceKlass* ik = _training_replay_queue.pop(TrainingReplayQueue_lock, THREAD); + InstanceKlass* ik = _training_replay_queue.pop(TrainingReplayQueue_lock, current); if (ik != nullptr) { - replay_training_at_init_impl(ik, THREAD); - _training_replay_queue.processing_done(TrainingReplayQueue_lock, THREAD); + replay_training_at_init_impl(ik, current); + _training_replay_queue.processing_done(TrainingReplayQueue_lock, current); } } } diff --git a/src/hotspot/share/compiler/compilationPolicy.hpp b/src/hotspot/share/compiler/compilationPolicy.hpp index bbc33c6df09dc..62989f94ebaf7 100644 --- a/src/hotspot/share/compiler/compilationPolicy.hpp +++ b/src/hotspot/share/compiler/compilationPolicy.hpp @@ -80,8 +80,8 @@ class Queue { } public: Queue() : _head(nullptr), _tail(nullptr), _processing(0) { } - void push(T* value, Monitor* lock, JavaThread* THREAD) { - MonitorLocker locker(THREAD, lock); + void push(T* value, Monitor* lock, JavaThread* current) { + MonitorLocker locker(current, lock); push_unlocked(value); locker.notify_all(); } @@ -89,8 +89,8 @@ class Queue { bool is_empty_unlocked() const { return _head == nullptr; } bool is_processing_unlocked() const { return _processing > 0; } - T* pop(Monitor* lock, JavaThread* THREAD) { - MonitorLocker locker(THREAD, lock); + T* pop(Monitor* lock, JavaThread* current) { + MonitorLocker locker(current, lock); while(is_empty_unlocked() && !CompileBroker::is_compilation_disabled_forever()) { locker.wait(); } @@ -98,14 +98,14 @@ class Queue { return value; } - T* try_pop(Monitor* lock, JavaThread* THREAD) { - MonitorLocker locker(THREAD, lock); + T* try_pop(Monitor* lock, JavaThread* current) { + MonitorLocker locker(current, lock); T* value = pop_unlocked(); return value; } - void processing_done(Monitor* lock, JavaThread* THREAD) { - MonitorLocker locker(THREAD, lock); + void processing_done(Monitor* lock, JavaThread* current) { + MonitorLocker locker(current, lock); processing_done_unlocked(); locker.notify_all(); } @@ -352,7 +352,7 @@ class CompilationPolicy : AllStatic { // m must be compiled before executing it static bool must_be_compiled(const methodHandle& m, int comp_level = CompLevel_any); static void maybe_compile_early(const methodHandle& m, TRAPS); - static void replay_training_at_init_impl(InstanceKlass* klass, TRAPS); + static void replay_training_at_init_impl(InstanceKlass* klass, JavaThread* current); public: static int min_invocations() { return Tier4MinInvocationThreshold; } static int c1_count() { return _c1_count; } @@ -362,9 +362,9 @@ class CompilationPolicy : AllStatic { // This supports the -Xcomp option. static void compile_if_required(const methodHandle& m, TRAPS); - static void wait_replay_training_at_init(JavaThread* THREAD); - static void replay_training_at_init(InstanceKlass* klass, JavaThread* THREAD); - static void replay_training_at_init_loop(JavaThread* THREAD); + static void wait_replay_training_at_init(JavaThread* current); + static void replay_training_at_init(InstanceKlass* klass, JavaThread* current); + static void replay_training_at_init_loop(JavaThread* current); // m is allowed to be compiled static bool can_be_compiled(const methodHandle& m, int comp_level = CompLevel_any); diff --git a/src/hotspot/share/oops/trainingData.cpp b/src/hotspot/share/oops/trainingData.cpp index c9da16a1222ee..4a78aa05929a2 100644 --- a/src/hotspot/share/oops/trainingData.cpp +++ b/src/hotspot/share/oops/trainingData.cpp @@ -98,7 +98,7 @@ void TrainingData::verify() { Key k(mtd->holder()); verify_archived_entry(td, &k); } - mtd->verify(true); + mtd->verify(/*verify_dep_counter*/true); } }); } @@ -110,7 +110,9 @@ void TrainingData::verify() { ktd->verify(); } else if (td->is_MethodTrainingData()) { MethodTrainingData* mtd = td->as_MethodTrainingData(); - mtd->verify(false); + // During the training run init deps tracking is not setup yet, + // don't verify it. + mtd->verify(/*verify_dep_counter*/false); } }); } diff --git a/src/hotspot/share/oops/trainingData.hpp b/src/hotspot/share/oops/trainingData.hpp index 502fa87fbe3e4..c6717f60d69be 100644 --- a/src/hotspot/share/oops/trainingData.hpp +++ b/src/hotspot/share/oops/trainingData.hpp @@ -674,9 +674,7 @@ class CompileTrainingData : public TrainingData { _init_deps.clear(); } void dec_init_deps_left(KlassTrainingData* ktd); - int init_deps_left() const { - return Atomic::load_acquire(&_init_deps_left); - } + int init_deps_left() const { return _init_deps_left; } uint compute_init_deps_left(bool count_initialized = false); void notice_inlined_method(CompileTask* task, const methodHandle& method) NOT_CDS_RETURN; From f7d6a4e023d9ea359860d065c5cdac11fb58e5c6 Mon Sep 17 00:00:00 2001 From: Igor Veresov Date: Thu, 21 Aug 2025 20:16:57 -0700 Subject: [PATCH 6/7] More renames --- src/hotspot/share/compiler/compilationPolicy.cpp | 12 ++++++------ src/hotspot/share/compiler/compilationPolicy.hpp | 2 +- src/hotspot/share/oops/trainingData.cpp | 6 +++--- src/hotspot/share/oops/trainingData.hpp | 6 ++++-- src/hotspot/share/runtime/java.cpp | 2 +- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/compiler/compilationPolicy.cpp b/src/hotspot/share/compiler/compilationPolicy.cpp index 5d4ab3a245d11..18410c288f4a2 100644 --- a/src/hotspot/share/compiler/compilationPolicy.cpp +++ b/src/hotspot/share/compiler/compilationPolicy.cpp @@ -138,7 +138,7 @@ void CompilationPolicy::compile_if_required(const methodHandle& m, TRAPS) { } } - void CompilationPolicy::wait_replay_training_at_init(JavaThread* current) { + void CompilationPolicy::wait_replay_training_to_finish(JavaThread* current) { MonitorLocker locker(current, TrainingReplayQueue_lock); while (!_training_replay_queue.is_empty_unlocked() || _training_replay_queue.is_processing_unlocked()) { locker.wait(); // let the replay training thread drain the queue @@ -157,7 +157,7 @@ void CompilationPolicy::replay_training_at_init_impl(InstanceKlass* klass, JavaT assert(klass->has_init_deps_processed(), ""); if (AOTCompileEagerly) { ktd->iterate_comp_deps([&](CompileTrainingData* ctd) { - if (ctd->init_deps_left() == 0) { + if (ctd->init_deps_left_acquire() == 0) { MethodTrainingData* mtd = ctd->method(); if (mtd->has_holder()) { const methodHandle mh(current, const_cast(mtd->holder())); @@ -189,7 +189,7 @@ void CompilationPolicyUtils::Queue::print_on(outputStream* st) { } void CompilationPolicy::replay_training_at_init_loop(JavaThread* current) { - while (!CompileBroker::is_compilation_disabled_forever()) { + while (!CompileBroker::is_compilation_disabled_forever() || AOTVerifyTrainingData) { InstanceKlass* ik = _training_replay_queue.pop(TrainingReplayQueue_lock, current); if (ik != nullptr) { replay_training_at_init_impl(ik, current); @@ -454,7 +454,7 @@ void CompilationPolicy::print_training_data_on(outputStream* st, const char* pr if (ctd == nullptr) { st->print("null"); } else { - st->print("%d", ctd->init_deps_left()); + st->print("%d", ctd->init_deps_left_acquire()); } } } @@ -1180,7 +1180,7 @@ CompLevel CompilationPolicy::trained_transition_from_none(const methodHandle& me CompileTrainingData* ctd = mtd->last_toplevel_compile(CompLevel_full_optimization); assert(ctd != nullptr, "Should have CTD for CompLevel_full_optimization"); // With SkipTier2IfPossible and all deps satisfied, go to level 4 immediately - if (SkipTier2IfPossible && ctd->init_deps_left() == 0) { + if (SkipTier2IfPossible && ctd->init_deps_left_acquire() == 0) { if (method->method_data() == nullptr) { create_mdo(method, THREAD); } @@ -1208,7 +1208,7 @@ CompLevel CompilationPolicy::trained_transition_from_limited_profile(const metho assert(training_has_profile, "Have to have a profile to be here"); // Check if the method is ready CompileTrainingData* ctd = mtd->last_toplevel_compile(CompLevel_full_optimization); - if (ctd != nullptr && ctd->init_deps_left() == 0) { + if (ctd != nullptr && ctd->init_deps_left_acquire() == 0) { if (method->method_data() == nullptr) { create_mdo(method, THREAD); } diff --git a/src/hotspot/share/compiler/compilationPolicy.hpp b/src/hotspot/share/compiler/compilationPolicy.hpp index 62989f94ebaf7..fd88e64660df4 100644 --- a/src/hotspot/share/compiler/compilationPolicy.hpp +++ b/src/hotspot/share/compiler/compilationPolicy.hpp @@ -362,7 +362,7 @@ class CompilationPolicy : AllStatic { // This supports the -Xcomp option. static void compile_if_required(const methodHandle& m, TRAPS); - static void wait_replay_training_at_init(JavaThread* current); + static void wait_replay_training_to_finish(JavaThread* current); static void replay_training_at_init(InstanceKlass* klass, JavaThread* current); static void replay_training_at_init_loop(JavaThread* current); diff --git a/src/hotspot/share/oops/trainingData.cpp b/src/hotspot/share/oops/trainingData.cpp index 4a78aa05929a2..0162c25c73a0e 100644 --- a/src/hotspot/share/oops/trainingData.cpp +++ b/src/hotspot/share/oops/trainingData.cpp @@ -241,7 +241,7 @@ CompileTrainingData* CompileTrainingData::make(CompileTask* task) { } -void CompileTrainingData::dec_init_deps_left(KlassTrainingData* ktd) { +void CompileTrainingData::dec_init_deps_left_release(KlassTrainingData* ktd) { LogStreamHandle(Trace, training) log; if (log.is_enabled()) { log.print("CTD "); print_on(&log); log.cr(); @@ -462,7 +462,7 @@ void KlassTrainingData::notice_fully_initialized() { TrainingDataLocker l; // Not a real lock if we don't collect the data, // that's why we need the atomic decrement below. for (int i = 0; i < comp_dep_count(); i++) { - comp_dep(i)->dec_init_deps_left(this); + comp_dep(i)->dec_init_deps_left_release(this); } holder()->set_has_init_deps_processed(); } @@ -629,7 +629,7 @@ void CompileTrainingData::verify(bool verify_dep_counter) { } if (verify_dep_counter) { - int init_deps_left1 = init_deps_left(); + int init_deps_left1 = init_deps_left_acquire(); int init_deps_left2 = compute_init_deps_left(); if (init_deps_left1 != init_deps_left2) { diff --git a/src/hotspot/share/oops/trainingData.hpp b/src/hotspot/share/oops/trainingData.hpp index c6717f60d69be..564a72ec44d01 100644 --- a/src/hotspot/share/oops/trainingData.hpp +++ b/src/hotspot/share/oops/trainingData.hpp @@ -673,8 +673,10 @@ class CompileTrainingData : public TrainingData { } _init_deps.clear(); } - void dec_init_deps_left(KlassTrainingData* ktd); - int init_deps_left() const { return _init_deps_left; } + void dec_init_deps_left_release(KlassTrainingData* ktd); + int init_deps_left_acquire() const { + return Atomic::load_acquire(&_init_deps_left); + } uint compute_init_deps_left(bool count_initialized = false); void notice_inlined_method(CompileTask* task, const methodHandle& method) NOT_CDS_RETURN; diff --git a/src/hotspot/share/runtime/java.cpp b/src/hotspot/share/runtime/java.cpp index 7cc4a42de856d..daa957cf6c375 100644 --- a/src/hotspot/share/runtime/java.cpp +++ b/src/hotspot/share/runtime/java.cpp @@ -519,7 +519,7 @@ void before_exit(JavaThread* thread, bool halt) { #if INCLUDE_CDS if (AOTVerifyTrainingData) { EXCEPTION_MARK; - CompilationPolicy::wait_replay_training_at_init(THREAD); + CompilationPolicy::wait_replay_training_to_finish(THREAD); TrainingData::verify(); } #endif From c33d94bcae9da698c771ee9633722f6b0b576633 Mon Sep 17 00:00:00 2001 From: Igor Veresov Date: Fri, 22 Aug 2025 13:22:27 -0700 Subject: [PATCH 7/7] One more nit --- src/hotspot/share/compiler/compilationPolicy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/compiler/compilationPolicy.hpp b/src/hotspot/share/compiler/compilationPolicy.hpp index fd88e64660df4..6a3a8d1ffb679 100644 --- a/src/hotspot/share/compiler/compilationPolicy.hpp +++ b/src/hotspot/share/compiler/compilationPolicy.hpp @@ -91,7 +91,7 @@ class Queue { T* pop(Monitor* lock, JavaThread* current) { MonitorLocker locker(current, lock); - while(is_empty_unlocked() && !CompileBroker::is_compilation_disabled_forever()) { + while (is_empty_unlocked() && (!CompileBroker::is_compilation_disabled_forever() || AOTVerifyTrainingData)) { locker.wait(); } T* value = pop_unlocked();