diff --git a/src/hotspot/share/gc/shared/gc_globals.hpp b/src/hotspot/share/gc/shared/gc_globals.hpp index 7f46f44908556..9df1f19a05f5f 100644 --- a/src/hotspot/share/gc/shared/gc_globals.hpp +++ b/src/hotspot/share/gc/shared/gc_globals.hpp @@ -469,7 +469,7 @@ \ product(size_t, SoftMaxHeapSize, 0, MANAGEABLE, \ "Soft limit for maximum heap size (in bytes)") \ - constraint(SoftMaxHeapSizeConstraintFunc,AfterMemoryInit) \ + constraint(SoftMaxHeapSizeConstraintFunc,AfterErgo) \ \ product(size_t, NewSize, ScaleForWordSize(1*M), \ "Initial new generation size (in bytes)") \ diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp index 4e12b1d41e8cc..af431b7e4ac33 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahGlobalHeuristics.cpp @@ -91,9 +91,10 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; log_info(gc, ergo)("Adaptive CSet Selection for GLOBAL. Max Young Evacuation: %zu" - "%s, Max Old Evacuation: %zu%s, Actual Free: %zu%s.", + "%s, Max Old Evacuation: %zu%s, Max Either Evacuation: %zu%s, Actual Free: %zu%s.", byte_size_in_proper_unit(max_young_cset), proper_unit_for_byte_size(max_young_cset), byte_size_in_proper_unit(max_old_cset), proper_unit_for_byte_size(max_old_cset), + byte_size_in_proper_unit(unaffiliated_young_memory), proper_unit_for_byte_size(unaffiliated_young_memory), byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); for (size_t idx = 0; idx < size; idx++) { @@ -136,9 +137,8 @@ void ShenandoahGlobalHeuristics::choose_global_collection_set(ShenandoahCollecti cset->add_region(r); } } - if (regions_transferred_to_old > 0) { - heap->generation_sizer()->force_transfer_to_old(regions_transferred_to_old); + assert(young_evac_reserve > regions_transferred_to_old * region_size_bytes, "young reserve cannot be negative"); heap->young_generation()->set_evacuation_reserve(young_evac_reserve - regions_transferred_to_old * region_size_bytes); heap->old_generation()->set_evacuation_reserve(old_evac_reserve + regions_transferred_to_old * region_size_bytes); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 2d0bbfd5e4a3c..d049e7edf0b57 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -602,12 +602,12 @@ void ShenandoahOldHeuristics::set_trigger_if_old_is_fragmented(size_t first_old_ } void ShenandoahOldHeuristics::set_trigger_if_old_is_overgrown() { - size_t old_used = _old_generation->used() + _old_generation->get_humongous_waste(); + // used() includes humongous waste + size_t old_used = _old_generation->used(); size_t trigger_threshold = _old_generation->usage_trigger_threshold(); // Detects unsigned arithmetic underflow assert(old_used <= _heap->capacity(), - "Old used (%zu, %zu) must not be more than heap capacity (%zu)", - _old_generation->used(), _old_generation->get_humongous_waste(), _heap->capacity()); + "Old used (%zu) must not be more than heap capacity (%zu)", _old_generation->used(), _heap->capacity()); if (old_used > trigger_threshold) { _growth_trigger = true; } @@ -679,7 +679,8 @@ bool ShenandoahOldHeuristics::should_start_gc() { if (_growth_trigger) { // Growth may be falsely triggered during mixed evacuations, before the mixed-evacuation candidates have been // evacuated. Before acting on a false trigger, we check to confirm the trigger condition is still satisfied. - const size_t current_usage = _old_generation->used() + _old_generation->get_humongous_waste(); + // _old_generation->used() includes humongous waste. + const size_t current_usage = _old_generation->used(); const size_t trigger_threshold = _old_generation->usage_trigger_threshold(); const size_t heap_size = heap->capacity(); const size_t ignore_threshold = (ShenandoahIgnoreOldGrowthBelowPercentage * heap_size) / 100; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 81154aff9f0ad..fbfcf7cf5909a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -1164,6 +1164,9 @@ void ShenandoahConcurrentGC::op_update_thread_roots() { void ShenandoahConcurrentGC::op_final_update_refs() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); + bool is_generational = heap->mode()->is_generational(); + ShenandoahGenerationalHeap* const gen_heap = is_generational? ShenandoahGenerationalHeap::heap(): nullptr; + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint"); assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references"); @@ -1187,7 +1190,7 @@ void ShenandoahConcurrentGC::op_final_update_refs() { heap->set_update_refs_in_progress(false); heap->set_has_forwarded_objects(false); - if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) { + if (is_generational && heap->is_concurrent_old_mark_in_progress()) { // When the SATB barrier is left on to support concurrent old gen mark, it may pick up writes to // objects in the collection set. After those objects are evacuated, the pointers in the // SATB are no longer safe. Once we have finished update references, we are guaranteed that @@ -1206,7 +1209,7 @@ void ShenandoahConcurrentGC::op_final_update_refs() { // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for // entire regions. Both of these relevant operations occur before final update refs. - ShenandoahGenerationalHeap::heap()->set_aging_cycle(false); + gen_heap->set_aging_cycle(false); } if (ShenandoahVerify) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 293391a86eba3..2f5ab4fc1aaec 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -52,12 +52,13 @@ static const char* partition_name(ShenandoahFreeSetPartitionId t) { class ShenandoahLeftRightIterator { private: - idx_t _idx; - idx_t _end; + index_type _idx; + index_type _end; ShenandoahRegionPartitions* _partitions; ShenandoahFreeSetPartitionId _partition; public: - explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions, ShenandoahFreeSetPartitionId partition, bool use_empty = false) + explicit ShenandoahLeftRightIterator(ShenandoahRegionPartitions* partitions, + ShenandoahFreeSetPartitionId partition, bool use_empty = false) : _idx(0), _end(0), _partitions(partitions), _partition(partition) { _idx = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); _end = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); @@ -71,11 +72,11 @@ class ShenandoahLeftRightIterator { return false; } - idx_t current() const { + index_type current() const { return _idx; } - idx_t next() { + index_type next() { _idx = _partitions->find_index_of_next_available_region(_partition, _idx + 1); return current(); } @@ -83,12 +84,13 @@ class ShenandoahLeftRightIterator { class ShenandoahRightLeftIterator { private: - idx_t _idx; - idx_t _end; + index_type _idx; + index_type _end; ShenandoahRegionPartitions* _partitions; ShenandoahFreeSetPartitionId _partition; public: - explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions, ShenandoahFreeSetPartitionId partition, bool use_empty = false) + explicit ShenandoahRightLeftIterator(ShenandoahRegionPartitions* partitions, + ShenandoahFreeSetPartitionId partition, bool use_empty = false) : _idx(0), _end(0), _partitions(partitions), _partition(partition) { _idx = use_empty ? _partitions->rightmost_empty(_partition) : _partitions->rightmost(_partition); _end = use_empty ? _partitions->leftmost_empty(_partition) : _partitions->leftmost(_partition); @@ -102,11 +104,11 @@ class ShenandoahRightLeftIterator { return false; } - idx_t current() const { + index_type current() const { return _idx; } - idx_t next() { + index_type next() { _idx = _partitions->find_index_of_previous_available_region(_partition, _idx - 1); return current(); } @@ -136,21 +138,21 @@ void ShenandoahRegionPartitions::dump_bitmap() const { dump_bitmap_range(0, _max-1); } -void ShenandoahRegionPartitions::dump_bitmap_range(idx_t start_region_idx, idx_t end_region_idx) const { - assert((start_region_idx >= 0) && (start_region_idx < (idx_t) _max), "precondition"); - assert((end_region_idx >= 0) && (end_region_idx < (idx_t) _max), "precondition"); - idx_t aligned_start = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].aligned_index(start_region_idx); - idx_t aligned_end = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].aligned_index(end_region_idx); - idx_t alignment = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].alignment(); +void ShenandoahRegionPartitions::dump_bitmap_range(index_type start_region_idx, index_type end_region_idx) const { + assert((start_region_idx >= 0) && (start_region_idx < (index_type) _max), "precondition"); + assert((end_region_idx >= 0) && (end_region_idx < (index_type) _max), "precondition"); + index_type aligned_start = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].aligned_index(start_region_idx); + index_type aligned_end = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].aligned_index(end_region_idx); + index_type alignment = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].alignment(); while (aligned_start <= aligned_end) { dump_bitmap_row(aligned_start); aligned_start += alignment; } } -void ShenandoahRegionPartitions::dump_bitmap_row(idx_t region_idx) const { - assert((region_idx >= 0) && (region_idx < (idx_t) _max), "precondition"); - idx_t aligned_idx = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].aligned_index(region_idx); +void ShenandoahRegionPartitions::dump_bitmap_row(index_type region_idx) const { + assert((region_idx >= 0) && (region_idx < (index_type) _max), "precondition"); + index_type aligned_idx = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].aligned_index(region_idx); uintx mutator_bits = _membership[int(ShenandoahFreeSetPartitionId::Mutator)].bits_at(aligned_idx); uintx collector_bits = _membership[int(ShenandoahFreeSetPartitionId::Collector)].bits_at(aligned_idx); uintx old_collector_bits = _membership[int(ShenandoahFreeSetPartitionId::OldCollector)].bits_at(aligned_idx); @@ -167,9 +169,27 @@ ShenandoahRegionPartitions::ShenandoahRegionPartitions(size_t max_regions, Shena _free_set(free_set), _membership{ ShenandoahSimpleBitMap(max_regions), ShenandoahSimpleBitMap(max_regions) , ShenandoahSimpleBitMap(max_regions) } { + initialize_old_collector(); make_all_regions_unavailable(); } +void ShenandoahFreeSet::prepare_to_promote_in_place(size_t idx, size_t bytes) { + shenandoah_assert_heaplocked(); + size_t min_remnant_size = PLAB::min_size() * HeapWordSize; + ShenandoahFreeSetPartitionId p = _partitions.membership(idx); + if (bytes >= min_remnant_size) { + assert((p == ShenandoahFreeSetPartitionId::Mutator) || (p == ShenandoahFreeSetPartitionId::Collector), + "PIP region must be associated with young"); + _partitions.increase_used(p, bytes); + _partitions.decrease_region_counts(p, 1); + _partitions.raw_clear_membership(idx, p); + recompute_total_young_used(); + recompute_total_global_used(); + } else { + assert(p == ShenandoahFreeSetPartitionId::NotFree, "We did not fill this region and do not need to adjust used"); + } +} + inline bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const { return r->is_empty() || (r->is_trash() && !_heap->is_concurrent_weak_root_in_progress()); } @@ -197,9 +217,56 @@ inline bool ShenandoahFreeSet::has_alloc_capacity(ShenandoahHeapRegion *r) const return alloc_capacity(r) > 0; } -inline idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId which_partition) const { +// This is used for unit testing. Do not use in production code. +void ShenandoahFreeSet::resize_old_collector_capacity(size_t regions) { + shenandoah_assert_heaplocked(); + size_t original_old_regions = _partitions.get_total_region_counts(ShenandoahFreeSetPartitionId::OldCollector); + size_t unaffiliated_mutator_regions = _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator); + size_t unaffiliated_collector_regions = _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Collector); + size_t unaffiliated_old_collector_regions = _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector); + if (regions > original_old_regions) { + size_t regions_to_transfer = regions - original_old_regions; + if (regions_to_transfer <= unaffiliated_mutator_regions + unaffiliated_collector_regions) { + size_t regions_from_mutator = + (regions_to_transfer > unaffiliated_mutator_regions)? unaffiliated_mutator_regions: regions_to_transfer; + regions_to_transfer -= regions_from_mutator; + size_t regions_from_collector = regions_to_transfer; + if (regions_from_mutator > 0) { + transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId::Mutator, ShenandoahFreeSetPartitionId::OldCollector, + regions_from_mutator); + } + if (regions_from_collector > 0) { + transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId::Collector, ShenandoahFreeSetPartitionId::OldCollector, + regions_from_mutator); + } + } else { + fatal("Could not resize old for unit test"); + } + } else if (regions < original_old_regions) { + size_t regions_to_transfer = original_old_regions - regions; + if (regions_to_transfer <= unaffiliated_old_collector_regions) { + transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId::OldCollector, ShenandoahFreeSetPartitionId::Mutator, + regions_to_transfer); + } else { + fatal("Could not resize old for unit test"); + } + } + // else, old generation is already appropriately sized +} + +void ShenandoahFreeSet::reset_bytes_allocated_since_gc_start() { + shenandoah_assert_heaplocked(); + _mutator_bytes_allocated_since_gc_start = 0; +} + +void ShenandoahFreeSet::increase_bytes_allocated(size_t bytes) { + shenandoah_assert_heaplocked(); + _mutator_bytes_allocated_since_gc_start += bytes; +} + +inline index_type ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId which_partition) const { assert (which_partition < NumPartitions, "selected free partition must be valid"); - idx_t idx = _leftmosts[int(which_partition)]; + index_type idx = _leftmosts[int(which_partition)]; if (idx >= _max) { return _max; } else { @@ -210,15 +277,21 @@ inline idx_t ShenandoahRegionPartitions::leftmost(ShenandoahFreeSetPartitionId w } } -inline idx_t ShenandoahRegionPartitions::rightmost(ShenandoahFreeSetPartitionId which_partition) const { +inline index_type ShenandoahRegionPartitions::rightmost(ShenandoahFreeSetPartitionId which_partition) const { assert (which_partition < NumPartitions, "selected free partition must be valid"); - idx_t idx = _rightmosts[int(which_partition)]; + index_type idx = _rightmosts[int(which_partition)]; // Cannot assert that membership[which_partition.is_set(idx) because this helper method may be used // to query the original value of leftmost when leftmost must be adjusted because the interval representing // which_partition is shrinking after the region that used to be leftmost is retired. return idx; } +void ShenandoahRegionPartitions::initialize_old_collector() { + _total_region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = 0; + _region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = 0; + _empty_region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = 0; +} + void ShenandoahRegionPartitions::make_all_regions_unavailable() { shenandoah_assert_heaplocked(); for (size_t partition_id = 0; partition_id < IntNumPartitions; partition_id++) { @@ -229,14 +302,21 @@ void ShenandoahRegionPartitions::make_all_regions_unavailable() { _rightmosts_empty[partition_id] = -1;; _capacity[partition_id] = 0; _used[partition_id] = 0; + _humongous_waste[partition_id] = 0; _available[partition_id] = FreeSetUnderConstruction; } + _total_region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = + _total_region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; _region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = _region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; + _empty_region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = + _empty_region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; } -void ShenandoahRegionPartitions::establish_mutator_intervals(idx_t mutator_leftmost, idx_t mutator_rightmost, - idx_t mutator_leftmost_empty, idx_t mutator_rightmost_empty, - size_t mutator_region_count, size_t mutator_used) { +void ShenandoahRegionPartitions::establish_mutator_intervals(index_type mutator_leftmost, index_type mutator_rightmost, + index_type mutator_leftmost_empty, index_type mutator_rightmost_empty, + size_t total_mutator_regions, size_t empty_mutator_regions, + size_t mutator_region_count, size_t mutator_used, + size_t mutator_humongous_waste_bytes) { shenandoah_assert_heaplocked(); _leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_leftmost; @@ -246,10 +326,14 @@ void ShenandoahRegionPartitions::establish_mutator_intervals(idx_t mutator_leftm _region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_region_count; _used[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_used; - _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_region_count * _region_size_bytes; + _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] = total_mutator_regions * _region_size_bytes; + _humongous_waste[int(ShenandoahFreeSetPartitionId::Mutator)] = mutator_humongous_waste_bytes; _available[int(ShenandoahFreeSetPartitionId::Mutator)] = _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] - _used[int(ShenandoahFreeSetPartitionId::Mutator)]; + _total_region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = total_mutator_regions; + _empty_region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] = empty_mutator_regions; + _leftmosts[int(ShenandoahFreeSetPartitionId::Collector)] = _max; _rightmosts[int(ShenandoahFreeSetPartitionId::Collector)] = -1; _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)] = _max; @@ -258,13 +342,21 @@ void ShenandoahRegionPartitions::establish_mutator_intervals(idx_t mutator_leftm _region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; _used[int(ShenandoahFreeSetPartitionId::Collector)] = 0; _capacity[int(ShenandoahFreeSetPartitionId::Collector)] = 0; + _humongous_waste[int(ShenandoahFreeSetPartitionId::Collector)] = 0; _available[int(ShenandoahFreeSetPartitionId::Collector)] = 0; + + _total_region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; + _empty_region_counts[int(ShenandoahFreeSetPartitionId::Collector)] = 0; } -void ShenandoahRegionPartitions::establish_old_collector_intervals(idx_t old_collector_leftmost, idx_t old_collector_rightmost, - idx_t old_collector_leftmost_empty, - idx_t old_collector_rightmost_empty, - size_t old_collector_region_count, size_t old_collector_used) { +void ShenandoahRegionPartitions::establish_old_collector_intervals(index_type old_collector_leftmost, + index_type old_collector_rightmost, + index_type old_collector_leftmost_empty, + index_type old_collector_rightmost_empty, + size_t total_old_collector_region_count, + size_t old_collector_empty, size_t old_collector_regions, + size_t old_collector_used, + size_t old_collector_humongous_waste_bytes) { shenandoah_assert_heaplocked(); _leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_leftmost; @@ -272,11 +364,15 @@ void ShenandoahRegionPartitions::establish_old_collector_intervals(idx_t old_col _leftmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_leftmost_empty; _rightmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_rightmost_empty; - _region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_region_count; + _region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_regions; _used[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_used; - _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_region_count * _region_size_bytes; + _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] = total_old_collector_region_count * _region_size_bytes; + _humongous_waste[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_humongous_waste_bytes; _available[int(ShenandoahFreeSetPartitionId::OldCollector)] = _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] - _used[int(ShenandoahFreeSetPartitionId::OldCollector)]; + + _total_region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = total_old_collector_region_count; + _empty_region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)] = old_collector_empty; } void ShenandoahRegionPartitions::increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { @@ -290,15 +386,119 @@ void ShenandoahRegionPartitions::increase_used(ShenandoahFreeSetPartitionId whic _used[int(which_partition)], _capacity[int(which_partition)], bytes); } -inline void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either_boundary( - ShenandoahFreeSetPartitionId partition, idx_t low_idx, idx_t high_idx) { +size_t ShenandoahRegionPartitions::get_used(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "Partition must be valid"); + return _used[int(which_partition)];; +} + +void ShenandoahRegionPartitions::increase_humongous_waste(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + _humongous_waste[int(which_partition)] += bytes; +} + +size_t ShenandoahRegionPartitions::get_humongous_waste(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "Partition must be valid"); + return _humongous_waste[int(which_partition)];; +} + +void ShenandoahRegionPartitions::set_capacity_of(ShenandoahFreeSetPartitionId which_partition, size_t value) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "selected free set must be valid"); + _capacity[int(which_partition)] = value; + _available[int(which_partition)] = value - _used[int(which_partition)]; +} + + +void ShenandoahRegionPartitions::increase_capacity(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + _capacity[int(which_partition)] += bytes; + _available[int(which_partition)] += bytes; +} + +void ShenandoahRegionPartitions::decrease_capacity(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + assert(_capacity[int(which_partition)] >= bytes, "Cannot remove more capacity bytes than are present"); + _capacity[int(which_partition)] -= bytes; + _available[int(which_partition)] -= bytes; +} + +size_t ShenandoahRegionPartitions::get_capacity(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "Partition must be valid"); + return _capacity[int(which_partition)];; +} + +void ShenandoahRegionPartitions::increase_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + _available[int(which_partition)] += bytes; +} + +void ShenandoahRegionPartitions::decrease_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + assert(_available[int(which_partition)] >= bytes, "Cannot remove more available bytes than are present"); + _available[int(which_partition)] -= bytes; +} + +size_t ShenandoahRegionPartitions::get_available(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "Partition must be valid"); + return _available[int(which_partition)];; +} + +void ShenandoahRegionPartitions::increase_total_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + _total_region_counts[int(which_partition)] += regions; +} + +void ShenandoahRegionPartitions::decrease_total_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + assert(_total_region_counts[int(which_partition)] >= regions, "Cannot remove more regions than are present"); + _total_region_counts[int(which_partition)] -= regions; +} + +void ShenandoahRegionPartitions::increase_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + _region_counts[int(which_partition)] += regions; +} + +void ShenandoahRegionPartitions::decrease_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + assert(_region_counts[int(which_partition)] >= regions, "Cannot remove more regions than are present"); + _region_counts[int(which_partition)] -= regions; +} + +void ShenandoahRegionPartitions::increase_empty_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + _empty_region_counts[int(which_partition)] += regions; +} + +void ShenandoahRegionPartitions::decrease_empty_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions) { + assert(_empty_region_counts[int(which_partition)] >= regions, "Cannot remove more regions than are present"); + _empty_region_counts[int(which_partition)] -= regions; +} + +void ShenandoahRegionPartitions::one_region_is_no_longer_empty(ShenandoahFreeSetPartitionId partition) { + decrease_empty_region_counts(partition, (size_t) 1); +} + +// All members of partition between low_idx and high_idx inclusive have been removed. +void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either_boundary( + ShenandoahFreeSetPartitionId partition, index_type low_idx, index_type high_idx, size_t num_regions) { assert((low_idx <= high_idx) && (low_idx >= 0) && (high_idx < _max), "Range must span legal index values"); + size_t span = high_idx + 1 - low_idx; + bool regions_are_contiguous = (span == num_regions); if (low_idx == leftmost(partition)) { assert (!_membership[int(partition)].is_set(low_idx), "Do not shrink interval if region not removed"); if (high_idx + 1 == _max) { - _leftmosts[int(partition)] = _max; + if (regions_are_contiguous) { + _leftmosts[int(partition)] = _max; + } else { + _leftmosts[int(partition)] = find_index_of_next_available_region(partition, low_idx + 1); + } } else { - _leftmosts[int(partition)] = find_index_of_next_available_region(partition, high_idx + 1); + if (regions_are_contiguous) { + _leftmosts[int(partition)] = find_index_of_next_available_region(partition, high_idx + 1); + } else { + _leftmosts[int(partition)] = find_index_of_next_available_region(partition, low_idx + 1); + } } if (_leftmosts_empty[int(partition)] < _leftmosts[int(partition)]) { // This gets us closer to where we need to be; we'll scan further when leftmosts_empty is requested. @@ -308,9 +508,17 @@ inline void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either if (high_idx == _rightmosts[int(partition)]) { assert (!_membership[int(partition)].is_set(high_idx), "Do not shrink interval if region not removed"); if (low_idx == 0) { - _rightmosts[int(partition)] = -1; + if (regions_are_contiguous) { + _rightmosts[int(partition)] = -1; + } else { + _rightmosts[int(partition)] = find_index_of_previous_available_region(partition, high_idx - 1); + } } else { - _rightmosts[int(partition)] = find_index_of_previous_available_region(partition, low_idx - 1); + if (regions_are_contiguous) { + _rightmosts[int(partition)] = find_index_of_previous_available_region(partition, low_idx - 1); + } else { + _rightmosts[int(partition)] = find_index_of_previous_available_region(partition, high_idx - 1); + } } if (_rightmosts_empty[int(partition)] > _rightmosts[int(partition)]) { // This gets us closer to where we need to be; we'll scan further when rightmosts_empty is requested. @@ -325,12 +533,55 @@ inline void ShenandoahRegionPartitions::shrink_interval_if_range_modifies_either } } -inline void ShenandoahRegionPartitions::shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, idx_t idx) { - shrink_interval_if_range_modifies_either_boundary(partition, idx, idx); +void ShenandoahRegionPartitions::establish_interval(ShenandoahFreeSetPartitionId partition, index_type low_idx, + index_type high_idx, index_type low_empty_idx, index_type high_empty_idx) { +#ifdef ASSERT + assert (partition < NumPartitions, "invalid partition"); + if (low_idx != max()) { + assert((low_idx <= high_idx) && (low_idx >= 0) && (high_idx < _max), "Range must span legal index values"); + assert (in_free_set(partition, low_idx), "Must be in partition of established interval"); + assert (in_free_set(partition, high_idx), "Must be in partition of established interval"); + } + if (low_empty_idx != max()) { + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(low_empty_idx); + assert (in_free_set(partition, low_empty_idx) && (r->is_trash() || r->free() == _region_size_bytes), + "Must be empty and in partition of established interval"); + r = ShenandoahHeap::heap()->get_region(high_empty_idx); + assert (in_free_set(partition, high_empty_idx), "Must be in partition of established interval"); + } +#endif + + _leftmosts[int(partition)] = low_idx; + _rightmosts[int(partition)] = high_idx; + _leftmosts_empty[int(partition)] = low_empty_idx; + _rightmosts_empty[int(partition)] = high_empty_idx; +} + +inline void ShenandoahRegionPartitions::shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, + index_type idx) { + shrink_interval_if_range_modifies_either_boundary(partition, idx, idx, 1); } -inline void ShenandoahRegionPartitions::expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, - idx_t idx, size_t region_available) { +// Some members of partition between low_idx and high_idx inclusive have been added. +void ShenandoahRegionPartitions:: +expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, index_type low_idx, index_type high_idx, + index_type low_empty_idx, index_type high_empty_idx) { + if (_leftmosts[int(partition)] > low_idx) { + _leftmosts[int(partition)] = low_idx; + } + if (_rightmosts[int(partition)] < high_idx) { + _rightmosts[int(partition)] = high_idx; + } + if (_leftmosts_empty[int(partition)] > low_empty_idx) { + _leftmosts_empty[int(partition)] = low_empty_idx; + } + if (_rightmosts_empty[int(partition)] < high_empty_idx) { + _rightmosts_empty[int(partition)] = high_empty_idx; + } +} + +void ShenandoahRegionPartitions::expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, + index_type idx, size_t region_available) { if (_leftmosts[int(partition)] > idx) { _leftmosts[int(partition)] = idx; } @@ -348,23 +599,31 @@ inline void ShenandoahRegionPartitions::expand_interval_if_boundary_modified(She } void ShenandoahRegionPartitions::retire_range_from_partition( - ShenandoahFreeSetPartitionId partition, idx_t low_idx, idx_t high_idx) { + ShenandoahFreeSetPartitionId partition, index_type low_idx, index_type high_idx) { // Note: we may remove from free partition even if region is not entirely full, such as when available < PLAB::min_size() assert ((low_idx < _max) && (high_idx < _max), "Both indices are sane: %zu and %zu < %zu", low_idx, high_idx, _max); assert (partition < NumPartitions, "Cannot remove from free partitions if not already free"); - for (idx_t idx = low_idx; idx <= high_idx; idx++) { + for (index_type idx = low_idx; idx <= high_idx; idx++) { +#ifdef ASSERT + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(idx); assert (in_free_set(partition, idx), "Must be in partition to remove from partition"); + assert(r->is_empty() || r->is_trash(), "Region must be empty or trash"); +#endif _membership[int(partition)].clear_bit(idx); } - _region_counts[int(partition)] -= high_idx + 1 - low_idx; - shrink_interval_if_range_modifies_either_boundary(partition, low_idx, high_idx); + size_t num_regions = high_idx + 1 - low_idx; + decrease_region_counts(partition, num_regions); + decrease_empty_region_counts(partition, num_regions); + shrink_interval_if_range_modifies_either_boundary(partition, low_idx, high_idx, num_regions); } -void ShenandoahRegionPartitions::retire_from_partition(ShenandoahFreeSetPartitionId partition, idx_t idx, size_t used_bytes) { +size_t ShenandoahRegionPartitions::retire_from_partition(ShenandoahFreeSetPartitionId partition, + index_type idx, size_t used_bytes) { + size_t waste_bytes = 0; // Note: we may remove from free partition even if region is not entirely full, such as when available < PLAB::min_size() assert (idx < _max, "index is sane: %zu < %zu", idx, _max); assert (partition < NumPartitions, "Cannot remove from free partitions if not already free"); @@ -372,14 +631,29 @@ void ShenandoahRegionPartitions::retire_from_partition(ShenandoahFreeSetPartitio if (used_bytes < _region_size_bytes) { // Count the alignment pad remnant of memory as used when we retire this region - increase_used(partition, _region_size_bytes - used_bytes); + size_t fill_padding = _region_size_bytes - used_bytes; + waste_bytes = fill_padding; + increase_used(partition, fill_padding); } _membership[int(partition)].clear_bit(idx); + decrease_region_counts(partition, 1); shrink_interval_if_boundary_modified(partition, idx); - _region_counts[int(partition)]--; + + // This region is fully used, whether or not top() equals end(). It + // is retired and no more memory will be allocated from within it. + + return waste_bytes; } -void ShenandoahRegionPartitions::make_free(idx_t idx, ShenandoahFreeSetPartitionId which_partition, size_t available) { +void ShenandoahRegionPartitions::unretire_to_partition(ShenandoahHeapRegion* r, ShenandoahFreeSetPartitionId which_partition) { + shenandoah_assert_heaplocked(); + make_free(r->index(), which_partition, r->free()); +} + + +// The caller is responsible for increasing capacity and available and used in which_partition, and decreasing the +// same quantities for the original partition +void ShenandoahRegionPartitions::make_free(index_type idx, ShenandoahFreeSetPartitionId which_partition, size_t available) { shenandoah_assert_heaplocked(); assert (idx < _max, "index is sane: %zu < %zu", idx, _max); assert (membership(idx) == ShenandoahFreeSetPartitionId::NotFree, "Cannot make free if already free"); @@ -387,11 +661,7 @@ void ShenandoahRegionPartitions::make_free(idx_t idx, ShenandoahFreeSetPartition assert (available <= _region_size_bytes, "Available cannot exceed region size"); _membership[int(which_partition)].set_bit(idx); - _capacity[int(which_partition)] += _region_size_bytes; - _used[int(which_partition)] += _region_size_bytes - available; - _available[int(which_partition)] += available; expand_interval_if_boundary_modified(which_partition, idx, available); - _region_counts[int(which_partition)]++; } bool ShenandoahRegionPartitions::is_mutator_partition(ShenandoahFreeSetPartitionId p) { @@ -410,9 +680,10 @@ bool ShenandoahRegionPartitions::available_implies_empty(size_t available_in_reg return (available_in_region == _region_size_bytes); } - -void ShenandoahRegionPartitions::move_from_partition_to_partition(idx_t idx, ShenandoahFreeSetPartitionId orig_partition, - ShenandoahFreeSetPartitionId new_partition, size_t available) { +// Do not adjust capacities, available, or used. Return used delta. +size_t ShenandoahRegionPartitions:: +move_from_partition_to_partition_with_deferred_accounting(index_type idx, ShenandoahFreeSetPartitionId orig_partition, + ShenandoahFreeSetPartitionId new_partition, size_t available) { ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(idx); shenandoah_assert_heaplocked(); assert (idx < _max, "index is sane: %zu < %zu", idx, _max); @@ -450,39 +721,39 @@ void ShenandoahRegionPartitions::move_from_partition_to_partition(idx_t idx, She _membership[int(orig_partition)].clear_bit(idx); _membership[int(new_partition)].set_bit(idx); + return used; +} +void ShenandoahRegionPartitions::move_from_partition_to_partition(index_type idx, ShenandoahFreeSetPartitionId orig_partition, + ShenandoahFreeSetPartitionId new_partition, size_t available) { + size_t used = move_from_partition_to_partition_with_deferred_accounting(idx, orig_partition, new_partition, available); + + decrease_used(orig_partition, used); + _total_region_counts[int(orig_partition)]--; + _region_counts[int(orig_partition)]--; _capacity[int(orig_partition)] -= _region_size_bytes; - _used[int(orig_partition)] -= used; - _available[int(orig_partition)] -= available; + _available[int(orig_partition)] -= _region_size_bytes; shrink_interval_if_boundary_modified(orig_partition, idx); - _capacity[int(new_partition)] += _region_size_bytes;; - _used[int(new_partition)] += used; - _available[int(new_partition)] += available; + _capacity[int(new_partition)] += _region_size_bytes; + _available[int(new_partition)] += _region_size_bytes; + _total_region_counts[int(new_partition)]++; + _region_counts[int(new_partition)]++; + increase_used(new_partition, used); expand_interval_if_boundary_modified(new_partition, idx, available); - _region_counts[int(orig_partition)]--; - _region_counts[int(new_partition)]++; + if (available == _region_size_bytes) { + _empty_region_counts[int(orig_partition)]--; + _empty_region_counts[int(new_partition)]++; + } } -const char* ShenandoahRegionPartitions::partition_membership_name(idx_t idx) const { +const char* ShenandoahRegionPartitions::partition_membership_name(index_type idx) const { return partition_name(membership(idx)); } -inline ShenandoahFreeSetPartitionId ShenandoahRegionPartitions::membership(idx_t idx) const { - assert (idx < _max, "index is sane: %zu < %zu", idx, _max); - ShenandoahFreeSetPartitionId result = ShenandoahFreeSetPartitionId::NotFree; - for (uint partition_id = 0; partition_id < UIntNumPartitions; partition_id++) { - if (_membership[partition_id].is_set(idx)) { - assert(result == ShenandoahFreeSetPartitionId::NotFree, "Region should reside in only one partition"); - result = (ShenandoahFreeSetPartitionId) partition_id; - } - } - return result; -} - #ifdef ASSERT -inline bool ShenandoahRegionPartitions::partition_id_matches(idx_t idx, ShenandoahFreeSetPartitionId test_partition) const { +inline bool ShenandoahRegionPartitions::partition_id_matches(index_type idx, ShenandoahFreeSetPartitionId test_partition) const { assert (idx < _max, "index is sane: %zu < %zu", idx, _max); assert (test_partition < ShenandoahFreeSetPartitionId::NotFree, "must be a valid partition"); @@ -495,15 +766,15 @@ inline bool ShenandoahRegionPartitions::is_empty(ShenandoahFreeSetPartitionId wh return (leftmost(which_partition) > rightmost(which_partition)); } -inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_region( - ShenandoahFreeSetPartitionId which_partition, idx_t start_index) const { - idx_t rightmost_idx = rightmost(which_partition); - idx_t leftmost_idx = leftmost(which_partition); +inline index_type ShenandoahRegionPartitions::find_index_of_next_available_region( + ShenandoahFreeSetPartitionId which_partition, index_type start_index) const { + index_type rightmost_idx = rightmost(which_partition); + index_type leftmost_idx = leftmost(which_partition); if ((rightmost_idx < leftmost_idx) || (start_index > rightmost_idx)) return _max; if (start_index < leftmost_idx) { start_index = leftmost_idx; } - idx_t result = _membership[int(which_partition)].find_first_set_bit(start_index, rightmost_idx + 1); + index_type result = _membership[int(which_partition)].find_first_set_bit(start_index, rightmost_idx + 1); if (result > rightmost_idx) { result = _max; } @@ -511,16 +782,16 @@ inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_region( return result; } -inline idx_t ShenandoahRegionPartitions::find_index_of_previous_available_region( - ShenandoahFreeSetPartitionId which_partition, idx_t last_index) const { - idx_t rightmost_idx = rightmost(which_partition); - idx_t leftmost_idx = leftmost(which_partition); +inline index_type ShenandoahRegionPartitions::find_index_of_previous_available_region( + ShenandoahFreeSetPartitionId which_partition, index_type last_index) const { + index_type rightmost_idx = rightmost(which_partition); + index_type leftmost_idx = leftmost(which_partition); // if (leftmost_idx == max) then (last_index < leftmost_idx) if (last_index < leftmost_idx) return -1; if (last_index > rightmost_idx) { last_index = rightmost_idx; } - idx_t result = _membership[int(which_partition)].find_last_set_bit(-1, last_index); + index_type result = _membership[int(which_partition)].find_last_set_bit(-1, last_index); if (result < leftmost_idx) { result = -1; } @@ -528,12 +799,13 @@ inline idx_t ShenandoahRegionPartitions::find_index_of_previous_available_region return result; } -inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_cluster_of_regions( - ShenandoahFreeSetPartitionId which_partition, idx_t start_index, size_t cluster_size) const { - idx_t rightmost_idx = rightmost(which_partition); - idx_t leftmost_idx = leftmost(which_partition); +inline index_type ShenandoahRegionPartitions::find_index_of_next_available_cluster_of_regions( + ShenandoahFreeSetPartitionId which_partition, index_type start_index, size_t cluster_size) const { + index_type rightmost_idx = rightmost(which_partition); + index_type leftmost_idx = leftmost(which_partition); if ((rightmost_idx < leftmost_idx) || (start_index > rightmost_idx)) return _max; - idx_t result = _membership[int(which_partition)].find_first_consecutive_set_bits(start_index, rightmost_idx + 1, cluster_size); + index_type result = + _membership[int(which_partition)].find_first_consecutive_set_bits(start_index, rightmost_idx + 1, cluster_size); if (result > rightmost_idx) { result = _max; } @@ -541,12 +813,12 @@ inline idx_t ShenandoahRegionPartitions::find_index_of_next_available_cluster_of return result; } -inline idx_t ShenandoahRegionPartitions::find_index_of_previous_available_cluster_of_regions( - ShenandoahFreeSetPartitionId which_partition, idx_t last_index, size_t cluster_size) const { - idx_t leftmost_idx = leftmost(which_partition); +inline index_type ShenandoahRegionPartitions::find_index_of_previous_available_cluster_of_regions( + ShenandoahFreeSetPartitionId which_partition, index_type last_index, size_t cluster_size) const { + index_type leftmost_idx = leftmost(which_partition); // if (leftmost_idx == max) then (last_index < leftmost_idx) if (last_index < leftmost_idx) return -1; - idx_t result = _membership[int(which_partition)].find_last_consecutive_set_bits(leftmost_idx - 1, last_index, cluster_size); + index_type result = _membership[int(which_partition)].find_last_consecutive_set_bits(leftmost_idx - 1, last_index, cluster_size); if (result <= leftmost_idx) { result = -1; } @@ -554,13 +826,13 @@ inline idx_t ShenandoahRegionPartitions::find_index_of_previous_available_cluste return result; } -idx_t ShenandoahRegionPartitions::leftmost_empty(ShenandoahFreeSetPartitionId which_partition) { +index_type ShenandoahRegionPartitions::leftmost_empty(ShenandoahFreeSetPartitionId which_partition) { assert (which_partition < NumPartitions, "selected free partition must be valid"); - idx_t max_regions = _max; + index_type max_regions = _max; if (_leftmosts_empty[int(which_partition)] == _max) { return _max; } - for (idx_t idx = find_index_of_next_available_region(which_partition, _leftmosts_empty[int(which_partition)]); + for (index_type idx = find_index_of_next_available_region(which_partition, _leftmosts_empty[int(which_partition)]); idx < max_regions; ) { assert(in_free_set(which_partition, idx), "Boundaries or find_last_set_bit failed: %zd", idx); if (_free_set->alloc_capacity(idx) == _region_size_bytes) { @@ -574,12 +846,12 @@ idx_t ShenandoahRegionPartitions::leftmost_empty(ShenandoahFreeSetPartitionId wh return _max; } -idx_t ShenandoahRegionPartitions::rightmost_empty(ShenandoahFreeSetPartitionId which_partition) { +index_type ShenandoahRegionPartitions::rightmost_empty(ShenandoahFreeSetPartitionId which_partition) { assert (which_partition < NumPartitions, "selected free partition must be valid"); if (_rightmosts_empty[int(which_partition)] < 0) { return -1; } - for (idx_t idx = find_index_of_previous_available_region(which_partition, _rightmosts_empty[int(which_partition)]); + for (index_type idx = find_index_of_previous_available_region(which_partition, _rightmosts_empty[int(which_partition)]); idx >= 0; ) { assert(in_free_set(which_partition, idx), "Boundaries or find_last_set_bit failed: %zd", idx); if (_free_set->alloc_capacity(idx) == _region_size_bytes) { @@ -595,33 +867,88 @@ idx_t ShenandoahRegionPartitions::rightmost_empty(ShenandoahFreeSetPartitionId w #ifdef ASSERT -void ShenandoahRegionPartitions::assert_bounds() { +void ShenandoahRegionPartitions::assert_bounds(bool validate_totals) { + + size_t capacities[UIntNumPartitions]; + size_t used[UIntNumPartitions]; + size_t regions[UIntNumPartitions]; + size_t humongous_waste[UIntNumPartitions]; + + // We don't know whether young retired regions belonged to Mutator or Collector before they were retired. + // We just tally the total, and divide it to make matches work if possible. + size_t young_retired_regions = 0; + size_t young_retired_used = 0; + size_t young_retired_capacity = 0; + size_t young_humongous_waste = 0; - idx_t leftmosts[UIntNumPartitions]; - idx_t rightmosts[UIntNumPartitions]; - idx_t empty_leftmosts[UIntNumPartitions]; - idx_t empty_rightmosts[UIntNumPartitions]; + index_type leftmosts[UIntNumPartitions]; + index_type rightmosts[UIntNumPartitions]; + index_type empty_leftmosts[UIntNumPartitions]; + index_type empty_rightmosts[UIntNumPartitions]; for (uint i = 0; i < UIntNumPartitions; i++) { leftmosts[i] = _max; empty_leftmosts[i] = _max; rightmosts[i] = -1; empty_rightmosts[i] = -1; + capacities[i] = 0; + used[i] = 0; + regions[i] = 0; + humongous_waste[i] = 0; } - for (idx_t i = 0; i < _max; i++) { + for (index_type i = 0; i < _max; i++) { ShenandoahFreeSetPartitionId partition = membership(i); + size_t capacity = _free_set->alloc_capacity(i); switch (partition) { case ShenandoahFreeSetPartitionId::NotFree: - break; + { + assert(!validate_totals || (capacity != _region_size_bytes), "Should not be retired if empty"); + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->get_region(i); + if (r->is_humongous()) { + if (r->is_old()) { + regions[int(ShenandoahFreeSetPartitionId::OldCollector)]++; + used[int(ShenandoahFreeSetPartitionId::OldCollector)] += _region_size_bytes; + capacities[int(ShenandoahFreeSetPartitionId::OldCollector)] += _region_size_bytes; + humongous_waste[int(ShenandoahFreeSetPartitionId::OldCollector)] += capacity; + } else { + assert(r->is_young(), "Must be young if not old"); + young_retired_regions++; + // Count entire region as used even if there is some waste. + young_retired_used += _region_size_bytes; + young_retired_capacity += _region_size_bytes; + young_humongous_waste += capacity; + } + } else { + assert(r->is_cset() || (capacity < PLAB::min_size() * HeapWordSize), + "Expect retired remnant size to be smaller than min plab size"); + // This region has been retired already or it is in the cset. In either case, we set capacity to zero + // so that the entire region will be counted as used. We count young cset regions as "retired". + capacity = 0; + if (r->is_old()) { + regions[int(ShenandoahFreeSetPartitionId::OldCollector)]++; + used[int(ShenandoahFreeSetPartitionId::OldCollector)] += _region_size_bytes - capacity; + capacities[int(ShenandoahFreeSetPartitionId::OldCollector)] += _region_size_bytes; + } else { + assert(r->is_young(), "Must be young if not old"); + young_retired_regions++; + young_retired_used += _region_size_bytes - capacity; + young_retired_capacity += _region_size_bytes; + } + } + } + break; case ShenandoahFreeSetPartitionId::Mutator: case ShenandoahFreeSetPartitionId::Collector: case ShenandoahFreeSetPartitionId::OldCollector: { - size_t capacity = _free_set->alloc_capacity(i); - bool is_empty = (capacity == _region_size_bytes); assert(capacity > 0, "free regions must have allocation capacity"); + bool is_empty = (capacity == _region_size_bytes); + regions[int(partition)]++; + used[int(partition)] += _region_size_bytes - capacity; + capacities[int(partition)] += _region_size_bytes; + if (i < leftmosts[int(partition)]) { leftmosts[int(partition)] = i; } @@ -657,22 +984,22 @@ void ShenandoahRegionPartitions::assert_bounds() { // If Mutator partition is empty, leftmosts will both equal max, rightmosts will both equal zero. // Likewise for empty region partitions. - idx_t beg_off = leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; - idx_t end_off = rightmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; + index_type beg_off = leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; + index_type end_off = rightmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; assert (beg_off >= leftmost(ShenandoahFreeSetPartitionId::Mutator), - "free regions before the leftmost: %zd, bound %zd", + "Mutator free regions before the leftmost: %zd, bound %zd", beg_off, leftmost(ShenandoahFreeSetPartitionId::Mutator)); assert (end_off <= rightmost(ShenandoahFreeSetPartitionId::Mutator), - "free regions past the rightmost: %zd, bound %zd", + "Mutator free regions past the rightmost: %zd, bound %zd", end_off, rightmost(ShenandoahFreeSetPartitionId::Mutator)); beg_off = empty_leftmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; end_off = empty_rightmosts[int(ShenandoahFreeSetPartitionId::Mutator)]; assert (beg_off >= leftmost_empty(ShenandoahFreeSetPartitionId::Mutator), - "free empty regions before the leftmost: %zd, bound %zd", + "Mutator free empty regions before the leftmost: %zd, bound %zd", beg_off, leftmost_empty(ShenandoahFreeSetPartitionId::Mutator)); assert (end_off <= rightmost_empty(ShenandoahFreeSetPartitionId::Mutator), - "free empty regions past the rightmost: %zd, bound %zd", + "Mutator free empty regions past the rightmost: %zd, bound %zd", end_off, rightmost_empty(ShenandoahFreeSetPartitionId::Mutator)); // Performance invariants. Failing these would not break the free partition, but performance would suffer. @@ -683,87 +1010,218 @@ void ShenandoahRegionPartitions::assert_bounds() { assert (leftmost(ShenandoahFreeSetPartitionId::Collector) == _max || partition_id_matches(leftmost(ShenandoahFreeSetPartitionId::Collector), ShenandoahFreeSetPartitionId::Collector), - "leftmost region should be free: %zd", leftmost(ShenandoahFreeSetPartitionId::Collector)); + "Collector leftmost region should be free: %zd", leftmost(ShenandoahFreeSetPartitionId::Collector)); assert (leftmost(ShenandoahFreeSetPartitionId::Collector) == _max || partition_id_matches(rightmost(ShenandoahFreeSetPartitionId::Collector), ShenandoahFreeSetPartitionId::Collector), - "rightmost region should be free: %zd", rightmost(ShenandoahFreeSetPartitionId::Collector)); + "Collector rightmost region should be free: %zd", rightmost(ShenandoahFreeSetPartitionId::Collector)); // If Collector partition is empty, leftmosts will both equal max, rightmosts will both equal zero. // Likewise for empty region partitions. beg_off = leftmosts[int(ShenandoahFreeSetPartitionId::Collector)]; end_off = rightmosts[int(ShenandoahFreeSetPartitionId::Collector)]; assert (beg_off >= leftmost(ShenandoahFreeSetPartitionId::Collector), - "free regions before the leftmost: %zd, bound %zd", + "Collector free regions before the leftmost: %zd, bound %zd", beg_off, leftmost(ShenandoahFreeSetPartitionId::Collector)); assert (end_off <= rightmost(ShenandoahFreeSetPartitionId::Collector), - "free regions past the rightmost: %zd, bound %zd", + "Collector free regions past the rightmost: %zd, bound %zd", end_off, rightmost(ShenandoahFreeSetPartitionId::Collector)); beg_off = empty_leftmosts[int(ShenandoahFreeSetPartitionId::Collector)]; end_off = empty_rightmosts[int(ShenandoahFreeSetPartitionId::Collector)]; assert (beg_off >= _leftmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)], - "free empty regions before the leftmost: %zd, bound %zd", + "Collector free empty regions before the leftmost: %zd, bound %zd", beg_off, leftmost_empty(ShenandoahFreeSetPartitionId::Collector)); assert (end_off <= _rightmosts_empty[int(ShenandoahFreeSetPartitionId::Collector)], - "free empty regions past the rightmost: %zd, bound %zd", + "Collector free empty regions past the rightmost: %zd, bound %zd", end_off, rightmost_empty(ShenandoahFreeSetPartitionId::Collector)); // Performance invariants. Failing these would not break the free partition, but performance would suffer. - assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) <= _max, "leftmost in bounds: %zd < %zd", + assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) <= _max, "OldCollector leftmost in bounds: %zd < %zd", leftmost(ShenandoahFreeSetPartitionId::OldCollector), _max); - assert (rightmost(ShenandoahFreeSetPartitionId::OldCollector) < _max, "rightmost in bounds: %zd < %zd", + assert (rightmost(ShenandoahFreeSetPartitionId::OldCollector) < _max, "OldCollector rightmost in bounds: %zd < %zd", rightmost(ShenandoahFreeSetPartitionId::OldCollector), _max); assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) == _max || partition_id_matches(leftmost(ShenandoahFreeSetPartitionId::OldCollector), ShenandoahFreeSetPartitionId::OldCollector), - "leftmost region should be free: %zd", leftmost(ShenandoahFreeSetPartitionId::OldCollector)); + "OldCollector leftmost region should be free: %zd", leftmost(ShenandoahFreeSetPartitionId::OldCollector)); assert (leftmost(ShenandoahFreeSetPartitionId::OldCollector) == _max || partition_id_matches(rightmost(ShenandoahFreeSetPartitionId::OldCollector), ShenandoahFreeSetPartitionId::OldCollector), - "rightmost region should be free: %zd", rightmost(ShenandoahFreeSetPartitionId::OldCollector)); + "OldCollector rightmost region should be free: %zd", rightmost(ShenandoahFreeSetPartitionId::OldCollector)); // If OldCollector partition is empty, leftmosts will both equal max, rightmosts will both equal zero. // Likewise for empty region partitions. beg_off = leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]; end_off = rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]; assert (beg_off >= leftmost(ShenandoahFreeSetPartitionId::OldCollector), - "free regions before the leftmost: %zd, bound %zd", + "OldCollector free regions before the leftmost: %zd, bound %zd", beg_off, leftmost(ShenandoahFreeSetPartitionId::OldCollector)); assert (end_off <= rightmost(ShenandoahFreeSetPartitionId::OldCollector), - "free regions past the rightmost: %zd, bound %zd", + "OldCollector free regions past the rightmost: %zd, bound %zd", end_off, rightmost(ShenandoahFreeSetPartitionId::OldCollector)); beg_off = empty_leftmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]; end_off = empty_rightmosts[int(ShenandoahFreeSetPartitionId::OldCollector)]; assert (beg_off >= _leftmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)], - "free empty regions before the leftmost: %zd, bound %zd", + "OldCollector free empty regions before the leftmost: %zd, bound %zd", beg_off, leftmost_empty(ShenandoahFreeSetPartitionId::OldCollector)); assert (end_off <= _rightmosts_empty[int(ShenandoahFreeSetPartitionId::OldCollector)], - "free empty regions past the rightmost: %zd, bound %zd", + "OldCollector free empty regions past the rightmost: %zd, bound %zd", end_off, rightmost_empty(ShenandoahFreeSetPartitionId::OldCollector)); + + if (validate_totals) { + // young_retired_regions need to be added to either Mutator or Collector partitions, 100% used. + // Give enough of young_retired_regions, young_retired_capacity, young_retired_user + // to the Mutator partition to top it off so that it matches the running totals. + // + // Give any remnants to the Collector partition. After topping off the Collector partition, its values + // should also match running totals. + + assert(young_retired_regions * _region_size_bytes == young_retired_capacity, "sanity"); + assert(young_retired_capacity == young_retired_used, "sanity"); + + + assert(capacities[int(ShenandoahFreeSetPartitionId::OldCollector)] + == _capacity[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old collector capacities must match"); + assert(used[int(ShenandoahFreeSetPartitionId::OldCollector)] + == _used[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old collector used must match"); + assert(regions[int(ShenandoahFreeSetPartitionId::OldCollector)] + == _total_region_counts[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old collector regions must match"); + assert(_capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] + >= _used[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old Collector capacity must be >= used"); + assert(_available[int(ShenandoahFreeSetPartitionId::OldCollector)] == + (_capacity[int(ShenandoahFreeSetPartitionId::OldCollector)] - _used[int(ShenandoahFreeSetPartitionId::OldCollector)]), + "Old Collector available must equal capacity minus used"); + assert(_humongous_waste[int(ShenandoahFreeSetPartitionId::OldCollector)] == + humongous_waste[int(ShenandoahFreeSetPartitionId::OldCollector)], "Old Collector humongous waste must match"); + + assert(_capacity[int(ShenandoahFreeSetPartitionId::Mutator)] >= capacities[int(ShenandoahFreeSetPartitionId::Mutator)], + "Capacity total must be >= counted tally"); + size_t mutator_capacity_shortfall = + _capacity[int(ShenandoahFreeSetPartitionId::Mutator)] - capacities[int(ShenandoahFreeSetPartitionId::Mutator)]; + assert(mutator_capacity_shortfall <= young_retired_capacity, "sanity"); + capacities[int(ShenandoahFreeSetPartitionId::Mutator)] += mutator_capacity_shortfall; + young_retired_capacity -= mutator_capacity_shortfall; + capacities[int(ShenandoahFreeSetPartitionId::Collector)] += young_retired_capacity; + + + assert(_used[int(ShenandoahFreeSetPartitionId::Mutator)] >= used[int(ShenandoahFreeSetPartitionId::Mutator)], + "Used total must be >= counted tally"); + size_t mutator_used_shortfall = + _used[int(ShenandoahFreeSetPartitionId::Mutator)] - used[int(ShenandoahFreeSetPartitionId::Mutator)]; + assert(mutator_used_shortfall <= young_retired_used, "sanity"); + used[int(ShenandoahFreeSetPartitionId::Mutator)] += mutator_used_shortfall; + young_retired_used -= mutator_used_shortfall; + used[int(ShenandoahFreeSetPartitionId::Collector)] += young_retired_used; + + assert(_total_region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] >= regions[int(ShenandoahFreeSetPartitionId::Mutator)], + "Region total must be >= counted tally"); + size_t mutator_regions_shortfall = + _total_region_counts[int(ShenandoahFreeSetPartitionId::Mutator)] - regions[int(ShenandoahFreeSetPartitionId::Mutator)]; + assert(mutator_regions_shortfall <= young_retired_regions, "sanity"); + regions[int(ShenandoahFreeSetPartitionId::Mutator)] += mutator_regions_shortfall; + young_retired_regions -= mutator_regions_shortfall; + regions[int(ShenandoahFreeSetPartitionId::Collector)] += young_retired_regions; + + assert(capacities[int(ShenandoahFreeSetPartitionId::Collector)] == _capacity[int(ShenandoahFreeSetPartitionId::Collector)], + "Collector capacities must match"); + assert(used[int(ShenandoahFreeSetPartitionId::Collector)] == _used[int(ShenandoahFreeSetPartitionId::Collector)], + "Collector used must match"); + assert(regions[int(ShenandoahFreeSetPartitionId::Collector)] == + _total_region_counts[int(ShenandoahFreeSetPartitionId::Collector)], "Collector regions must match"); + assert(_capacity[int(ShenandoahFreeSetPartitionId::Collector)] >= _used[int(ShenandoahFreeSetPartitionId::Collector)], + "Collector Capacity must be >= used"); + assert(_available[int(ShenandoahFreeSetPartitionId::Collector)] == + (_capacity[int(ShenandoahFreeSetPartitionId::Collector)] - _used[int(ShenandoahFreeSetPartitionId::Collector)]), + "Collector Available must equal capacity minus used"); + + assert(capacities[int(ShenandoahFreeSetPartitionId::Mutator)] == _capacity[int(ShenandoahFreeSetPartitionId::Mutator)], + "Mutator capacities must match"); + assert(used[int(ShenandoahFreeSetPartitionId::Mutator)] == _used[int(ShenandoahFreeSetPartitionId::Mutator)], + "Mutator used must match"); + assert(regions[int(ShenandoahFreeSetPartitionId::Mutator)] == _total_region_counts[int(ShenandoahFreeSetPartitionId::Mutator)], + "Mutator regions must match"); + assert(_capacity[int(ShenandoahFreeSetPartitionId::Mutator)] >= _used[int(ShenandoahFreeSetPartitionId::Mutator)], + "Mutator capacity must be >= used"); + assert(_available[int(ShenandoahFreeSetPartitionId::Mutator)] == + (_capacity[int(ShenandoahFreeSetPartitionId::Mutator)] - _used[int(ShenandoahFreeSetPartitionId::Mutator)]), + "Mutator available must equal capacity minus used"); + assert(_humongous_waste[int(ShenandoahFreeSetPartitionId::Mutator)] == young_humongous_waste, + "Mutator humongous waste must match"); + } } #endif ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) : _heap(heap), _partitions(max_regions, this), - _alloc_bias_weight(0) + _total_humongous_waste(0), + _alloc_bias_weight(0), + _total_young_used(0), + _total_old_used(0), + _total_global_used(0), + _young_affiliated_regions(0), + _old_affiliated_regions(0), + _global_affiliated_regions(0), + _young_unaffiliated_regions(0), + _global_unaffiliated_regions(0), + _total_young_regions(0), + _total_global_regions(0) { clear_internal(); } +// was pip_pad_bytes void ShenandoahFreeSet::add_promoted_in_place_region_to_old_collector(ShenandoahHeapRegion* region) { shenandoah_assert_heaplocked(); size_t plab_min_size_in_bytes = ShenandoahGenerationalHeap::heap()->plab_min_size() * HeapWordSize; - size_t idx = region->index(); - size_t capacity = alloc_capacity(region); - assert(_partitions.membership(idx) == ShenandoahFreeSetPartitionId::NotFree, + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t available_in_region = alloc_capacity(region); + size_t region_index = region->index(); + ShenandoahFreeSetPartitionId p = _partitions.membership(region_index); + assert(_partitions.membership(region_index) == ShenandoahFreeSetPartitionId::NotFree, "Regions promoted in place should have been excluded from Mutator partition"); - if (capacity >= plab_min_size_in_bytes) { - _partitions.make_free(idx, ShenandoahFreeSetPartitionId::OldCollector, capacity); - _heap->old_generation()->augment_promoted_reserve(capacity); + + // If region had been retired, its end-of-region alignment pad had been counted as used within the Mutator partition + size_t used_while_awaiting_pip = region_size_bytes; + size_t used_after_pip = region_size_bytes; + if (available_in_region >= plab_min_size_in_bytes) { + used_after_pip -= available_in_region; + } else { + if (available_in_region >= ShenandoahHeap::min_fill_size() * HeapWordSize) { + size_t fill_words = available_in_region / HeapWordSize; + ShenandoahHeap::heap()->old_generation()->card_scan()->register_object(region->top()); + region->allocate_fill(fill_words); + } + available_in_region = 0; + } + + assert(p == ShenandoahFreeSetPartitionId::NotFree, "pip region must be NotFree"); + assert(region->is_young(), "pip region must be young"); + _partitions.decrease_used(ShenandoahFreeSetPartitionId::Mutator, used_while_awaiting_pip); + + // decrease capacity adjusts available + _partitions.decrease_capacity(ShenandoahFreeSetPartitionId::Mutator, region_size_bytes); + _partitions.decrease_total_region_counts(ShenandoahFreeSetPartitionId::Mutator, 1); + + _partitions.increase_total_region_counts(ShenandoahFreeSetPartitionId::OldCollector, 1); + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::OldCollector, region_size_bytes); + _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, used_after_pip); + region->set_affiliation(ShenandoahAffiliation::OLD_GENERATION); + if (available_in_region > 0) { + assert(available_in_region >= plab_min_size_in_bytes, "enforced above"); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::OldCollector, 1); + // make_free() adjusts bounds for OldCollector partition + _partitions.make_free(region_index, ShenandoahFreeSetPartitionId::OldCollector, available_in_region); + _heap->old_generation()->augment_promoted_reserve(available_in_region); + assert(available_in_region != region_size_bytes, "Nothing to promote in place"); } + // else, leave this region as NotFree + + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); } HeapWord* ShenandoahFreeSet::allocate_from_partition_with_affiliation(ShenandoahAffiliation affiliation, @@ -782,7 +1240,7 @@ HeapWord* ShenandoahFreeSet::allocate_from_partition_with_affiliation(Shenandoah template HeapWord* ShenandoahFreeSet::allocate_with_affiliation(Iter& iterator, ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) { - for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + for (index_type idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { ShenandoahHeapRegion* r = _heap->get_region(idx); if (r->affiliation() == affiliation) { HeapWord* result = try_allocate_in(r, req, in_new_region); @@ -867,9 +1325,9 @@ void ShenandoahFreeSet::update_allocation_bias() { // 1. Eventual collection set has fewer regions because we have packed newly allocated objects into fewer regions // 2. We preserve the "empty" regions longer into the GC cycle, reducing likelihood of allocation failures // late in the GC cycle. - idx_t non_empty_on_left = (_partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator) + index_type non_empty_on_left = (_partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator) - _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator)); - idx_t non_empty_on_right = (_partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator) + index_type non_empty_on_right = (_partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator) - _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator)); _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Mutator, (non_empty_on_right < non_empty_on_left)); _alloc_bias_weight = INITIAL_ALLOC_BIAS_WEIGHT; @@ -878,7 +1336,7 @@ void ShenandoahFreeSet::update_allocation_bias() { template HeapWord* ShenandoahFreeSet::allocate_from_regions(Iter& iterator, ShenandoahAllocRequest &req, bool &in_new_region) { - for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + for (index_type idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { ShenandoahHeapRegion* r = _heap->get_region(idx); size_t min_size = (req.type() == ShenandoahAllocRequest::_alloc_tlab) ? req.min_size() : req.size(); if (alloc_capacity(r) >= min_size * HeapWordSize) { @@ -947,7 +1405,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_from_mutator(ShenandoahAllocRequest& r // The collector prefers to keep longer lived regions toward the right side of the heap, so it always // searches for regions from right to left here. ShenandoahRightLeftIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::Mutator, true); - for (idx_t idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { + for (index_type idx = iterator.current(); iterator.has_next(); idx = iterator.next()) { ShenandoahHeapRegion* r = _heap->get_region(idx); if (can_allocate_from(r)) { if (req.is_old()) { @@ -1035,7 +1493,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah log_debug(gc, free)("Using new region (%zu) for %s (" PTR_FORMAT ").", r->index(), ShenandoahAllocRequest::alloc_type_to_string(req.type()), p2i(&req)); assert(!r->is_affiliated(), "New region %zu should be unaffiliated", r->index()); - r->set_affiliation(req.affiliation()); if (r->is_old()) { // Any OLD region allocated during concurrent coalesce-and-fill does not need to be coalesced and filled because @@ -1047,8 +1504,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah r->end_preemptible_coalesce_and_fill(); _heap->old_generation()->clear_cards_for(r); } - _heap->generation_for(r->affiliation())->increment_affiliated_region_count(); - #ifdef ASSERT ShenandoahMarkingContext* const ctx = _heap->marking_context(); assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); @@ -1121,6 +1576,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah if (req.is_mutator_alloc()) { assert(req.is_young(), "Mutator allocations always come from young generation."); _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, req.actual_size() * HeapWordSize); + increase_bytes_allocated(req.actual_size() * HeapWordSize); } else { assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); @@ -1129,19 +1585,38 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // PLABs be made parsable at the end of evacuation. This is enabled by retiring all plabs at end of evacuation. r->set_update_watermark(r->top()); if (r->is_old()) { - _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, req.actual_size() * HeapWordSize); + _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, (req.actual_size() + req.waste()) * HeapWordSize); assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "old-gen allocations use PLAB or shared allocation"); // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab } else { - _partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, req.actual_size() * HeapWordSize); + _partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, (req.actual_size() + req.waste()) * HeapWordSize); } } } - static const size_t min_capacity = (size_t) (ShenandoahHeapRegion::region_size_bytes() * (1.0 - 1.0 / ShenandoahEvacWaste)); size_t ac = alloc_capacity(r); - - if (((result == nullptr) && (ac < min_capacity)) || (alloc_capacity(r) < PLAB::min_size() * HeapWordSize)) { + ShenandoahFreeSetPartitionId orig_partition; + ShenandoahGeneration* request_generation = nullptr; + if (req.is_mutator_alloc()) { + request_generation = _heap->mode()->is_generational()? _heap->young_generation(): _heap->global_generation(); + orig_partition = ShenandoahFreeSetPartitionId::Mutator; + } else if (req.type() == ShenandoahAllocRequest::_alloc_gclab) { + request_generation = _heap->mode()->is_generational()? _heap->young_generation(): _heap->global_generation(); + orig_partition = ShenandoahFreeSetPartitionId::Collector; + } else if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + request_generation = _heap->old_generation(); + orig_partition = ShenandoahFreeSetPartitionId::OldCollector; + } else { + assert(req.type() == ShenandoahAllocRequest::_alloc_shared_gc, "Unexpected allocation type"); + if (req.is_old()) { + request_generation = _heap->old_generation(); + orig_partition = ShenandoahFreeSetPartitionId::OldCollector; + } else { + request_generation = _heap->mode()->is_generational()? _heap->young_generation(): _heap->global_generation(); + orig_partition = ShenandoahFreeSetPartitionId::Collector; + } + } + if (alloc_capacity(r) < PLAB::min_size() * HeapWordSize) { // Regardless of whether this allocation succeeded, if the remaining memory is less than PLAB:min_size(), retire this region. // Note that retire_from_partition() increases used to account for waste. @@ -1149,24 +1624,19 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // then retire the region so that subsequent searches can find available memory more quickly. size_t idx = r->index(); - ShenandoahFreeSetPartitionId orig_partition; - if (req.is_mutator_alloc()) { - orig_partition = ShenandoahFreeSetPartitionId::Mutator; - } else if (req.type() == ShenandoahAllocRequest::_alloc_gclab) { - orig_partition = ShenandoahFreeSetPartitionId::Collector; - } else if (req.type() == ShenandoahAllocRequest::_alloc_plab) { - orig_partition = ShenandoahFreeSetPartitionId::OldCollector; - } else { - assert(req.type() == ShenandoahAllocRequest::_alloc_shared_gc, "Unexpected allocation type"); - if (req.is_old()) { - orig_partition = ShenandoahFreeSetPartitionId::OldCollector; - } else { - orig_partition = ShenandoahFreeSetPartitionId::Collector; - } + if ((result != nullptr) && in_new_region) { + _partitions.one_region_is_no_longer_empty(orig_partition); } - _partitions.retire_from_partition(orig_partition, idx, r->used()); - _partitions.assert_bounds(); + size_t waste_bytes = _partitions.retire_from_partition(orig_partition, idx, r->used()); + if (req.is_mutator_alloc() && (waste_bytes > 0)) { + increase_bytes_allocated(waste_bytes); + } + } else if ((result != nullptr) && in_new_region) { + _partitions.one_region_is_no_longer_empty(orig_partition); } + recompute_total_used(); + recompute_total_affiliated(); // could optimize: only recompute affiliated for orig_partition and global + _partitions.assert_bounds(true); return result; } @@ -1175,29 +1645,29 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req, bo shenandoah_assert_heaplocked(); size_t words_size = req.size(); - idx_t num = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize); + index_type num = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize); assert(req.is_young(), "Humongous regions always allocated in YOUNG"); ShenandoahGeneration* generation = _heap->generation_for(req.affiliation()); // Check if there are enough regions left to satisfy allocation. - if (num > (idx_t) _partitions.count(ShenandoahFreeSetPartitionId::Mutator)) { + if (num > (index_type) _partitions.count(ShenandoahFreeSetPartitionId::Mutator)) { return nullptr; } - idx_t start_range = _partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator); - idx_t end_range = _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator) + 1; - idx_t last_possible_start = end_range - num; + index_type start_range = _partitions.leftmost_empty(ShenandoahFreeSetPartitionId::Mutator); + index_type end_range = _partitions.rightmost_empty(ShenandoahFreeSetPartitionId::Mutator) + 1; + index_type last_possible_start = end_range - num; // Find the continuous interval of $num regions, starting from $beg and ending in $end, // inclusive. Contiguous allocations are biased to the beginning. - idx_t beg = _partitions.find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId::Mutator, + index_type beg = _partitions.find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId::Mutator, start_range, num); if (beg > last_possible_start) { // Hit the end, goodbye return nullptr; } - idx_t end = beg; + index_type end = beg; while (true) { // We've confirmed num contiguous regions belonging to Mutator partition, so no need to confirm membership. @@ -1205,12 +1675,12 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req, bo // the existing range, we can exploit that certain regions are already known to be in the Mutator free set. while (!can_allocate_from(_heap->get_region(end))) { // region[end] is not empty, so we restart our search after region[end] - idx_t slide_delta = end + 1 - beg; + index_type slide_delta = end + 1 - beg; if (beg + slide_delta > last_possible_start) { // no room to slide return nullptr; } - for (idx_t span_end = beg + num; slide_delta > 0; slide_delta--) { + for (index_type span_end = beg + num; slide_delta > 0; slide_delta--) { if (!_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, span_end)) { beg = _partitions.find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId::Mutator, span_end + 1, num); @@ -1236,67 +1706,187 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req, bo end++; } - size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask(); - // Initialize regions: - for (idx_t i = beg; i <= end; i++) { - ShenandoahHeapRegion* r = _heap->get_region(i); - r->try_recycle_under_lock(); - - assert(i == beg || _heap->get_region(i - 1)->index() + 1 == r->index(), "Should be contiguous"); - assert(r->is_empty(), "Should be empty"); - - r->set_affiliation(req.affiliation()); - - if (is_humongous) { + size_t total_used = 0; + const size_t used_words_in_last_region = words_size & ShenandoahHeapRegion::region_size_words_mask(); + size_t waste_bytes; + // Retire regions from free partition and initialize them. + if (is_humongous) { + // Humongous allocation retires all regions at once: no allocation is possible anymore. + // retire_range_from_partition() will adjust bounds on Mutator free set if appropriate and will recompute affiliated. + _partitions.retire_range_from_partition(ShenandoahFreeSetPartitionId::Mutator, beg, end); + for (index_type i = beg; i <= end; i++) { + ShenandoahHeapRegion* r = _heap->get_region(i); + assert(i == beg || _heap->get_region(i - 1)->index() + 1 == r->index(), "Should be contiguous"); + r->try_recycle_under_lock(); + assert(r->is_empty(), "Should be empty"); + r->set_affiliation(req.affiliation()); if (i == beg) { r->make_humongous_start(); } else { r->make_humongous_cont(); } - } else { - r->make_regular_allocation(req.affiliation()); - } - - // Trailing region may be non-full, record the remainder there - size_t used_words; - if ((i == end) && (remainder != 0)) { - used_words = remainder; - } else { - used_words = ShenandoahHeapRegion::region_size_words(); + if ((i == end) && (used_words_in_last_region > 0)) { + r->set_top(r->bottom() + used_words_in_last_region); + } else { + // if used_words_in_last_region is zero, then the end region is fully consumed. + r->set_top(r->end()); + } + r->set_update_watermark(r->bottom()); } - r->set_update_watermark(r->bottom()); - r->set_top(r->bottom() + used_words); - } - generation->increase_affiliated_region_count(num); - - size_t total_used = 0; - if (is_humongous) { - // Humongous allocation retires all regions at once: no allocation is possible anymore. - _partitions.retire_range_from_partition(ShenandoahFreeSetPartitionId::Mutator, beg, end); total_used = ShenandoahHeapRegion::region_size_bytes() * num; + waste_bytes = + (used_words_in_last_region == 0)? 0: ShenandoahHeapRegion::region_size_bytes() - used_words_in_last_region * HeapWordSize; } else { // Non-humongous allocation retires only the regions that cannot be used for allocation anymore. - for (idx_t i = beg; i <= end; i++) { + waste_bytes = 0; + for (index_type i = beg; i <= end; i++) { ShenandoahHeapRegion* r = _heap->get_region(i); - if (r->free() < PLAB::min_size() * HeapWordSize) { - _partitions.retire_from_partition(ShenandoahFreeSetPartitionId::Mutator, i, r->used()); + assert(i == beg || _heap->get_region(i - 1)->index() + 1 == r->index(), "Should be contiguous"); + assert(r->is_empty(), "Should be empty"); + r->try_recycle_under_lock(); + r->set_affiliation(req.affiliation()); + r->make_regular_allocation(req.affiliation()); + if ((i == end) && (used_words_in_last_region > 0)) { + r->set_top(r->bottom() + used_words_in_last_region); + } else { + // if used_words_in_last_region is zero, then the end region is fully consumed. + r->set_top(r->end()); } + r->set_update_watermark(r->bottom()); total_used += r->used(); + if (r->free() < PLAB::min_size() * HeapWordSize) { + // retire_from_partition() will adjust bounds on Mutator free set if appropriate and will recompute affiliated. + // It also increases used for the waste bytes, which includes bytes filled at retirement and bytes too small + // to be filled. Only the last iteration may have non-zero waste_bytes. + waste_bytes += _partitions.retire_from_partition(ShenandoahFreeSetPartitionId::Mutator, i, r->used()); + } + } + _partitions.decrease_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator, num); + if (waste_bytes > 0) { + // For humongous allocations, waste_bytes are included in total_used. Since this is not humongous, + // we need to account separately for the waste_bytes. + increase_bytes_allocated(waste_bytes); } } _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, total_used); - _partitions.assert_bounds(); - + increase_bytes_allocated(total_used); req.set_actual_size(words_size); - if (remainder != 0 && is_humongous) { - req.set_waste(ShenandoahHeapRegion::region_size_words() - remainder); + // If !is_humongous, the "waste" is made availabe for new allocation + if (waste_bytes > 0) { + req.set_waste(waste_bytes / HeapWordSize); + if (is_humongous) { + _partitions.increase_humongous_waste(ShenandoahFreeSetPartitionId::Mutator, waste_bytes); + _total_humongous_waste += waste_bytes; + } } + recompute_total_young_used(); + recompute_total_global_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); return _heap->get_region(beg)->bottom(); } class ShenandoahRecycleTrashedRegionClosure final : public ShenandoahHeapRegionClosure { +private: + static const ssize_t SentinelUsed = -1; + static const ssize_t SentinelIndex = -1; + static const size_t MaxSavedRegions = 128; + + ShenandoahRegionPartitions* _partitions; + volatile size_t _recycled_region_count; + ssize_t _region_indices[MaxSavedRegions]; + ssize_t _region_used[MaxSavedRegions]; + + void get_lock_and_flush_buffer(size_t region_count, size_t overflow_region_used, size_t overflow_region_index) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahHeapLocker locker(heap->lock()); + size_t recycled_regions = Atomic::load(&_recycled_region_count); + size_t region_tallies[int(ShenandoahRegionPartitions::NumPartitions)]; + size_t used_byte_tallies[int(ShenandoahRegionPartitions::NumPartitions)]; + for (int p = 0; p < int(ShenandoahRegionPartitions::NumPartitions); p++) { + region_tallies[p] = 0; + used_byte_tallies[p] = 0; + } + ShenandoahFreeSetPartitionId p = _partitions->membership(overflow_region_index); + used_byte_tallies[int(p)] += overflow_region_used; + if (region_count <= recycled_regions) { + // _recycled_region_count has not been decremented after I incremented it to obtain region_count, so I will + // try to flush the buffer. + + // Multiple worker threads may attempt to flush this buffer. The first thread to acquire the lock does the work. + // _recycled_region_count is only decreased while holding the heap lock. + if (region_count > recycled_regions) { + region_count = recycled_regions; + } + for (size_t i = 0; i < region_count; i++) { + ssize_t used; + // wait for other threads to finish updating their entries within the region buffer before processing entry + do { + used = _region_used[i]; + } while (used == SentinelUsed); + ssize_t index; + do { + index = _region_indices[i]; + } while (index == SentinelIndex); + + ShenandoahFreeSetPartitionId p = _partitions->membership(index); + assert(p != ShenandoahFreeSetPartitionId::NotFree, "Trashed regions should be in a free partition"); + used_byte_tallies[int(p)] += used; + region_tallies[int(p)]++; + } + if (region_count > 0) { + for (size_t i = 0; i < MaxSavedRegions; i++) { + _region_indices[i] = SentinelIndex; + _region_used[i] = SentinelUsed; + } + } + + // The almost last thing we do before releasing the lock is to set the _recycled_region_count to 0. What happens next? + // + // 1. Any worker thread that attempted to buffer a new region while we were flushing the buffer will have seen + // that _recycled_region_count > MaxSavedRegions. All such worker threads will first wait for the lock, then + // discover that the _recycled_region_count is zero, then, while holding the lock, they will process the + // region so it doesn't have to be placed into the buffer. This handles the large majority of cases. + // + // 2. However, there's a race that can happen, which will result in someewhat different behavior. Suppose + // this thread resets _recycled_region_count to 0. Then some other worker thread increments _recycled_region_count + // in order to stores its region into the buffer and suppose this happens before all of the other worker threads + // which are waiting to acquire the heap lock have finished their efforts to flush the buffer. If this happens, + // then the workers who are waiting to acquire the heap lock and flush the buffer will find that _recycled_region_count + // has decreased from the value it held when they last tried to increment its value. In this case, these worker + // threads will process their overflow region while holding the lock, but they will not attempt to process regions + // newly placed into the buffer. Otherwise, confusion could result. + // + // Assumption: all worker threads who are attempting to acquire lock and flush buffer will finish their efforts before + // the buffer once again overflows. + // How could we avoid depending on this assumption? + // 1. Let MaxSavedRegions be as large as number of regions, or at least as large as the collection set. + // 2. Keep a count of how many times the buffer has been flushed per instantation of the + // ShenandoahRecycleTrashedRegionClosure object, and only consult/update this value while holding the heap lock. + // Need to think about how this helps resolve the race. + _recycled_region_count = 0; + } else { + // Some other thread has already processed the buffer, resetting _recycled_region_count to zero. Its current value + // may be greater than zero because other workers may have accumulated entries into the buffer. But it is "extremely" + // unlikely that it will overflow again before all waiting workers have had a chance to clear their state. While I've + // got the heap lock, I'll go ahead and update the global state for my overflow region. I'll let other heap regions + // accumulate in the buffer to be processed when the buffer is once again full. + region_count = 0; + } + for (size_t p = 0; p < int(ShenandoahRegionPartitions::NumPartitions); p++) { + _partitions->decrease_used(ShenandoahFreeSetPartitionId(p), used_byte_tallies[p]); + } + } + public: - ShenandoahRecycleTrashedRegionClosure(): ShenandoahHeapRegionClosure() {} + ShenandoahRecycleTrashedRegionClosure(ShenandoahRegionPartitions* p): ShenandoahHeapRegionClosure() { + _partitions = p; + _recycled_region_count = 0; + for (size_t i = 0; i < MaxSavedRegions; i++) { + _region_indices[i] = SentinelIndex; + _region_used[i] = SentinelUsed; + } + } void heap_region_do(ShenandoahHeapRegion* r) { r->try_recycle(); @@ -1314,10 +1904,31 @@ void ShenandoahFreeSet::recycle_trash() { ShenandoahHeap* heap = ShenandoahHeap::heap(); heap->assert_gc_workers(heap->workers()->active_workers()); - ShenandoahRecycleTrashedRegionClosure closure; + ShenandoahRecycleTrashedRegionClosure closure(&_partitions); heap->parallel_heap_region_iterate(&closure); } +bool ShenandoahFreeSet::transfer_one_region_from_mutator_to_old_collector(size_t idx, size_t alloc_capacity) { + ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); + ShenandoahYoungGeneration* young_gen = gen_heap->young_generation(); + ShenandoahOldGeneration* old_gen = gen_heap->old_generation(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + if ((young_unaffiliated_regions() > 0) && + (((_partitions.get_total_region_counts(ShenandoahFreeSetPartitionId::OldCollector) + 1) * region_size_bytes) + <= gen_heap->generation_sizer()->max_size_for(old_gen)) && + ((total_young_regions() - 1) * region_size_bytes >= gen_heap->generation_sizer()->min_size_for(young_gen))) { + _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, + ShenandoahFreeSetPartitionId::OldCollector, alloc_capacity); + gen_heap->old_generation()->augment_evacuation_reserve(alloc_capacity); + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); + return true; + } else { + return false; + } +} + bool ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { const size_t idx = r->index(); @@ -1325,14 +1936,9 @@ bool ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { assert(can_allocate_from(r), "Should not be allocated"); ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); - const size_t region_capacity = alloc_capacity(r); + const size_t region_alloc_capacity = alloc_capacity(r); - bool transferred = gen_heap->generation_sizer()->transfer_to_old(1); - if (transferred) { - _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, - ShenandoahFreeSetPartitionId::OldCollector, region_capacity); - _partitions.assert_bounds(); - _heap->old_generation()->augment_evacuation_reserve(region_capacity); + if (transfer_one_region_from_mutator_to_old_collector(idx, region_alloc_capacity)) { return true; } @@ -1344,7 +1950,7 @@ bool ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { // partition. // 1. Find a temporarily unusable trash region in the old collector partition ShenandoahRightLeftIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::OldCollector, true); - idx_t unusable_trash = -1; + index_type unusable_trash = -1; for (unusable_trash = iterator.current(); iterator.has_next(); unusable_trash = iterator.next()) { const ShenandoahHeapRegion* region = _heap->get_region(unusable_trash); if (region->is_trash() && _heap->is_concurrent_weak_root_in_progress()) { @@ -1362,15 +1968,15 @@ bool ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { // 3. Move this usable region from the mutator partition to the old collector partition _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, - ShenandoahFreeSetPartitionId::OldCollector, region_capacity); - - _partitions.assert_bounds(); - + ShenandoahFreeSetPartitionId::OldCollector, region_alloc_capacity); + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); // 4. Do not adjust capacities for generations, we just swapped the regions that have already // been accounted for. However, we should adjust the evacuation reserves as those may have changed. shenandoah_assert_heaplocked(); const size_t reserve = _heap->old_generation()->get_evacuation_reserve(); - _heap->old_generation()->set_evacuation_reserve(reserve - unusable_capacity + region_capacity); + _heap->old_generation()->set_evacuation_reserve(reserve - unusable_capacity + region_alloc_capacity); return true; } } @@ -1388,8 +1994,9 @@ void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) { size_t ac = alloc_capacity(r); _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, ShenandoahFreeSetPartitionId::Collector, ac); - _partitions.assert_bounds(); - + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); // We do not ensure that the region is no longer trash, relying on try_allocate_in(), which always comes next, // to recycle trash before attempting to allocate anything in the region. } @@ -1401,6 +2008,9 @@ void ShenandoahFreeSet::clear() { void ShenandoahFreeSet::clear_internal() { shenandoah_assert_heaplocked(); _partitions.make_all_regions_unavailable(); + reset_bytes_allocated_since_gc_start(); + recompute_total_used(); + recompute_total_affiliated(); _alloc_bias_weight = 0; _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::Mutator, true); @@ -1408,7 +2018,7 @@ void ShenandoahFreeSet::clear_internal() { _partitions.set_bias_from_left_to_right(ShenandoahFreeSetPartitionId::OldCollector, false); } -void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regions, size_t &old_cset_regions, +void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_trashed_regions, size_t &old_trashed_regions, size_t &first_old_region, size_t &last_old_region, size_t &old_region_count) { clear_internal(); @@ -1416,37 +2026,60 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi first_old_region = _heap->num_regions(); last_old_region = 0; old_region_count = 0; - old_cset_regions = 0; - young_cset_regions = 0; + old_trashed_regions = 0; + young_trashed_regions = 0; + + size_t old_cset_regions = 0; + size_t young_cset_regions = 0; size_t region_size_bytes = _partitions.region_size_bytes(); - size_t max_regions = _partitions.max_regions(); + size_t max_regions = _partitions.max(); size_t mutator_leftmost = max_regions; size_t mutator_rightmost = 0; size_t mutator_leftmost_empty = max_regions; size_t mutator_rightmost_empty = 0; - size_t mutator_regions = 0; - size_t mutator_used = 0; size_t old_collector_leftmost = max_regions; size_t old_collector_rightmost = 0; size_t old_collector_leftmost_empty = max_regions; size_t old_collector_rightmost_empty = 0; - size_t old_collector_regions = 0; + + size_t mutator_empty = 0; + size_t old_collector_empty = 0; + + // These two variables represent the total used within each partition, including humongous waste and retired regions + size_t mutator_used = 0; size_t old_collector_used = 0; + // These two variables represent memory that is wasted within humongous regions due to alignment padding + size_t mutator_humongous_waste = 0; + size_t old_collector_humongous_waste = 0; + + // These two variables track regions that have allocatable memory + size_t mutator_regions = 0; + size_t old_collector_regions = 0; + + // These two variables track regions that are not empty within each partition + size_t affiliated_mutator_regions = 0; + size_t affiliated_old_collector_regions = 0; + + // These two variables represent the total capacity of each partition, including retired regions + size_t total_mutator_regions = 0; + size_t total_old_collector_regions = 0; + + bool is_generational = _heap->mode()->is_generational(); size_t num_regions = _heap->num_regions(); for (size_t idx = 0; idx < num_regions; idx++) { ShenandoahHeapRegion* region = _heap->get_region(idx); if (region->is_trash()) { - // Trashed regions represent regions that had been in the collection partition but have not yet been "cleaned up". - // The cset regions are not "trashed" until we have finished update refs. + // Trashed regions represent immediate garbage identified by final mark and regions that had been in the collection + // partition but have not yet been "cleaned up" following update refs. if (region->is_old()) { - old_cset_regions++; + old_trashed_regions++; } else { assert(region->is_young(), "Trashed region should be old or young"); - young_cset_regions++; + young_trashed_regions++; } } else if (region->is_old()) { // count both humongous and regular regions, but don't count trash (cset) regions. @@ -1461,7 +2094,7 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi // Do not add regions that would almost surely fail allocation size_t ac = alloc_capacity(region); - if (ac > PLAB::min_size() * HeapWordSize) { + if (ac >= PLAB::min_size() * HeapWordSize) { if (region->is_trash() || !region->is_old()) { // Both young and old collected regions (trashed) are placed into the Mutator set _partitions.raw_assign_membership(idx, ShenandoahFreeSetPartitionId::Mutator); @@ -1472,14 +2105,18 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi mutator_rightmost = idx; } if (ac == region_size_bytes) { + mutator_empty++; if (idx < mutator_leftmost_empty) { mutator_leftmost_empty = idx; } if (idx > mutator_rightmost_empty) { mutator_rightmost_empty = idx; } + } else { + affiliated_mutator_regions++; } mutator_regions++; + total_mutator_regions++; mutator_used += (region_size_bytes - ac); } else { // !region->is_trash() && region is_old() @@ -1490,20 +2127,67 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi if (idx > old_collector_rightmost) { old_collector_rightmost = idx; } - if (ac == region_size_bytes) { - if (idx < old_collector_leftmost_empty) { - old_collector_leftmost_empty = idx; - } - if (idx > old_collector_rightmost_empty) { - old_collector_rightmost_empty = idx; - } - } + assert(ac != region_size_bytes, "Empty regions should be in mutator partition"); + affiliated_old_collector_regions++; old_collector_regions++; - old_collector_used += (region_size_bytes - ac); + total_old_collector_regions++; + old_collector_used += region_size_bytes - ac; + } + } else { + // This region does not have enough free to be part of the free set. Count all of its memory as used. + assert(_partitions.membership(idx) == ShenandoahFreeSetPartitionId::NotFree, "Region should have been retired"); + if (region->is_old()) { + old_collector_used += region_size_bytes; + total_old_collector_regions++; + affiliated_old_collector_regions++; + } else { + mutator_used += region_size_bytes; + total_mutator_regions++; + affiliated_mutator_regions++; + } + } + } else { + // This region does not allow allocation (it is retired or is humongous or is in cset). + // Retired and humongous regions generally have no alloc capacity, but cset regions may have large alloc capacity. + if (region->is_cset()) { + if (region->is_old()) { + old_cset_regions++; + } else { + young_cset_regions++; + } + } else { + assert(_partitions.membership(idx) == ShenandoahFreeSetPartitionId::NotFree, "Region should have been retired"); + size_t ac = alloc_capacity(region); + size_t humongous_waste_bytes = 0; + if (region->is_humongous_start()) { + oop obj = cast_to_oop(region->bottom()); + size_t byte_size = obj->size() * HeapWordSize; + size_t region_span = ShenandoahHeapRegion::required_regions(byte_size); + humongous_waste_bytes = region_span * ShenandoahHeapRegion::region_size_bytes() - byte_size; + } + if (region->is_old()) { + old_collector_used += region_size_bytes; + total_old_collector_regions++; + old_collector_humongous_waste += humongous_waste_bytes; + affiliated_old_collector_regions++; + } else { + mutator_used += region_size_bytes; + total_mutator_regions++; + mutator_humongous_waste += humongous_waste_bytes; + affiliated_mutator_regions++; } } } } + // At the start of evacuation, the cset regions are not counted as part of Mutator or OldCollector partitions. + + // At the end of GC, when we rebuild rebuild freeset (which happens before we have recycled the collection set), we treat + // all cset regions as part of capacity, as fully available, as unaffiliated. We place trashed regions into the Mutator + // partition. + + // No need to update generation sizes here. These are the sizes already recognized by the generations. These + // adjustments allow the freeset tallies to match the generation tallies. + log_debug(gc, free)(" At end of prep_to_rebuild, mutator_leftmost: %zu" ", mutator_rightmost: %zu" ", mutator_leftmost_empty: %zu" @@ -1512,7 +2196,6 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi ", mutator_used: %zu", mutator_leftmost, mutator_rightmost, mutator_leftmost_empty, mutator_rightmost_empty, mutator_regions, mutator_used); - log_debug(gc, free)(" old_collector_leftmost: %zu" ", old_collector_rightmost: %zu" ", old_collector_leftmost_empty: %zu" @@ -1521,16 +2204,37 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi ", old_collector_used: %zu", old_collector_leftmost, old_collector_rightmost, old_collector_leftmost_empty, old_collector_rightmost_empty, old_collector_regions, old_collector_used); + log_debug(gc, free)(" total_mutator_regions: %zu, total_old_collector_regions: %zu" + ", mutator_empty: %zu, old_collector_empty: %zu", + total_mutator_regions, total_old_collector_regions, mutator_empty, old_collector_empty); + index_type rightmost_idx = (mutator_leftmost == max_regions)? -1: (index_type) mutator_rightmost; + index_type rightmost_empty_idx = (mutator_leftmost_empty == max_regions)? -1: (index_type) mutator_rightmost_empty; - idx_t rightmost_idx = (mutator_leftmost == max_regions)? -1: (idx_t) mutator_rightmost; - idx_t rightmost_empty_idx = (mutator_leftmost_empty == max_regions)? -1: (idx_t) mutator_rightmost_empty; _partitions.establish_mutator_intervals(mutator_leftmost, rightmost_idx, mutator_leftmost_empty, rightmost_empty_idx, - mutator_regions, mutator_used); - rightmost_idx = (old_collector_leftmost == max_regions)? -1: (idx_t) old_collector_rightmost; - rightmost_empty_idx = (old_collector_leftmost_empty == max_regions)? -1: (idx_t) old_collector_rightmost_empty; - _partitions.establish_old_collector_intervals(old_collector_leftmost, rightmost_idx, old_collector_leftmost_empty, - rightmost_empty_idx, old_collector_regions, old_collector_used); + total_mutator_regions + young_cset_regions, mutator_empty, mutator_regions, + mutator_used + young_cset_regions * region_size_bytes, mutator_humongous_waste); + rightmost_idx = (old_collector_leftmost == max_regions)? -1: (index_type) old_collector_rightmost; + rightmost_empty_idx = (old_collector_leftmost_empty == max_regions)? -1: (index_type) old_collector_rightmost_empty; + _partitions.establish_old_collector_intervals(old_collector_leftmost, rightmost_idx, + old_collector_leftmost_empty, rightmost_empty_idx, + total_old_collector_regions + old_cset_regions, + old_collector_empty, old_collector_regions, + old_collector_used + old_cset_regions * region_size_bytes, + old_collector_humongous_waste); + _total_humongous_waste = mutator_humongous_waste + old_collector_humongous_waste; + _total_young_regions = total_mutator_regions + young_cset_regions; + _total_global_regions = _total_young_regions + total_old_collector_regions + old_cset_regions; + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); +#ifdef ASSERT + if (_heap->mode()->is_generational()) { + assert(young_affiliated_regions() == _heap->young_generation()->get_affiliated_region_count(), "sanity"); + } else { + assert(young_affiliated_regions() == _heap->global_generation()->get_affiliated_region_count(), "sanity"); + } +#endif log_debug(gc, free)(" After find_regions_with_alloc_capacity(), Mutator range [%zd, %zd]," " Old Collector range [%zd, %zd]", _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator), @@ -1539,6 +2243,101 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regi _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector)); } +void ShenandoahFreeSet::transfer_humongous_regions_from_mutator_to_old_collector(size_t xfer_regions, + size_t humongous_waste_bytes) { + shenandoah_assert_heaplocked(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + + _partitions.decrease_humongous_waste(ShenandoahFreeSetPartitionId::Mutator, humongous_waste_bytes); + _partitions.decrease_used(ShenandoahFreeSetPartitionId::Mutator, xfer_regions * region_size_bytes); + _partitions.decrease_total_region_counts(ShenandoahFreeSetPartitionId::Mutator, xfer_regions); + _partitions.decrease_capacity(ShenandoahFreeSetPartitionId::Mutator, xfer_regions * region_size_bytes); + + _partitions.increase_total_region_counts(ShenandoahFreeSetPartitionId::OldCollector, xfer_regions); + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::OldCollector, xfer_regions * region_size_bytes); + _partitions.increase_humongous_waste(ShenandoahFreeSetPartitionId::OldCollector, humongous_waste_bytes); + _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, xfer_regions * region_size_bytes); + + // _total_humongous_waste, _total_global_regions are unaffected by transfer + _total_young_regions -= xfer_regions; + recompute_total_young_used(); + recompute_total_old_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); + // global_used is unaffected by this transfer + + // No need to adjust ranges because humongous regions are not allocatable +} + +void ShenandoahFreeSet::transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId source, + ShenandoahFreeSetPartitionId dest, + size_t num_regions) { + assert(dest != source, "precondition"); + shenandoah_assert_heaplocked(); + const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t transferred_regions = 0; + size_t used_transfer = 0; + index_type source_low_idx = _partitions.max(); + index_type source_high_idx = -1; + index_type dest_low_idx = _partitions.max(); + index_type dest_high_idx = -1; + ShenandoahLeftRightIterator iterator(&_partitions, source, true); + for (index_type idx = iterator.current(); transferred_regions < num_regions && iterator.has_next(); idx = iterator.next()) { + // Note: can_allocate_from() denotes that region is entirely empty + if (can_allocate_from(idx)) { + if (idx < source_low_idx) { + source_low_idx = idx; + } + if (idx > source_high_idx) { + source_high_idx = idx; + } + if (idx < dest_low_idx) { + dest_low_idx = idx; + } + if (idx > dest_high_idx) { + dest_high_idx = idx; + } + used_transfer += _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, source, dest, region_size_bytes); + transferred_regions++; + } + } + + // All transferred regions are empty. + assert(used_transfer == 0, "empty regions should have no used"); + _partitions.expand_interval_if_range_modifies_either_boundary(dest, dest_low_idx, + dest_high_idx, dest_low_idx, dest_high_idx); + _partitions.shrink_interval_if_range_modifies_either_boundary(source, source_low_idx, source_high_idx, + transferred_regions); + + _partitions.decrease_region_counts(source, transferred_regions); + _partitions.decrease_empty_region_counts(source, transferred_regions); + _partitions.decrease_total_region_counts(source, transferred_regions); + _partitions.decrease_capacity(source, transferred_regions * region_size_bytes); + + _partitions.increase_total_region_counts(dest, transferred_regions); + _partitions.increase_capacity(dest, transferred_regions * region_size_bytes); + _partitions.increase_region_counts(dest, transferred_regions); + _partitions.increase_empty_region_counts(dest, transferred_regions); + + if (source == ShenandoahFreeSetPartitionId::OldCollector) { + assert((dest == ShenandoahFreeSetPartitionId::Collector) || (dest == ShenandoahFreeSetPartitionId::Mutator), "sanity"); + _total_young_regions += transferred_regions; + } else { + assert((source == ShenandoahFreeSetPartitionId::Collector) || (source == ShenandoahFreeSetPartitionId::Mutator), "sanity"); + if (dest == ShenandoahFreeSetPartitionId::OldCollector) { + _total_young_regions -= transferred_regions; + } else { + assert((dest == ShenandoahFreeSetPartitionId::Collector) || (dest == ShenandoahFreeSetPartitionId::Mutator), "sanity"); + // No adjustments to total_young_regions if transferring within young + } + } + + // _total_global_regions unaffected by transfer + recompute_total_used(); + _partitions.assert_bounds(true); + // Should not need to recompute_total_affiliated() because all transferred regions are empty. +} + // Returns number of regions transferred, adds transferred bytes to var argument bytes_transferred size_t ShenandoahFreeSet::transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector, size_t max_xfer_regions, @@ -1546,33 +2345,122 @@ size_t ShenandoahFreeSet::transfer_empty_regions_from_collector_set_to_mutator_s shenandoah_assert_heaplocked(); const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); size_t transferred_regions = 0; + size_t used_transfer = 0; + index_type collector_low_idx = _partitions.max(); + index_type collector_high_idx = -1; + index_type mutator_low_idx = _partitions.max(); + index_type mutator_high_idx = -1; ShenandoahLeftRightIterator iterator(&_partitions, which_collector, true); - for (idx_t idx = iterator.current(); transferred_regions < max_xfer_regions && iterator.has_next(); idx = iterator.next()) { + for (index_type idx = iterator.current(); transferred_regions < max_xfer_regions && iterator.has_next(); idx = iterator.next()) { // Note: can_allocate_from() denotes that region is entirely empty if (can_allocate_from(idx)) { - _partitions.move_from_partition_to_partition(idx, which_collector, ShenandoahFreeSetPartitionId::Mutator, region_size_bytes); + if (idx < collector_low_idx) { + collector_low_idx = idx; + } + if (idx > collector_high_idx) { + collector_high_idx = idx; + } + if (idx < mutator_low_idx) { + mutator_low_idx = idx; + } + if (idx > mutator_high_idx) { + mutator_high_idx = idx; + } + used_transfer += _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, which_collector, + ShenandoahFreeSetPartitionId::Mutator, + region_size_bytes); transferred_regions++; bytes_transferred += region_size_bytes; } } + // All transferred regions are empty. + assert(used_transfer == 0, "empty regions should have no used"); + _partitions.expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Mutator, mutator_low_idx, + mutator_high_idx, mutator_low_idx, mutator_high_idx); + _partitions.shrink_interval_if_range_modifies_either_boundary(which_collector, collector_low_idx, collector_high_idx, + transferred_regions); + + _partitions.decrease_region_counts(which_collector, transferred_regions); + _partitions.decrease_empty_region_counts(which_collector, transferred_regions); + _partitions.decrease_total_region_counts(which_collector, transferred_regions); + _partitions.decrease_capacity(which_collector, transferred_regions * region_size_bytes); + + _partitions.increase_total_region_counts(ShenandoahFreeSetPartitionId::Mutator, transferred_regions); + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::Mutator, transferred_regions * region_size_bytes); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::Mutator, transferred_regions); + _partitions.increase_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator, transferred_regions); + + if (which_collector == ShenandoahFreeSetPartitionId::OldCollector) { + _total_young_regions += transferred_regions; + } + // _total_global_regions unaffected by transfer + recompute_total_used(); + _partitions.assert_bounds(true); + // Should not need to recompute_total_affiliated() because all transferred regions are empty. return transferred_regions; } // Returns number of regions transferred, adds transferred bytes to var argument bytes_transferred -size_t ShenandoahFreeSet::transfer_non_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector, - size_t max_xfer_regions, - size_t& bytes_transferred) { +size_t ShenandoahFreeSet:: +transfer_non_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector, + size_t max_xfer_regions, size_t& bytes_transferred) { shenandoah_assert_heaplocked(); + size_t region_size_bytes = _partitions.region_size_bytes(); size_t transferred_regions = 0; + size_t used_transfer = 0; + index_type collector_low_idx = _partitions.max(); + index_type collector_high_idx = -1; + index_type mutator_low_idx = _partitions.max(); + index_type mutator_high_idx = -1; + ShenandoahLeftRightIterator iterator(&_partitions, which_collector, false); - for (idx_t idx = iterator.current(); transferred_regions < max_xfer_regions && iterator.has_next(); idx = iterator.next()) { + for (index_type idx = iterator.current(); transferred_regions < max_xfer_regions && iterator.has_next(); idx = iterator.next()) { size_t ac = alloc_capacity(idx); if (ac > 0) { - _partitions.move_from_partition_to_partition(idx, which_collector, ShenandoahFreeSetPartitionId::Mutator, ac); + if (idx < collector_low_idx) { + collector_low_idx = idx; + } + if (idx > collector_high_idx) { + collector_high_idx = idx; + } + if (idx < mutator_low_idx) { + mutator_low_idx = idx; + } + if (idx > mutator_high_idx) { + mutator_high_idx = idx; + } + assert (ac < region_size_bytes, "Move empty regions with different function"); + used_transfer += _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, which_collector, + ShenandoahFreeSetPartitionId::Mutator, + ac); transferred_regions++; bytes_transferred += ac; } } + // _empty_region_counts is unaffected, because we transfer only non-empty regions here. + + _partitions.decrease_used(which_collector, used_transfer); + _partitions.expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Mutator, + mutator_low_idx, mutator_high_idx, _partitions.max(), -1); + _partitions.shrink_interval_if_range_modifies_either_boundary(which_collector, collector_low_idx, collector_high_idx, + transferred_regions); + + _partitions.decrease_region_counts(which_collector, transferred_regions); + _partitions.decrease_capacity(which_collector, transferred_regions * region_size_bytes); + _partitions.decrease_total_region_counts(which_collector, transferred_regions); + + _partitions.increase_total_region_counts(ShenandoahFreeSetPartitionId::Mutator, transferred_regions); + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::Mutator, transferred_regions * region_size_bytes); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::Mutator, transferred_regions); + _partitions.increase_used(ShenandoahFreeSetPartitionId::Mutator, used_transfer); + + if (which_collector == ShenandoahFreeSetPartitionId::OldCollector) { + _total_young_regions += transferred_regions; + } + // _total_global_regions unaffected by transfer + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); return transferred_regions; } @@ -1599,9 +2487,6 @@ void ShenandoahFreeSet::move_regions_from_collector_to_mutator(size_t max_xfer_r transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId::OldCollector, max_xfer_regions, old_collector_xfer); max_xfer_regions -= old_collector_regions; - if (old_collector_regions > 0) { - ShenandoahGenerationalHeap::cast(_heap)->generation_sizer()->transfer_to_young(old_collector_regions); - } } // If there are any non-empty regions within Collector partition, we can also move them to the Mutator free partition @@ -1621,9 +2506,8 @@ void ShenandoahFreeSet::move_regions_from_collector_to_mutator(size_t max_xfer_r byte_size_in_proper_unit(old_collector_xfer), proper_unit_for_byte_size(old_collector_xfer)); } - // Overwrite arguments to represent the amount of memory in each generation that is about to be recycled -void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_cset_regions, size_t &old_cset_regions, +void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_trashed_regions, size_t &old_trashed_regions, size_t &first_old_region, size_t &last_old_region, size_t &old_region_count) { shenandoah_assert_heaplocked(); // This resets all state information, removing all regions from all sets. @@ -1632,66 +2516,54 @@ void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_cset_regions, size_t &o // This places regions that have alloc_capacity into the old_collector set if they identify as is_old() or the // mutator set otherwise. All trashed (cset) regions are affiliated young and placed in mutator set. - find_regions_with_alloc_capacity(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count); -} - -void ShenandoahFreeSet::establish_generation_sizes(size_t young_region_count, size_t old_region_count) { - assert(young_region_count + old_region_count == ShenandoahHeap::heap()->num_regions(), "Sanity"); - if (ShenandoahHeap::heap()->mode()->is_generational()) { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - ShenandoahOldGeneration* old_gen = heap->old_generation(); - ShenandoahYoungGeneration* young_gen = heap->young_generation(); - size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - - size_t original_old_capacity = old_gen->max_capacity(); - size_t new_old_capacity = old_region_count * region_size_bytes; - size_t new_young_capacity = young_region_count * region_size_bytes; - old_gen->set_capacity(new_old_capacity); - young_gen->set_capacity(new_young_capacity); - - if (new_old_capacity > original_old_capacity) { - size_t region_count = (new_old_capacity - original_old_capacity) / region_size_bytes; - log_info(gc, ergo)("Transfer %zu region(s) from %s to %s, yielding increased size: " PROPERFMT, - region_count, young_gen->name(), old_gen->name(), PROPERFMTARGS(new_old_capacity)); - } else if (new_old_capacity < original_old_capacity) { - size_t region_count = (original_old_capacity - new_old_capacity) / region_size_bytes; - log_info(gc, ergo)("Transfer %zu region(s) from %s to %s, yielding increased size: " PROPERFMT, - region_count, old_gen->name(), young_gen->name(), PROPERFMTARGS(new_young_capacity)); - } - // This balances generations, so clear any pending request to balance. - old_gen->set_region_balance(0); - } + find_regions_with_alloc_capacity(young_trashed_regions, old_trashed_regions, + first_old_region, last_old_region, old_region_count); } -void ShenandoahFreeSet::finish_rebuild(size_t young_cset_regions, size_t old_cset_regions, size_t old_region_count, +void ShenandoahFreeSet::finish_rebuild(size_t young_trashed_regions, size_t old_trashed_regions, size_t old_region_count, bool have_evacuation_reserves) { shenandoah_assert_heaplocked(); size_t young_reserve(0), old_reserve(0); if (_heap->mode()->is_generational()) { - compute_young_and_old_reserves(young_cset_regions, old_cset_regions, have_evacuation_reserves, + compute_young_and_old_reserves(young_trashed_regions, old_trashed_regions, have_evacuation_reserves, young_reserve, old_reserve); } else { young_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve; old_reserve = 0; } - // Move some of the mutator regions in the Collector and OldCollector partitions in order to satisfy + // Move some of the mutator regions into the Collector and OldCollector partitions in order to satisfy // young_reserve and old_reserve. - reserve_regions(young_reserve, old_reserve, old_region_count); - size_t young_region_count = _heap->num_regions() - old_region_count; - establish_generation_sizes(young_region_count, old_region_count); + size_t young_used_regions, old_used_regions, young_used_bytes, old_used_bytes; + reserve_regions(young_reserve, old_reserve, old_region_count, young_used_regions, old_used_regions, + young_used_bytes, old_used_bytes); + _total_young_regions = _heap->num_regions() - old_region_count; + _total_global_regions = _heap->num_regions(); establish_old_collector_alloc_bias(); - _partitions.assert_bounds(); + _partitions.assert_bounds(true); log_status(); } -void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions, size_t old_cset_regions, +/** + * Set young_reserve_result and old_reserve_result to the number of bytes that we desire to set aside to hold the + * results of evacuation to young and old collector spaces respectively during the next evacuation phase. Overwrite + * old_generation region balance in case the original value is incompatible with the current reality. + * + * These values are determined by how much memory is currently available within each generation, which is + * represented by: + * 1. Memory currently available within old and young + * 2. Trashed regions currently residing in young and old, which will become available momentarily + * 3. The value of old_generation->get_region_balance() which represents the number of regions that we plan + * to transfer from old generation to young generation. Prior to each invocation of compute_young_and_old_reserves(), + * this value should computed by ShenandoahGenerationalHeap::compute_old_generation_balance(). + */ +void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_trashed_regions, size_t old_trashed_regions, bool have_evacuation_reserves, size_t& young_reserve_result, size_t& old_reserve_result) const { shenandoah_assert_generational(); + shenandoah_assert_heaplocked(); const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - ShenandoahOldGeneration* const old_generation = _heap->old_generation(); size_t old_available = old_generation->available(); size_t old_unaffiliated_regions = old_generation->free_unaffiliated_regions(); @@ -1700,18 +2572,23 @@ void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions size_t young_unaffiliated_regions = young_generation->free_unaffiliated_regions(); // Add in the regions we anticipate to be freed by evacuation of the collection set - old_unaffiliated_regions += old_cset_regions; - young_unaffiliated_regions += young_cset_regions; + old_unaffiliated_regions += old_trashed_regions; + old_available += old_trashed_regions * region_size_bytes; + young_unaffiliated_regions += young_trashed_regions; // Consult old-region balance to make adjustments to current generation capacities and availability. - // The generation region transfers take place after we rebuild. - const ssize_t old_region_balance = old_generation->get_region_balance(); + // The generation region transfers take place after we rebuild. old_region_balance represents number of regions + // to transfer from old to young. + ssize_t old_region_balance = old_generation->get_region_balance(); if (old_region_balance != 0) { #ifdef ASSERT if (old_region_balance > 0) { - assert(old_region_balance <= checked_cast(old_unaffiliated_regions), "Cannot transfer regions that are affiliated"); + assert(old_region_balance <= checked_cast(old_unaffiliated_regions), + "Cannot transfer %zd regions that are affiliated (old_trashed: %zu, old_unaffiliated: %zu)", + old_region_balance, old_trashed_regions, old_unaffiliated_regions); } else { - assert(0 - old_region_balance <= checked_cast(young_unaffiliated_regions), "Cannot transfer regions that are affiliated"); + assert(0 - old_region_balance <= checked_cast(young_unaffiliated_regions), + "Cannot transfer regions that are affiliated"); } #endif @@ -1732,9 +2609,18 @@ void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions const size_t old_evac_reserve = old_generation->get_evacuation_reserve(); young_reserve_result = young_generation->get_evacuation_reserve(); old_reserve_result = promoted_reserve + old_evac_reserve; - assert(old_reserve_result <= old_available, - "Cannot reserve (%zu + %zu) more OLD than is available: %zu", - promoted_reserve, old_evac_reserve, old_available); + if (old_reserve_result > old_available) { + // Try to transfer memory from young to old. + size_t old_deficit = old_reserve_result - old_available; + size_t old_region_deficit = (old_deficit + region_size_bytes - 1) / region_size_bytes; + if (young_unaffiliated_regions < old_region_deficit) { + old_region_deficit = young_unaffiliated_regions; + } + young_unaffiliated_regions -= old_region_deficit; + old_unaffiliated_regions += old_region_deficit; + old_region_balance -= old_region_deficit; + old_generation->set_region_balance(old_region_balance); + } } else { // We are rebuilding at end of GC, so we set aside budgets specified on command line (or defaults) young_reserve_result = (young_capacity * ShenandoahEvacReserve) / 100; @@ -1764,69 +2650,240 @@ void ShenandoahFreeSet::compute_young_and_old_reserves(size_t young_cset_regions // into the collector set or old collector set in order to assure that the memory available for allocations within // the collector set is at least to_reserve and the memory available for allocations within the old collector set // is at least to_reserve_old. -void ShenandoahFreeSet::reserve_regions(size_t to_reserve, size_t to_reserve_old, size_t &old_region_count) { - for (size_t i = _heap->num_regions(); i > 0; i--) { - size_t idx = i - 1; - ShenandoahHeapRegion* r = _heap->get_region(idx); - if (!_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx)) { - continue; - } +void ShenandoahFreeSet::reserve_regions(size_t to_reserve, size_t to_reserve_old, size_t &old_region_count, + size_t &young_used_regions, size_t &old_used_regions, + size_t &young_used_bytes, size_t &old_used_bytes) { + const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - size_t ac = alloc_capacity(r); - assert (ac > 0, "Membership in free set implies has capacity"); - assert (!r->is_old() || r->is_trash(), "Except for trash, mutator_is_free regions should not be affiliated OLD"); + young_used_regions = 0; + old_used_regions = 0; + young_used_bytes = 0; + old_used_bytes = 0; - bool move_to_old_collector = _partitions.available_in(ShenandoahFreeSetPartitionId::OldCollector) < to_reserve_old; - bool move_to_collector = _partitions.available_in(ShenandoahFreeSetPartitionId::Collector) < to_reserve; + index_type mutator_low_idx = _partitions.max(); + index_type mutator_high_idx = -1; + index_type mutator_empty_low_idx = _partitions.max(); + index_type mutator_empty_high_idx = -1; - if (!move_to_collector && !move_to_old_collector) { - // We've satisfied both to_reserve and to_reserved_old - break; - } + index_type collector_low_idx = _partitions.max(); + index_type collector_high_idx = -1; + index_type collector_empty_low_idx = _partitions.max(); + index_type collector_empty_high_idx = -1; - if (move_to_old_collector) { - // We give priority to OldCollector partition because we desire to pack OldCollector regions into higher - // addresses than Collector regions. Presumably, OldCollector regions are more "stable" and less likely to - // be collected in the near future. - if (r->is_trash() || !r->is_affiliated()) { - // OLD regions that have available memory are already in the old_collector free set. - _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, - ShenandoahFreeSetPartitionId::OldCollector, ac); - log_trace(gc, free)(" Shifting region %zu from mutator_free to old_collector_free", idx); + index_type old_collector_low_idx = _partitions.max(); + index_type old_collector_high_idx = -1; + index_type old_collector_empty_low_idx = _partitions.max(); + index_type old_collector_empty_high_idx = -1; + + size_t used_to_collector = 0; + size_t used_to_old_collector = 0; + size_t regions_to_collector = 0; + size_t regions_to_old_collector = 0; + size_t empty_regions_to_collector = 0; + size_t empty_regions_to_old_collector = 0; + + size_t old_collector_available = _partitions.available_in(ShenandoahFreeSetPartitionId::OldCollector);; + size_t collector_available = _partitions.available_in(ShenandoahFreeSetPartitionId::Collector); + + for (size_t i = _heap->num_regions(); i > 0; i--) { + index_type idx = i - 1; + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx)) { + // Note: trashed regions have region_size_bytes alloc capacity. + size_t ac = alloc_capacity(r); + assert (ac > 0, "Membership in free set implies has capacity"); + assert (!r->is_old() || r->is_trash(), "Except for trash, mutator_is_free regions should not be affiliated OLD"); + + bool move_to_old_collector = old_collector_available < to_reserve_old; + bool move_to_collector = collector_available < to_reserve; + + if (move_to_old_collector) { + // We give priority to OldCollector partition because we desire to pack OldCollector regions into higher + // addresses than Collector regions. Presumably, OldCollector regions are more "stable" and less likely to + // be collected in the near future. + if (r->is_trash() || !r->is_affiliated()) { + // OLD regions that have available memory are already in the old_collector free set. + assert(r->is_empty() || r->is_trash(), "Not affiliated implies region %zu is empty", r->index()); + if (idx < old_collector_low_idx) { + old_collector_low_idx = idx; + } + if (idx > old_collector_high_idx) { + old_collector_high_idx = idx; + } + if (idx < old_collector_empty_low_idx) { + old_collector_empty_low_idx = idx; + } + if (idx > old_collector_empty_high_idx) { + old_collector_empty_high_idx = idx; + } + used_to_old_collector += + _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, ShenandoahFreeSetPartitionId::Mutator, + ShenandoahFreeSetPartitionId::OldCollector, ac); + old_collector_available += ac; + regions_to_old_collector++; + empty_regions_to_old_collector++; + + log_trace(gc, free)(" Shifting region %zu from mutator_free to old_collector_free", idx); + log_trace(gc, free)(" Shifted Mutator range [%zd, %zd]," + " Old Collector range [%zd, %zd]", + _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator), + _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator), + _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector), + _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector)); + old_region_count++; + continue; + } + } + + if (move_to_collector) { + // Note: In a previous implementation, regions were only placed into the survivor space (collector_is_free) if + // they were entirely empty. This has the effect of causing new Mutator allocation to reside next to objects + // that have already survived at least one GC, mixing ephemeral with longer-lived objects in the same region. + // Any objects that have survived a GC are less likely to immediately become garbage, so a region that contains + // survivor objects is less likely to be selected for the collection set. This alternative implementation allows + // survivor regions to continue accumulating other survivor objects, and makes it more likely that ephemeral objects + // occupy regions comprised entirely of ephemeral objects. These regions are highly likely to be included in the next + // collection set, and they are easily evacuated because they have low density of live objects. + if (idx < collector_low_idx) { + collector_low_idx = idx; + } + if (idx > collector_high_idx) { + collector_high_idx = idx; + } + if (ac == region_size_bytes) { + if (idx < collector_empty_low_idx) { + collector_empty_low_idx = idx; + } + if (idx > collector_empty_high_idx) { + collector_empty_high_idx = idx; + } + empty_regions_to_collector++; + } + used_to_collector += + _partitions.move_from_partition_to_partition_with_deferred_accounting(idx, ShenandoahFreeSetPartitionId::Mutator, + ShenandoahFreeSetPartitionId::Collector, ac); + collector_available += ac; + regions_to_collector++; + if (ac != region_size_bytes) { + young_used_regions++; + young_used_bytes = region_size_bytes - ac; + } + + log_trace(gc, free)(" Shifting region %zu from mutator_free to collector_free", idx); log_trace(gc, free)(" Shifted Mutator range [%zd, %zd]," - " Old Collector range [%zd, %zd]", + " Collector range [%zd, %zd]", _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator), _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator), - _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector), - _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector)); - - old_region_count++; + _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector), + _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector)); continue; } - } - - if (move_to_collector) { - // Note: In a previous implementation, regions were only placed into the survivor space (collector_is_free) if - // they were entirely empty. This has the effect of causing new Mutator allocation to reside next to objects - // that have already survived at least one GC, mixing ephemeral with longer-lived objects in the same region. - // Any objects that have survived a GC are less likely to immediately become garbage, so a region that contains - // survivor objects is less likely to be selected for the collection set. This alternative implementation allows - // survivor regions to continue accumulating other survivor objects, and makes it more likely that ephemeral objects - // occupy regions comprised entirely of ephemeral objects. These regions are highly likely to be included in the next - // collection set, and they are easily evacuated because they have low density of live objects. - _partitions.move_from_partition_to_partition(idx, ShenandoahFreeSetPartitionId::Mutator, - ShenandoahFreeSetPartitionId::Collector, ac); - log_trace(gc, free)(" Shifting region %zu from mutator_free to collector_free", idx); - log_trace(gc, free)(" Shifted Mutator range [%zd, %zd]," - " Collector range [%zd, %zd]", - _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator), - _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator), - _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector), - _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector)); + // Mutator region is not moved to Collector or OldCollector. Still, do the accounting. + if (idx < mutator_low_idx) { + mutator_low_idx = idx; + } + if (idx > mutator_high_idx) { + mutator_high_idx = idx; + } + if ((ac == region_size_bytes) && (idx < mutator_empty_low_idx)) { + mutator_empty_low_idx = idx; + } + if ((ac == region_size_bytes) && (idx > mutator_empty_high_idx)) { + mutator_empty_high_idx = idx; + } + if (ac != region_size_bytes) { + young_used_regions++; + young_used_bytes += region_size_bytes - ac; + } + } else { + // Region is not in Mutator partition. Do the accounting. + ShenandoahFreeSetPartitionId p = _partitions.membership(idx); + size_t ac = alloc_capacity(r); + assert(ac != region_size_bytes, "Empty regions should be in Mutator partion at entry to reserve_regions"); + if (p == ShenandoahFreeSetPartitionId::Collector) { + if (ac != region_size_bytes) { + young_used_regions++; + young_used_bytes = region_size_bytes - ac; + } + // else, unaffiliated region has no used + } else if (p == ShenandoahFreeSetPartitionId::OldCollector) { + if (ac != region_size_bytes) { + old_used_regions++; + old_used_bytes = region_size_bytes - ac; + } + // else, unaffiliated region has no used + } else if (p == ShenandoahFreeSetPartitionId::NotFree) { + // This region has been retired + if (r->is_old()) { + old_used_regions++; + old_used_bytes += region_size_bytes - ac; + } else { + assert(r->is_young(), "Retired region should be old or young"); + young_used_regions++; + young_used_bytes += region_size_bytes - ac; + } + } else { + assert(p == ShenandoahFreeSetPartitionId::OldCollector, "Not mutator and not NotFree, so must be OldCollector"); + assert(!r->is_empty(), "Empty regions should be in Mutator partition at entry to reserve_regions"); + if (idx < old_collector_low_idx) { + old_collector_low_idx = idx; + } + if (idx > old_collector_high_idx) { + old_collector_high_idx = idx; + } + if (idx < old_collector_empty_low_idx) { + old_collector_empty_low_idx = idx; + } + if (idx > old_collector_empty_high_idx) { + old_collector_empty_high_idx = idx; + } + } } } + _partitions.decrease_used(ShenandoahFreeSetPartitionId::Mutator, used_to_old_collector + used_to_collector); + _partitions.decrease_region_counts(ShenandoahFreeSetPartitionId::Mutator, regions_to_old_collector + regions_to_collector); + _partitions.decrease_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator, + empty_regions_to_old_collector + empty_regions_to_collector); + _partitions.decrease_total_region_counts(ShenandoahFreeSetPartitionId::Mutator, + regions_to_old_collector + regions_to_collector); + // decrease_capacity() also decreases available + _partitions.decrease_capacity(ShenandoahFreeSetPartitionId::Mutator, + (regions_to_old_collector + regions_to_collector) * region_size_bytes); + + _partitions.increase_total_region_counts(ShenandoahFreeSetPartitionId::Collector, regions_to_collector); + // increase_capacity() also increases available + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::Collector, regions_to_collector * region_size_bytes); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::Collector, regions_to_collector); + _partitions.increase_empty_region_counts(ShenandoahFreeSetPartitionId::Collector, empty_regions_to_collector); + + _partitions.increase_total_region_counts(ShenandoahFreeSetPartitionId::OldCollector, regions_to_old_collector); + // increase_capacity() also increases available + _partitions.increase_capacity(ShenandoahFreeSetPartitionId::OldCollector, regions_to_old_collector * region_size_bytes); + _partitions.increase_region_counts(ShenandoahFreeSetPartitionId::OldCollector, regions_to_old_collector); + _partitions.increase_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector, empty_regions_to_old_collector); + + if (used_to_collector > 0) { + _partitions.increase_used(ShenandoahFreeSetPartitionId::Collector, used_to_collector); + } + + if (used_to_old_collector > 0) { + _partitions.increase_used(ShenandoahFreeSetPartitionId::OldCollector, used_to_old_collector); + } + + _partitions.expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Collector, + collector_low_idx, collector_high_idx, + collector_empty_low_idx, collector_empty_high_idx); + _partitions.expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::OldCollector, + old_collector_low_idx, old_collector_high_idx, + old_collector_empty_low_idx, old_collector_empty_high_idx); + _partitions.establish_interval(ShenandoahFreeSetPartitionId::Mutator, + mutator_low_idx, mutator_high_idx, mutator_empty_low_idx, mutator_empty_high_idx); + + recompute_total_used(); + recompute_total_affiliated(); + _partitions.assert_bounds(true); if (LogTarget(Info, gc, free)::is_enabled()) { size_t old_reserve = _partitions.available_in(ShenandoahFreeSetPartitionId::OldCollector); if (old_reserve < to_reserve_old) { @@ -1845,19 +2902,19 @@ void ShenandoahFreeSet::establish_old_collector_alloc_bias() { ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_heaplocked(); - idx_t left_idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector); - idx_t right_idx = _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector); - idx_t middle = (left_idx + right_idx) / 2; + index_type left_idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector); + index_type right_idx = _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector); + index_type middle = (left_idx + right_idx) / 2; size_t available_in_first_half = 0; size_t available_in_second_half = 0; - for (idx_t index = left_idx; index < middle; index++) { + for (index_type index = left_idx; index < middle; index++) { if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, index)) { ShenandoahHeapRegion* r = heap->get_region((size_t) index); available_in_first_half += r->free(); } } - for (idx_t index = middle; index <= right_idx; index++) { + for (index_type index = middle; index <= right_idx; index++) { if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, index)) { ShenandoahHeapRegion* r = heap->get_region(index); available_in_second_half += r->free(); @@ -1959,7 +3016,7 @@ void ShenandoahFreeSet::log_status() { LogStream ls(lt); { - idx_t last_idx = 0; + index_type last_idx = 0; size_t max = 0; size_t max_contig = 0; size_t empty_contig = 0; @@ -1967,14 +3024,17 @@ void ShenandoahFreeSet::log_status() { size_t total_used = 0; size_t total_free = 0; size_t total_free_ext = 0; + size_t total_trashed_free = 0; - for (idx_t idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator); + for (index_type idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::Mutator); idx <= _partitions.rightmost(ShenandoahFreeSetPartitionId::Mutator); idx++) { if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::Mutator, idx)) { ShenandoahHeapRegion *r = _heap->get_region(idx); size_t free = alloc_capacity(r); max = MAX2(max, free); - if (r->is_empty()) { + size_t used_in_region = r->used(); + if (r->is_empty() || r->is_trash()) { + used_in_region = 0; total_free_ext += free; if (last_idx + 1 == idx) { empty_contig++; @@ -1984,7 +3044,7 @@ void ShenandoahFreeSet::log_status() { } else { empty_contig = 0; } - total_used += r->used(); + total_used += used_in_region; total_free += free; max_contig = MAX2(max_contig, empty_contig); last_idx = idx; @@ -1992,12 +3052,13 @@ void ShenandoahFreeSet::log_status() { } size_t max_humongous = max_contig * ShenandoahHeapRegion::region_size_bytes(); + // capacity() is capacity of mutator + // used() is used of mutator size_t free = capacity() - used(); - // Since certain regions that belonged to the Mutator free partition at the time of most recent rebuild may have been // retired, the sum of used and capacities within regions that are still in the Mutator free partition may not match // my internally tracked values of used() and free(). - assert(free == total_free, "Free memory should match"); + assert(free == total_free, "Free memory (%zu) should match calculated memory (%zu)", free, total_free); ls.print("Free: %zu%s, Max: %zu%s regular, %zu%s humongous, ", byte_size_in_proper_unit(total_free), proper_unit_for_byte_size(total_free), byte_size_in_proper_unit(max), proper_unit_for_byte_size(max), @@ -2031,7 +3092,7 @@ void ShenandoahFreeSet::log_status() { size_t total_free = 0; size_t total_used = 0; - for (idx_t idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector); + for (index_type idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::Collector); idx <= _partitions.rightmost(ShenandoahFreeSetPartitionId::Collector); idx++) { if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::Collector, idx)) { ShenandoahHeapRegion *r = _heap->get_region(idx); @@ -2052,7 +3113,7 @@ void ShenandoahFreeSet::log_status() { size_t total_free = 0; size_t total_used = 0; - for (idx_t idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector); + for (index_type idx = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector); idx <= _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector); idx++) { if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, idx)) { ShenandoahHeapRegion *r = _heap->get_region(idx); @@ -2070,6 +3131,21 @@ void ShenandoahFreeSet::log_status() { } } +void ShenandoahFreeSet::decrease_humongous_waste_for_regular_bypass(ShenandoahHeapRegion*r, size_t waste) { + shenandoah_assert_heaplocked(); + assert(_partitions.membership(r->index()) == ShenandoahFreeSetPartitionId::NotFree, "Humongous regions should be NotFree"); + ShenandoahFreeSetPartitionId p = + r->is_old()? ShenandoahFreeSetPartitionId::OldCollector: ShenandoahFreeSetPartitionId::Mutator; + _partitions.decrease_humongous_waste(p, waste); + if (waste >= PLAB::min_size() * HeapWordSize) { + _partitions.decrease_used(p, waste); + _partitions.unretire_to_partition(r, p); + } + _total_humongous_waste -= waste; + recompute_total_used(); +} + + HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_region) { shenandoah_assert_heaplocked(); if (ShenandoahHeapRegion::requires_humongous(req.size())) { @@ -2099,19 +3175,19 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_ void ShenandoahFreeSet::print_on(outputStream* out) const { out->print_cr("Mutator Free Set: %zu", _partitions.count(ShenandoahFreeSetPartitionId::Mutator)); ShenandoahLeftRightIterator mutator(const_cast(&_partitions), ShenandoahFreeSetPartitionId::Mutator); - for (idx_t index = mutator.current(); mutator.has_next(); index = mutator.next()) { + for (index_type index = mutator.current(); mutator.has_next(); index = mutator.next()) { _heap->get_region(index)->print_on(out); } out->print_cr("Collector Free Set: %zu", _partitions.count(ShenandoahFreeSetPartitionId::Collector)); ShenandoahLeftRightIterator collector(const_cast(&_partitions), ShenandoahFreeSetPartitionId::Collector); - for (idx_t index = collector.current(); collector.has_next(); index = collector.next()) { + for (index_type index = collector.current(); collector.has_next(); index = collector.next()) { _heap->get_region(index)->print_on(out); } if (_heap->mode()->is_generational()) { out->print_cr("Old Collector Free Set: %zu", _partitions.count(ShenandoahFreeSetPartitionId::OldCollector)); - for (idx_t index = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector); + for (index_type index = _partitions.leftmost(ShenandoahFreeSetPartitionId::OldCollector); index <= _partitions.rightmost(ShenandoahFreeSetPartitionId::OldCollector); index++) { if (_partitions.in_free_set(ShenandoahFreeSetPartitionId::OldCollector, index)) { _heap->get_region(index)->print_on(out); @@ -2125,7 +3201,7 @@ double ShenandoahFreeSet::internal_fragmentation() { double linear = 0; ShenandoahLeftRightIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::Mutator); - for (idx_t index = iterator.current(); iterator.has_next(); index = iterator.next()) { + for (index_type index = iterator.current(); iterator.has_next(); index = iterator.next()) { ShenandoahHeapRegion* r = _heap->get_region(index); size_t used = r->used(); squared += used * used; @@ -2141,13 +3217,13 @@ double ShenandoahFreeSet::internal_fragmentation() { } double ShenandoahFreeSet::external_fragmentation() { - idx_t last_idx = 0; + index_type last_idx = 0; size_t max_contig = 0; size_t empty_contig = 0; size_t free = 0; ShenandoahLeftRightIterator iterator(&_partitions, ShenandoahFreeSetPartitionId::Mutator); - for (idx_t index = iterator.current(); iterator.has_next(); index = iterator.next()) { + for (index_type index = iterator.current(); iterator.has_next(); index = iterator.next()) { ShenandoahHeapRegion* r = _heap->get_region(index); if (r->is_empty()) { free += ShenandoahHeapRegion::region_size_bytes(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 8ad7055b3d6a4..e7b52b02f840d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -35,8 +35,12 @@ enum class ShenandoahFreeSetPartitionId : uint8_t { Mutator, // Region is in the Mutator free set: available memory is available to mutators. Collector, // Region is in the Collector free set: available memory is reserved for evacuations. OldCollector, // Region is in the Old Collector free set: - // available memory is reserved for old evacuations and for promotions.. - NotFree // Region is in no free set: it has no available memory + // available memory is reserved for old evacuations and for promotions. + NotFree // Region is in no free set: it has no available memory. Consult region affiliation + // to determine whether this retired region is young or old. If young, the region + // is considered to be part of the Mutator partition. (When we retire from the + // Collector partition, we decrease total_region_count for Collector and increaese + // for Mutator, making similar adjustments to used (net impact on available is neutral). }; // ShenandoahRegionPartitions provides an abstraction to help organize the implementation of ShenandoahFreeSet. This @@ -45,64 +49,92 @@ enum class ShenandoahFreeSetPartitionId : uint8_t { // for which the ShenandoahFreeSetPartitionId is not equal to NotFree. class ShenandoahRegionPartitions { -private: +public: // We do not maintain counts, capacity, or used for regions that are not free. Informally, if a region is NotFree, it is // in no partition. NumPartitions represents the size of an array that may be indexed by Mutator or Collector. static constexpr ShenandoahFreeSetPartitionId NumPartitions = ShenandoahFreeSetPartitionId::NotFree; static constexpr int IntNumPartitions = int(ShenandoahFreeSetPartitionId::NotFree); static constexpr uint UIntNumPartitions = uint(ShenandoahFreeSetPartitionId::NotFree); - const ssize_t _max; // The maximum number of heap regions +private: + const index_type _max; // The maximum number of heap regions const size_t _region_size_bytes; const ShenandoahFreeSet* _free_set; // For each partition, we maintain a bitmap of which regions are affiliated with his partition. ShenandoahSimpleBitMap _membership[UIntNumPartitions]; - // For each partition, we track an interval outside of which a region affiliated with that partition is guaranteed // not to be found. This makes searches for free space more efficient. For each partition p, _leftmosts[p] // represents its least index, and its _rightmosts[p] its greatest index. Empty intervals are indicated by the // canonical [_max, -1]. - ssize_t _leftmosts[UIntNumPartitions]; - ssize_t _rightmosts[UIntNumPartitions]; + index_type _leftmosts[UIntNumPartitions]; + index_type _rightmosts[UIntNumPartitions]; // Allocation for humongous objects needs to find regions that are entirely empty. For each partion p, _leftmosts_empty[p] // represents the first region belonging to this partition that is completely empty and _rightmosts_empty[p] represents the // last region that is completely empty. If there is no completely empty region in this partition, this is represented // by the canonical [_max, -1]. - ssize_t _leftmosts_empty[UIntNumPartitions]; - ssize_t _rightmosts_empty[UIntNumPartitions]; - - // For each partition p, _capacity[p] represents the total amount of memory within the partition at the time - // of the most recent rebuild, _used[p] represents the total amount of memory that has been allocated within this - // partition (either already allocated as of the rebuild, or allocated since the rebuild). _capacity[p] and _used[p] - // are denoted in bytes. Note that some regions that had been assigned to a particular partition at rebuild time - // may have been retired following the rebuild. The tallies for these regions are still reflected in _capacity[p] - // and _used[p], even though the region may have been removed from the free set. + index_type _leftmosts_empty[UIntNumPartitions]; + index_type _rightmosts_empty[UIntNumPartitions]; + + // For each partition p: + // _capacity[p] represents the total amount of memory within the partition, including retired regions, as adjusted + // by transfers of memory between partitions + // _used[p] represents the total amount of memory that has been allocated within this partition (either already + // allocated as of the rebuild, or allocated since the rebuild). + // _available[p] represents the total amount of memory that can be allocated within partition p, calculated from + // _capacity[p] minus _used[p], where the difference is computed and assigned under heap lock + // + // Unlike capacity, which represents the total amount of memory representing each partition as of the moment + // the freeset was most recently constructed: + // + // _region_counts[p] represents the number of regions associated with the partition which currently have available memory. + // When a region is retired from partition p, _region_counts[p] is decremented. + // _total_region_counts[p] is _total_capacity[p] / RegionSizeBytes. + // _empty_region_counts[p] is number of regions associated with p which are entirely empty + // + // capacity and used values are expressed in bytes. + // + // When a region is retired, the used[p] is increased to account for alignment waste. capacity is unaffected. + // + // When a region is "flipped", we adjust capacities and region counts for original and destination partitions. We also + // adjust used values when flipping from mutator to collector. Flip to old collector does not need to adjust used because + // only empty regions can be flipped to old collector. + // + // All memory quantities (capacity, available, used) are represented in bytes. + size_t _capacity[UIntNumPartitions]; + size_t _used[UIntNumPartitions]; size_t _available[UIntNumPartitions]; + + // Measured in bytes. + size_t _allocated_since_gc_start[UIntNumPartitions]; + + // Some notes: + // _retired_regions[p] is _total_region_counts[p] - _region_counts[p] + // _empty_region_counts[p] <= _region_counts[p] <= _total_region_counts[p] + // generation_used is (_total_region_counts[p] - _region_counts[p]) * region_size_bytes + _used[p] + // (affiliated regions is total_region_counts - empty_region_counts) size_t _region_counts[UIntNumPartitions]; + size_t _total_region_counts[UIntNumPartitions]; + size_t _empty_region_counts[UIntNumPartitions]; + + // Humongous waste, in bytes, can exist in Mutator partition for recently allocated humongous objects + // and in OldCollector partition for humongous objects that have been promoted in place. + size_t _humongous_waste[UIntNumPartitions]; // For each partition p, _left_to_right_bias is true iff allocations are normally made from lower indexed regions // before higher indexed regions. bool _left_to_right_bias[UIntNumPartitions]; - // Shrink the intervals associated with partition when region idx is removed from this free set - inline void shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, ssize_t idx); - - // Shrink the intervals associated with partition when regions low_idx through high_idx inclusive are removed from this free set - inline void shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, - ssize_t low_idx, ssize_t high_idx); - inline void expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, ssize_t idx, size_t capacity); - inline bool is_mutator_partition(ShenandoahFreeSetPartitionId p); inline bool is_young_collector_partition(ShenandoahFreeSetPartitionId p); inline bool is_old_collector_partition(ShenandoahFreeSetPartitionId p); inline bool available_implies_empty(size_t available); #ifndef PRODUCT - void dump_bitmap_row(ssize_t region_idx) const; - void dump_bitmap_range(ssize_t start_region_idx, ssize_t end_region_idx) const; + void dump_bitmap_row(index_type region_idx) const; + void dump_bitmap_range(index_type start_region_idx, index_type end_region_idx) const; void dump_bitmap() const; #endif public: @@ -111,6 +143,11 @@ class ShenandoahRegionPartitions { static const size_t FreeSetUnderConstruction = SIZE_MAX; + inline index_type max() const { return _max; } + + // At initialization, reset OldCollector tallies + void initialize_old_collector(); + // Remove all regions from all partitions and reset all bounds void make_all_regions_unavailable(); @@ -119,70 +156,113 @@ class ShenandoahRegionPartitions { _membership[int(p)].set_bit(idx); } + // Clear the partition id for a particular region without adjusting interval bounds or usage/capacity tallies + inline void raw_clear_membership(size_t idx, ShenandoahFreeSetPartitionId p) { + _membership[int(p)].clear_bit(idx); + } + + inline void one_region_is_no_longer_empty(ShenandoahFreeSetPartitionId partition); + // Set the Mutator intervals, usage, and capacity according to arguments. Reset the Collector intervals, used, capacity // to represent empty Collector free set. We use this at the end of rebuild_free_set() to avoid the overhead of making // many redundant incremental adjustments to the mutator intervals as the free set is being rebuilt. - void establish_mutator_intervals(ssize_t mutator_leftmost, ssize_t mutator_rightmost, - ssize_t mutator_leftmost_empty, ssize_t mutator_rightmost_empty, - size_t mutator_region_count, size_t mutator_used); + void establish_mutator_intervals(index_type mutator_leftmost, index_type mutator_rightmost, + index_type mutator_leftmost_empty, index_type mutator_rightmost_empty, + size_t total_mutator_regions, size_t empty_mutator_regions, + size_t mutator_region_count, size_t mutator_used, size_t mutator_humongous_words_waste); // Set the OldCollector intervals, usage, and capacity according to arguments. We use this at the end of rebuild_free_set() // to avoid the overhead of making many redundant incremental adjustments to the mutator intervals as the free set is being // rebuilt. - void establish_old_collector_intervals(ssize_t old_collector_leftmost, ssize_t old_collector_rightmost, - ssize_t old_collector_leftmost_empty, ssize_t old_collector_rightmost_empty, - size_t old_collector_region_count, size_t old_collector_used); + void establish_old_collector_intervals(index_type old_collector_leftmost, index_type old_collector_rightmost, + index_type old_collector_leftmost_empty, index_type old_collector_rightmost_empty, + size_t total_old_collector_region_count, size_t old_collector_empty, + size_t old_collector_regions, size_t old_collector_used, + size_t old_collector_humongous_words_waste); + + void establish_interval(ShenandoahFreeSetPartitionId partition, index_type low_idx, index_type high_idx, + index_type low_empty_idx, index_type high_empty_idx); + + // Shrink the intervals associated with partition when region idx is removed from this free set + inline void shrink_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, index_type idx); + + // Shrink the intervals associated with partition when regions low_idx through high_idx inclusive are removed from this free set + void shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, + index_type low_idx, index_type high_idx, size_t num_regions); + + void expand_interval_if_boundary_modified(ShenandoahFreeSetPartitionId partition, index_type idx, size_t capacity); + void expand_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, + index_type low_idx, index_type high_idx, + index_type low_empty_idx, index_type high_empty_idx); // Retire region idx from within partition, , leaving its capacity and used as part of the original free partition's totals. // Requires that region idx is in in the Mutator or Collector partitions. Hereafter, identifies this region as NotFree. // Any remnant of available memory at the time of retirement is added to the original partition's total of used bytes. - void retire_from_partition(ShenandoahFreeSetPartitionId p, ssize_t idx, size_t used_bytes); + // Return the number of waste bytes (if any). + size_t retire_from_partition(ShenandoahFreeSetPartitionId p, index_type idx, size_t used_bytes); // Retire all regions between low_idx and high_idx inclusive from within partition. Requires that each region idx is // in the same Mutator or Collector partition. Hereafter, identifies each region as NotFree. Assumes that each region // is now considered fully used, since the region is presumably used to represent a humongous object. - void retire_range_from_partition(ShenandoahFreeSetPartitionId partition, ssize_t low_idx, ssize_t high_idx); + void retire_range_from_partition(ShenandoahFreeSetPartitionId partition, index_type low_idx, index_type high_idx); + + void unretire_to_partition(ShenandoahHeapRegion* region, ShenandoahFreeSetPartitionId which_partition); // Place region idx into free set which_partition. Requires that idx is currently NotFree. - void make_free(ssize_t idx, ShenandoahFreeSetPartitionId which_partition, size_t region_capacity); + void make_free(index_type idx, ShenandoahFreeSetPartitionId which_partition, size_t region_capacity); - // Place region idx into free partition new_partition, adjusting used and capacity totals for the original and new partition - // given that available bytes can still be allocated within this region. Requires that idx is currently not NotFree. - void move_from_partition_to_partition(ssize_t idx, ShenandoahFreeSetPartitionId orig_partition, + // Place region idx into free partition new_partition, not adjusting used and capacity totals for the original and new partition. + // available represents bytes that can still be allocated within this region. Requires that idx is currently not NotFree. + size_t move_from_partition_to_partition_with_deferred_accounting(index_type idx, ShenandoahFreeSetPartitionId orig_partition, + ShenandoahFreeSetPartitionId new_partition, size_t available); + + // Place region idx into free partition new_partition, adjusting used and capacity totals for the original and new partition. + // available represents bytes that can still be allocated within this region. Requires that idx is currently not NotFree. + void move_from_partition_to_partition(index_type idx, ShenandoahFreeSetPartitionId orig_partition, ShenandoahFreeSetPartitionId new_partition, size_t available); - const char* partition_membership_name(ssize_t idx) const; + const char* partition_membership_name(index_type idx) const; // Return the index of the next available region >= start_index, or maximum_regions if not found. - inline ssize_t find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition, ssize_t start_index) const; + inline index_type find_index_of_next_available_region(ShenandoahFreeSetPartitionId which_partition, + index_type start_index) const; // Return the index of the previous available region <= last_index, or -1 if not found. - inline ssize_t find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition, ssize_t last_index) const; + inline index_type find_index_of_previous_available_region(ShenandoahFreeSetPartitionId which_partition, + index_type last_index) const; // Return the index of the next available cluster of cluster_size regions >= start_index, or maximum_regions if not found. - inline ssize_t find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition, - ssize_t start_index, size_t cluster_size) const; + inline index_type find_index_of_next_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition, + index_type start_index, size_t cluster_size) const; // Return the index of the previous available cluster of cluster_size regions <= last_index, or -1 if not found. - inline ssize_t find_index_of_previous_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition, - ssize_t last_index, size_t cluster_size) const; + inline index_type find_index_of_previous_available_cluster_of_regions(ShenandoahFreeSetPartitionId which_partition, + index_type last_index, size_t cluster_size) const; - inline bool in_free_set(ShenandoahFreeSetPartitionId which_partition, ssize_t idx) const { + inline bool in_free_set(ShenandoahFreeSetPartitionId which_partition, index_type idx) const { return _membership[int(which_partition)].is_set(idx); } // Returns the ShenandoahFreeSetPartitionId affiliation of region idx, NotFree if this region is not currently in any partition. // This does not enforce that free_set membership implies allocation capacity. - inline ShenandoahFreeSetPartitionId membership(ssize_t idx) const; + inline ShenandoahFreeSetPartitionId membership(index_type idx) const { + assert (idx < _max, "index is sane: %zu < %zu", idx, _max); + ShenandoahFreeSetPartitionId result = ShenandoahFreeSetPartitionId::NotFree; + for (uint partition_id = 0; partition_id < UIntNumPartitions; partition_id++) { + if (_membership[partition_id].is_set(idx)) { + assert(result == ShenandoahFreeSetPartitionId::NotFree, "Region should reside in only one partition"); + result = (ShenandoahFreeSetPartitionId) partition_id; + } + } + return result; + } #ifdef ASSERT // Returns true iff region idx's membership is which_partition. If which_partition represents a free set, asserts // that the region has allocation capacity. - inline bool partition_id_matches(ssize_t idx, ShenandoahFreeSetPartitionId which_partition) const; + inline bool partition_id_matches(index_type idx, ShenandoahFreeSetPartitionId which_partition) const; #endif - inline size_t max_regions() const { return _max; } - inline size_t region_size_bytes() const { return _region_size_bytes; }; // The following four methods return the left-most and right-most bounds on ranges of regions representing @@ -192,14 +272,62 @@ class ShenandoahRegionPartitions { // leftmost() and leftmost_empty() return _max, rightmost() and rightmost_empty() return 0 // otherwise, expect the following: // 0 <= leftmost <= leftmost_empty <= rightmost_empty <= rightmost < _max - inline ssize_t leftmost(ShenandoahFreeSetPartitionId which_partition) const; - inline ssize_t rightmost(ShenandoahFreeSetPartitionId which_partition) const; - ssize_t leftmost_empty(ShenandoahFreeSetPartitionId which_partition); - ssize_t rightmost_empty(ShenandoahFreeSetPartitionId which_partition); + inline index_type leftmost(ShenandoahFreeSetPartitionId which_partition) const; + inline index_type rightmost(ShenandoahFreeSetPartitionId which_partition) const; + index_type leftmost_empty(ShenandoahFreeSetPartitionId which_partition); + index_type rightmost_empty(ShenandoahFreeSetPartitionId which_partition); inline bool is_empty(ShenandoahFreeSetPartitionId which_partition) const; + inline void increase_total_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline void decrease_total_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline size_t get_total_region_counts(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "selected free set must be valid"); + return _total_region_counts[int(which_partition)]; + } + + inline void increase_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline void decrease_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline size_t get_region_counts(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "selected free set must be valid"); + return _region_counts[int(which_partition)]; + } + + inline void increase_empty_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline void decrease_empty_region_counts(ShenandoahFreeSetPartitionId which_partition, size_t regions); + inline size_t get_empty_region_counts(ShenandoahFreeSetPartitionId which_partition) { + assert (which_partition < NumPartitions, "selected free set must be valid"); + return _empty_region_counts[int(which_partition)]; + } + + inline void increase_capacity(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline void decrease_capacity(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline size_t get_capacity(ShenandoahFreeSetPartitionId which_partition); + + inline void increase_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline void decrease_available(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline size_t get_available(ShenandoahFreeSetPartitionId which_partition); + inline void increase_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline void decrease_used(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + assert (_used[int(which_partition)] >= bytes, "Must not use less than zero after decrease"); + _used[int(which_partition)] -= bytes; + _available[int(which_partition)] += bytes; + } + + inline size_t get_used(ShenandoahFreeSetPartitionId which_partition); + + inline void increase_humongous_waste(ShenandoahFreeSetPartitionId which_partition, size_t bytes); + inline void decrease_humongous_waste(ShenandoahFreeSetPartitionId which_partition, size_t bytes) { + shenandoah_assert_heaplocked(); + assert (which_partition < NumPartitions, "Partition must be valid"); + assert(_humongous_waste[int(which_partition)] >= bytes, "Cannot decrease waste beyond what is there"); + _humongous_waste[int(which_partition)] -= bytes; + } + + inline size_t get_humongous_waste(ShenandoahFreeSetPartitionId which_partition); inline void set_bias_from_left_to_right(ShenandoahFreeSetPartitionId which_partition, bool value) { assert (which_partition < NumPartitions, "selected free set must be valid"); @@ -227,10 +355,17 @@ class ShenandoahRegionPartitions { assert(_available[int(which_partition)] == _capacity[int(which_partition)] - _used[int(which_partition)], "Expect available (%zu) equals capacity (%zu) - used (%zu) for partition %s", _available[int(which_partition)], _capacity[int(which_partition)], _used[int(which_partition)], - partition_membership_name(ssize_t(which_partition))); + partition_membership_name(index_type(which_partition))); return _available[int(which_partition)]; } + // Returns bytes of humongous waste + inline size_t humongous_waste(ShenandoahFreeSetPartitionId which_partition) const { + assert (which_partition < NumPartitions, "selected free set must be valid"); + // This may be called with or without the global heap lock. Changes to _humongous_waste[] are always made with heap lock. + return _humongous_waste[int(which_partition)]; + } + // Return available_in assuming caller does not hold the heap lock. In production builds, available is // returned without acquiring the lock. In debug builds, the global heap lock is acquired in order to // enforce a consistency assert. @@ -243,17 +378,12 @@ class ShenandoahRegionPartitions { (_available[int(which_partition)] == _capacity[int(which_partition)] - _used[int(which_partition)]), "Expect available (%zu) equals capacity (%zu) - used (%zu) for partition %s", _available[int(which_partition)], _capacity[int(which_partition)], _used[int(which_partition)], - partition_membership_name(ssize_t(which_partition))); + partition_membership_name(index_type(which_partition))); #endif return _available[int(which_partition)]; } - inline void set_capacity_of(ShenandoahFreeSetPartitionId which_partition, size_t value) { - shenandoah_assert_heaplocked(); - assert (which_partition < NumPartitions, "selected free set must be valid"); - _capacity[int(which_partition)] = value; - _available[int(which_partition)] = value - _used[int(which_partition)]; - } + inline void set_capacity_of(ShenandoahFreeSetPartitionId which_partition, size_t value); inline void set_used_by(ShenandoahFreeSetPartitionId which_partition, size_t value) { shenandoah_assert_heaplocked(); @@ -284,7 +414,7 @@ class ShenandoahRegionPartitions { // idx >= leftmost && // idx <= rightmost // } - void assert_bounds() NOT_DEBUG_RETURN; + void assert_bounds(bool validate_totals) NOT_DEBUG_RETURN; }; // Publicly, ShenandoahFreeSet represents memory that is available to mutator threads. The public capacity(), used(), @@ -316,6 +446,8 @@ class ShenandoahFreeSet : public CHeapObj { ShenandoahHeap* const _heap; ShenandoahRegionPartitions _partitions; + size_t _total_humongous_waste; + HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r); // Return the address of memory allocated, setting in_new_region to true iff the allocation is taken @@ -330,6 +462,70 @@ class ShenandoahFreeSet : public CHeapObj { const ssize_t INITIAL_ALLOC_BIAS_WEIGHT = 256; + // bytes used by young + size_t _total_young_used; + inline void recompute_total_young_used() { + shenandoah_assert_heaplocked(); + size_t region_size_bytes = _partitions.region_size_bytes(); + _total_young_used = (_partitions.used_by(ShenandoahFreeSetPartitionId::Mutator) + + _partitions.used_by(ShenandoahFreeSetPartitionId::Collector)); + } + + // bytes used by old + size_t _total_old_used; + inline void recompute_total_old_used() { + shenandoah_assert_heaplocked(); + size_t region_size_bytes = _partitions.region_size_bytes(); + _total_old_used =_partitions.used_by(ShenandoahFreeSetPartitionId::OldCollector); + } + + // bytes used by global + size_t _total_global_used; + // Prerequisite: _total_young_used and _total_old_used are valid + inline void recompute_total_global_used() { + shenandoah_assert_heaplocked(); + _total_global_used = _total_young_used + _total_old_used; + } + + inline void recompute_total_used() { + recompute_total_young_used(); + recompute_total_old_used(); + recompute_total_global_used(); + } + + size_t _young_affiliated_regions; + size_t _old_affiliated_regions; + size_t _global_affiliated_regions; + + size_t _young_unaffiliated_regions; + size_t _global_unaffiliated_regions; + + size_t _total_young_regions; + size_t _total_global_regions; + + size_t _mutator_bytes_allocated_since_gc_start; + + inline void recompute_total_affiliated() { + shenandoah_assert_heaplocked(); + _young_unaffiliated_regions = (_partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Mutator) + + _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::Collector)); + _young_affiliated_regions = ((_partitions.get_total_region_counts(ShenandoahFreeSetPartitionId::Mutator) + + _partitions.get_total_region_counts(ShenandoahFreeSetPartitionId::Collector)) - + _young_unaffiliated_regions); + _old_affiliated_regions = (_partitions.get_total_region_counts(ShenandoahFreeSetPartitionId::OldCollector) - + _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector)); + _global_unaffiliated_regions = + _young_unaffiliated_regions + _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector); + _global_affiliated_regions = _young_affiliated_regions + _old_affiliated_regions; +#ifdef ASSERT + if (ShenandoahHeap::heap()->mode()->is_generational()) { + assert(_young_affiliated_regions * ShenandoahHeapRegion::region_size_bytes() >= _total_young_used, "sanity"); + assert(_old_affiliated_regions * ShenandoahHeapRegion::region_size_bytes() >= _total_old_used, "sanity"); + } + assert(_global_affiliated_regions * ShenandoahHeapRegion::region_size_bytes() >= _total_global_used, "sanity"); +#endif + } + // Increases used memory for the partition if the allocation is successful. `in_new_region` will be set // if this is the first allocation in the region. HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region); @@ -347,6 +543,8 @@ class ShenandoahFreeSet : public CHeapObj { // Precondition: ShenandoahHeapRegion::requires_humongous(req.size()) HeapWord* allocate_contiguous(ShenandoahAllocRequest& req, bool is_humongous); + bool transfer_one_region_from_mutator_to_old_collector(size_t idx, size_t alloc_capacity); + // Change region r from the Mutator partition to the GC's Collector or OldCollector partition. This requires that the // region is entirely empty. // @@ -374,7 +572,8 @@ class ShenandoahFreeSet : public CHeapObj { // Search for allocation in region with same affiliation as request, using given iterator. template - HeapWord* allocate_with_affiliation(Iter& iterator, ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_with_affiliation(Iter& iterator, ShenandoahAffiliation affiliation, + ShenandoahAllocRequest& req, bool& in_new_region); // Return true if the respective generation for this request has free regions. bool can_allocate_in_new_region(const ShenandoahAllocRequest& req); @@ -392,6 +591,10 @@ class ShenandoahFreeSet : public CHeapObj { inline bool has_alloc_capacity(ShenandoahHeapRegion *r) const; + void transfer_empty_regions_from_to(ShenandoahFreeSetPartitionId source_partition, + ShenandoahFreeSetPartitionId dest_partition, + size_t num_regions); + size_t transfer_empty_regions_from_collector_set_to_mutator_set(ShenandoahFreeSetPartitionId which_collector, size_t max_xfer_regions, size_t& bytes_transferred); @@ -399,12 +602,8 @@ class ShenandoahFreeSet : public CHeapObj { size_t max_xfer_regions, size_t& bytes_transferred); - // Determine whether we prefer to allocate from left to right or from right to left within the OldCollector free-set. void establish_old_collector_alloc_bias(); - - // Set max_capacity for young and old generations - void establish_generation_sizes(size_t young_region_count, size_t old_region_count); size_t get_usable_free_words(size_t free_bytes) const; // log status, assuming lock has already been acquired by the caller. @@ -415,10 +614,81 @@ class ShenandoahFreeSet : public CHeapObj { ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions); + inline size_t max_regions() const { return _partitions.max(); } + ShenandoahFreeSetPartitionId membership(size_t index) const { return _partitions.membership(index); } + inline void shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId partition, + index_type low_idx, index_type high_idx, size_t num_regions) { + return _partitions.shrink_interval_if_range_modifies_either_boundary(partition, low_idx, high_idx, num_regions); + } + + void reset_bytes_allocated_since_gc_start(); + + void increase_bytes_allocated(size_t bytes); + + inline size_t get_bytes_allocated_since_gc_start() const { + return _mutator_bytes_allocated_since_gc_start; + } + // Public because ShenandoahRegionPartitions assertions require access. inline size_t alloc_capacity(ShenandoahHeapRegion *r) const; inline size_t alloc_capacity(size_t idx) const; + // Return bytes used by old + inline size_t old_used() { + return _total_old_used; + } + + void prepare_to_promote_in_place(size_t idx, size_t bytes); + + // This is used for unit testing. Not for preoduction. Invokes exit() if old cannot be resized. + void resize_old_collector_capacity(size_t desired_regions); + + // Return bytes used by young + inline size_t young_used() { + return _total_young_used; + } + + // Return bytes used by global + inline size_t global_used() { + return _total_global_used; + } + + size_t global_unaffiliated_regions() { + return _global_unaffiliated_regions; + } + + size_t young_unaffiliated_regions() { + return _young_unaffiliated_regions; + } + + size_t old_unaffiliated_regions() { + return _partitions.get_empty_region_counts(ShenandoahFreeSetPartitionId::OldCollector); + } + + size_t young_affiliated_regions() { + return _young_affiliated_regions; + } + + size_t old_affiliated_regions() { + return _old_affiliated_regions; + } + + size_t global_affiliated_regions() { + return _global_affiliated_regions; + } + + size_t total_young_regions() { + return _total_young_regions; + } + + size_t total_old_regions() { + return _partitions.get_total_region_counts(ShenandoahFreeSetPartitionId::OldCollector); + } + + size_t total_global_regions() { + return _total_global_regions; + } + void clear(); // Examine the existing free set representation, capturing the current state into var arguments: @@ -464,6 +734,8 @@ class ShenandoahFreeSet : public CHeapObj { // for evacuation, invoke this to make regions available for mutator allocations. void move_regions_from_collector_to_mutator(size_t cset_regions); + void transfer_humongous_regions_from_mutator_to_old_collector(size_t xfer_regions, size_t humongous_waste_words); + void recycle_trash(); // Acquire heap lock and log status, assuming heap lock is not acquired by the caller. @@ -482,6 +754,12 @@ class ShenandoahFreeSet : public CHeapObj { inline size_t used() const { return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator); } inline size_t available() const { return _partitions.available_in_not_locked(ShenandoahFreeSetPartitionId::Mutator); } + inline size_t total_humongous_waste() const { return _total_humongous_waste; } + inline size_t humongous_waste_in_mutator() const { return _partitions.humongous_waste(ShenandoahFreeSetPartitionId::Mutator); } + inline size_t humongous_waste_in_old() const { return _partitions.humongous_waste(ShenandoahFreeSetPartitionId::OldCollector); } + + void decrease_humongous_waste_for_regular_bypass(ShenandoahHeapRegion* r, size_t waste); + HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region); /* @@ -539,7 +817,8 @@ class ShenandoahFreeSet : public CHeapObj { // Ensure that Collector has at least to_reserve bytes of available memory, and OldCollector has at least old_reserve // bytes of available memory. On input, old_region_count holds the number of regions already present in the // OldCollector partition. Upon return, old_region_count holds the updated number of regions in the OldCollector partition. - void reserve_regions(size_t to_reserve, size_t old_reserve, size_t &old_region_count); + void reserve_regions(size_t to_reserve, size_t old_reserve, size_t &old_region_count, + size_t &young_used_regions, size_t &old_used_regions, size_t &young_used_bytes, size_t &old_used_bytes); // Reserve space for evacuations, with regions reserved for old evacuations placed to the right // of regions reserved of young evacuations. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 27ff45e67de19..dfe3022af49ec 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -239,7 +239,6 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { worker_slices[i] = new ShenandoahHeapRegionSet(); } - ShenandoahGenerationalHeap::TransferResult result; { // The rest of code performs region moves, where region status is undefined // until all phases run together. @@ -253,14 +252,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { phase4_compact_objects(worker_slices); - result = phase5_epilog(); - } - if (heap->mode()->is_generational()) { - LogTarget(Info, gc, ergo) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - result.print_on("Full GC", &ls); - } + phase5_epilog(); } // Resize metaspace @@ -993,23 +985,6 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { r->set_live_data(live); r->reset_alloc_metadata(); } - - void update_generation_usage() { - if (_is_generational) { - _heap->old_generation()->establish_usage(_old_regions, _old_usage, _old_humongous_waste); - _heap->young_generation()->establish_usage(_young_regions, _young_usage, _young_humongous_waste); - } else { - assert(_old_regions == 0, "Old regions only expected in generational mode"); - assert(_old_usage == 0, "Old usage only expected in generational mode"); - assert(_old_humongous_waste == 0, "Old humongous waste only expected in generational mode"); - } - - // In generational mode, global usage should be the sum of young and old. This is also true - // for non-generational modes except that there are no old regions. - _heap->global_generation()->establish_usage(_old_regions + _young_regions, - _old_usage + _young_usage, - _old_humongous_waste + _young_humongous_waste); - } }; void ShenandoahFullGC::compact_humongous_objects() { @@ -1128,10 +1103,9 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s } } -ShenandoahGenerationalHeap::TransferResult ShenandoahFullGC::phase5_epilog() { +void ShenandoahFullGC::phase5_epilog() { GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer); ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahGenerationalHeap::TransferResult result; // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer // and must ensure the bitmap is in sync. @@ -1146,12 +1120,6 @@ ShenandoahGenerationalHeap::TransferResult ShenandoahFullGC::phase5_epilog() { ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild); ShenandoahPostCompactClosure post_compact; heap->heap_region_iterate(&post_compact); - post_compact.update_generation_usage(); - - if (heap->mode()->is_generational()) { - ShenandoahGenerationalFullGC::balance_generations_after_gc(heap); - } - heap->collection_set()->clear(); size_t young_cset_regions, old_cset_regions; size_t first_old, last_old, num_old; @@ -1177,8 +1145,6 @@ ShenandoahGenerationalHeap::TransferResult ShenandoahFullGC::phase5_epilog() { // We defer generation resizing actions until after cset regions have been recycled. We do this even following an // abbreviated cycle. if (heap->mode()->is_generational()) { - result = ShenandoahGenerationalFullGC::balance_generations_after_rebuilding_free_set(); ShenandoahGenerationalFullGC::rebuild_remembered_set(heap); } - return result; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp index b0b8c7bf0c599..45ed341de1799 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp @@ -82,8 +82,7 @@ class ShenandoahFullGC : public ShenandoahGC { void phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices); void phase3_update_references(); void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices); - ShenandoahGenerationalHeap::TransferResult phase5_epilog(); - + void phase5_epilog(); void distribute_slices(ShenandoahHeapRegionSet** worker_slices); void calculate_target_humongous_objects(); void compact_humongous_objects(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index f686334d3d515..4325aba8eadcf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -148,19 +148,8 @@ ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode return _heuristics; } -size_t ShenandoahGeneration::bytes_allocated_since_gc_start() const { - return Atomic::load(&_bytes_allocated_since_gc_start); -} - -void ShenandoahGeneration::reset_bytes_allocated_since_gc_start() { - Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0); -} - -void ShenandoahGeneration::increase_allocated(size_t bytes) { - Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed); -} - void ShenandoahGeneration::set_evacuation_reserve(size_t new_val) { + shenandoah_assert_heaplocked(); _evacuation_reserve = new_val; } @@ -272,7 +261,7 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* const heap // maximum_young_evacuation_reserve is upper bound on memory to be evacuated out of young const size_t maximum_young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; - const size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_generation->available_with_reserve()); + size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_generation->available_with_reserve()); // maximum_old_evacuation_reserve is an upper bound on memory evacuated from old and evacuated to old (promoted), // clamped by the old generation space available. @@ -352,6 +341,11 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* const heap const size_t consumed_by_advance_promotion = select_aged_regions(old_promo_reserve); assert(consumed_by_advance_promotion <= maximum_old_evacuation_reserve, "Cannot promote more than available old-gen memory"); + // If any regions have been selected for promotion in place, this has the effect of decreasing available within mutator + // and collector partitions, due to padding of remnant memory within each promoted in place region. This will affect + // young_evacuation_reserve but not old_evacuation_reserve or consumed_by_advance_promotion. So recompute. + young_evacuation_reserve = MIN2(young_evacuation_reserve, young_generation->available_with_reserve()); + // Note that unused old_promo_reserve might not be entirely consumed_by_advance_promotion. Do not transfer this // to old_evacuation_reserve because this memory is likely very fragmented, and we do not want to increase the likelihood // of old evacuation failure. @@ -435,7 +429,9 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* const heap, size_t excess_old = old_available - old_consumed; size_t unaffiliated_old_regions = old_generation->free_unaffiliated_regions(); size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes; - assert(old_available >= unaffiliated_old, "Unaffiliated old is a subset of old available"); + assert(old_available >= unaffiliated_old, + "Unaffiliated old (%zu is %zu * %zu) is a subset of old available (%zu)", + unaffiliated_old, unaffiliated_old_regions, region_size_bytes, old_available); // Make sure old_evac_committed is unaffiliated if (old_evacuated_committed > 0) { @@ -465,15 +461,10 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* const heap, size_t excess_regions = excess_old / region_size_bytes; regions_to_xfer = MIN2(excess_regions, unaffiliated_old_regions); } - if (regions_to_xfer > 0) { - bool result = ShenandoahGenerationalHeap::cast(heap)->generation_sizer()->transfer_to_young(regions_to_xfer); - assert(excess_old >= regions_to_xfer * region_size_bytes, - "Cannot transfer (%zu, %zu) more than excess old (%zu)", - regions_to_xfer, region_size_bytes, excess_old); excess_old -= regions_to_xfer * region_size_bytes; - log_debug(gc, ergo)("%s transferred %zu excess regions to young before start of evacuation", - result? "Successfully": "Unsuccessfully", regions_to_xfer); + log_debug(gc, ergo)("Before start of evacuation, total_promotion reserve is young_advance_promoted_reserve: %zu " + "plus excess: old: %zu", young_advance_promoted_reserve_used, excess_old); } // Add in the excess_old memory to hold unanticipated promotions, if any. If there are more unanticipated @@ -531,6 +522,8 @@ size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { assert_no_in_place_promotions(); auto const heap = ShenandoahGenerationalHeap::heap(); + ShenandoahYoungGeneration* young_gen = heap->young_generation(); + ShenandoahFreeSet* free_set = heap->free_set(); bool* const candidate_regions_for_promotion_by_copy = heap->collection_set()->preselected_regions(); ShenandoahMarkingContext* const ctx = heap->marking_context(); @@ -547,12 +540,22 @@ size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { // Sort the promotion-eligible regions in order of increasing live-data-bytes so that we can first reclaim regions that require // less evacuation effort. This prioritizes garbage first, expanding the allocation pool early before we reclaim regions that // have more live data. - const size_t num_regions = heap->num_regions(); + const index_type num_regions = heap->num_regions(); ResourceMark rm; AgedRegionData* sorted_regions = NEW_RESOURCE_ARRAY(AgedRegionData, num_regions); - for (size_t i = 0; i < num_regions; i++) { + ShenandoahFreeSet* freeset = heap->free_set(); + + // Any region that is to be promoted in place needs to be retired from its Collector or Mutator partition. + index_type pip_low_collector_idx = freeset->max_regions(); + index_type pip_high_collector_idx = -1; + index_type pip_low_mutator_idx = freeset->max_regions(); + index_type pip_high_mutator_idx = -1; + size_t collector_regions_to_pip = 0; + size_t mutator_regions_to_pip = 0; + + for (index_type i = 0; i < num_regions; i++) { ShenandoahHeapRegion* const r = heap->get_region(i); if (r->is_empty() || !r->has_live() || !r->is_young() || !r->is_regular()) { // skip over regions that aren't regular young with some live data @@ -561,8 +564,7 @@ size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { if (r->age() >= tenuring_threshold) { if ((r->garbage() < old_garbage_threshold)) { // This tenure-worthy region has too little garbage, so we do not want to expend the copying effort to - // reclaim the garbage; instead this region may be eligible for promotion-in-place to the - // old generation. + // reclaim the garbage; instead this region may be eligible for promotion-in-place to old generation. HeapWord* tams = ctx->top_at_mark_start(r); HeapWord* original_top = r->top(); if (!heap->is_concurrent_old_mark_in_progress() && tams == original_top) { @@ -571,18 +573,46 @@ size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { // we use this field to indicate that this region should be promoted in place during the evacuation // phase. r->save_top_before_promote(); - - size_t remnant_size = r->free() / HeapWordSize; - if (remnant_size > ShenandoahHeap::min_fill_size()) { - ShenandoahHeap::fill_with_object(original_top, remnant_size); + size_t remnant_bytes = r->free(); + size_t remnant_words = remnant_bytes / HeapWordSize; + if (remnant_words >= ShenandoahHeap::min_fill_size()) { + ShenandoahHeap::fill_with_object(original_top, remnant_words); // Fill the remnant memory within this region to assure no allocations prior to promote in place. Otherwise, // newly allocated objects will not be parsable when promote in place tries to register them. Furthermore, any // new allocations would not necessarily be eligible for promotion. This addresses both issues. r->set_top(r->end()); - promote_in_place_pad += remnant_size * HeapWordSize; + // The region r is either in the Mutator or Collector partition if remnant_words > heap()->plab_min_size. + // Otherwise, the region is in the NotFree partition. + ShenandoahFreeSetPartitionId p = free_set->membership(i); + if (p == ShenandoahFreeSetPartitionId::Mutator) { + mutator_regions_to_pip++; + if (i < pip_low_mutator_idx) { + pip_low_mutator_idx = i; + } + if (i > pip_high_mutator_idx) { + pip_high_mutator_idx = i; + } + } else if (p == ShenandoahFreeSetPartitionId::Collector) { + collector_regions_to_pip++; + if (i < pip_low_collector_idx) { + pip_low_collector_idx = i; + } + if (i > pip_high_collector_idx) { + pip_high_collector_idx = i; + } + } else { + assert((p == ShenandoahFreeSetPartitionId::NotFree) && (remnant_words < heap->plab_min_size()), + "Should be NotFree if not in Collector or Mutator partitions"); + // In this case, we'll count the remnant_bytes as used even though we will not create a fill object. + } + promote_in_place_pad += remnant_bytes; + free_set->prepare_to_promote_in_place(i, remnant_bytes); } else { // Since the remnant is so small that it cannot be filled, we don't have to worry about any accidental // allocations occurring within this region before the region is promoted in place. + + // This region was already not in the Collector or Mutator set, so no need to remove it. + assert(free_set->membership(i) == ShenandoahFreeSetPartitionId::NotFree, "sanity"); } } // Else, we do not promote this region (either in place or by copy) because it has received new allocations. @@ -622,6 +652,19 @@ size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { // Note that we keep going even if one region is excluded from selection. // Subsequent regions may be selected if they have smaller live data. } + + // Retire any regions that have been selected for promote in place + if (collector_regions_to_pip > 0) { + freeset->shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Collector, + pip_low_collector_idx, pip_high_collector_idx, + collector_regions_to_pip); + } + if (mutator_regions_to_pip > 0) { + freeset->shrink_interval_if_range_modifies_either_boundary(ShenandoahFreeSetPartitionId::Mutator, + pip_low_mutator_idx, pip_high_mutator_idx, + mutator_regions_to_pip); + } + // Sort in increasing order according to live data bytes. Note that candidates represents the number of regions // that qualify to be promoted by evacuation. if (candidates > 0) { @@ -752,9 +795,15 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { // We are preparing for evacuation. At this time, we ignore cset region tallies. size_t first_old, last_old, num_old; - heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); + _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); + + if (heap->mode()->is_generational()) { + ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); + gen_heap->compute_old_generation_balance(young_cset_regions, old_cset_regions); + } + // Free set construction uses reserve quantities, because they are known to be valid here - heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true); + _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true); } } @@ -804,8 +853,9 @@ ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type, _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), _affiliated_region_count(0), _humongous_waste(0), _evacuation_reserve(0), - _used(0), _bytes_allocated_since_gc_start(0), + _used(0), _max_capacity(max_capacity), + _free_set(nullptr), _heuristics(nullptr) { _is_marking_complete.set(); @@ -824,6 +874,11 @@ ShenandoahGeneration::~ShenandoahGeneration() { delete _task_queues; } +void ShenandoahGeneration::post_initialize(ShenandoahHeap* heap) { + _free_set = heap->free_set(); + assert(_free_set != nullptr, "bad initialization order"); +} + void ShenandoahGeneration::reserve_task_queues(uint workers) { _task_queues->reserve(workers); } @@ -851,159 +906,119 @@ void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { } } -size_t ShenandoahGeneration::increment_affiliated_region_count() { - shenandoah_assert_heaplocked_or_safepoint(); - // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced - // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with - // a coherent value. - return Atomic::add(&_affiliated_region_count, (size_t) 1); -} - -size_t ShenandoahGeneration::decrement_affiliated_region_count() { - shenandoah_assert_heaplocked_or_safepoint(); - // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced - // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with - // a coherent value. - auto affiliated_region_count = Atomic::sub(&_affiliated_region_count, (size_t) 1); - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (used() + _humongous_waste <= affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), - "used + humongous cannot exceed regions"); - return affiliated_region_count; -} - -size_t ShenandoahGeneration::decrement_affiliated_region_count_without_lock() { - return Atomic::sub(&_affiliated_region_count, (size_t) 1); -} - -size_t ShenandoahGeneration::increase_affiliated_region_count(size_t delta) { - shenandoah_assert_heaplocked_or_safepoint(); - return Atomic::add(&_affiliated_region_count, delta); -} - -size_t ShenandoahGeneration::decrease_affiliated_region_count(size_t delta) { - shenandoah_assert_heaplocked_or_safepoint(); - assert(Atomic::load(&_affiliated_region_count) >= delta, "Affiliated region count cannot be negative"); - - auto const affiliated_region_count = Atomic::sub(&_affiliated_region_count, delta); - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_used + _humongous_waste <= affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), - "used + humongous cannot exceed regions"); - return affiliated_region_count; -} - -void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) { - assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); - Atomic::store(&_affiliated_region_count, num_regions); - Atomic::store(&_used, num_bytes); - _humongous_waste = humongous_waste; -} - -void ShenandoahGeneration::increase_used(size_t bytes) { - Atomic::add(&_used, bytes); -} - -void ShenandoahGeneration::increase_humongous_waste(size_t bytes) { - if (bytes > 0) { - Atomic::add(&_humongous_waste, bytes); +size_t ShenandoahGeneration::used_regions() const { + size_t result; + switch (_type) { + case ShenandoahGenerationType::OLD: + result = _free_set->old_affiliated_regions(); + break; + case ShenandoahGenerationType::YOUNG: + result = _free_set->young_affiliated_regions(); + break; + case ShenandoahGenerationType::GLOBAL: + case ShenandoahGenerationType::NON_GEN: + default: + result = _free_set->global_affiliated_regions(); + break; } +#ifdef KELVIN_DEBUG + log_info(gc)("used_regions(_type: %d) returning %zu", _type, result); +#endif + return result; } -void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) { - if (bytes > 0) { - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes), - "Waste (%zu) cannot be negative (after subtracting %zu)", _humongous_waste, bytes); - Atomic::sub(&_humongous_waste, bytes); +size_t ShenandoahGeneration::max_capacity() const { + size_t total_regions; + switch (_type) { + case ShenandoahGenerationType::OLD: + total_regions = _free_set->total_old_regions(); + break; + case ShenandoahGenerationType::YOUNG: + total_regions = _free_set->total_young_regions(); + break; + case ShenandoahGenerationType::GLOBAL: + case ShenandoahGenerationType::NON_GEN: + default: + total_regions = _free_set->total_global_regions(); + break; } -} - -void ShenandoahGeneration::decrease_used(size_t bytes) { - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_used >= bytes), "cannot reduce bytes used by generation below zero"); - Atomic::sub(&_used, bytes); -} - -size_t ShenandoahGeneration::used_regions() const { - return Atomic::load(&_affiliated_region_count); +#ifdef KELVIN_DEBUG + log_info(gc)("max_capacity(_type: %d) returning %zu", _type, total_regions * ShenandoahHeapRegion::region_size_bytes()); +#endif + return total_regions * ShenandoahHeapRegion::region_size_bytes(); } size_t ShenandoahGeneration::free_unaffiliated_regions() const { - size_t result = max_capacity() / ShenandoahHeapRegion::region_size_bytes(); - auto const used_regions = this->used_regions(); - if (used_regions > result) { - result = 0; - } else { - result -= used_regions; + size_t free_regions; + switch (_type) { + case ShenandoahGenerationType::OLD: + free_regions = _free_set->old_unaffiliated_regions(); + break; + case ShenandoahGenerationType::YOUNG: + free_regions = _free_set->young_unaffiliated_regions(); + break; + case ShenandoahGenerationType::GLOBAL: + case ShenandoahGenerationType::NON_GEN: + default: + free_regions = _free_set->global_unaffiliated_regions(); + break; } - return result; +#ifdef KELVIN_DEBUG + log_info(gc)("free_unaffiliated_regions(_type: %d) returning %zu", _type, free_regions); +#endif + return free_regions; } size_t ShenandoahGeneration::used_regions_size() const { - return used_regions() * ShenandoahHeapRegion::region_size_bytes(); + size_t used_regions; + switch (_type) { + case ShenandoahGenerationType::OLD: + used_regions = _free_set->old_affiliated_regions(); + break; + case ShenandoahGenerationType::YOUNG: + used_regions = _free_set->young_affiliated_regions(); + break; + case ShenandoahGenerationType::GLOBAL: + case ShenandoahGenerationType::NON_GEN: + default: + used_regions = _free_set->global_affiliated_regions(); + break; + } +#ifdef KELVIN_DEBUG + log_info(gc)("used_regions_size(_type: %d) returning %zu", _type, used_regions * ShenandoahHeapRegion::region_size_bytes()); +#endif + return used_regions * ShenandoahHeapRegion::region_size_bytes(); } size_t ShenandoahGeneration::available() const { +#ifdef KELVIN_DEBUG + log_info(gc)("available(_type: %d) returning %zu", _type, available(max_capacity())); +#endif return available(max_capacity()); } // For ShenandoahYoungGeneration, Include the young available that may have been reserved for the Collector. size_t ShenandoahGeneration::available_with_reserve() const { +#ifdef KELVIN_DEBUG + log_info(gc)("available_with_reserve(_type: %d) returning %zu", _type, available(max_capacity())); +#endif return available(max_capacity()); } size_t ShenandoahGeneration::soft_available() const { +#ifdef KELVIN_DEBUG + log_info(gc)("soft_available(_type: %d) returning %zu", _type, available(ShenandoahHeap::heap()->soft_max_capacity())); +#endif return available(ShenandoahHeap::heap()->soft_max_capacity()); } size_t ShenandoahGeneration::available(size_t capacity) const { - size_t in_use = used() + get_humongous_waste(); - return in_use > capacity ? 0 : capacity - in_use; -} - -size_t ShenandoahGeneration::increase_capacity(size_t increment) { - shenandoah_assert_heaplocked_or_safepoint(); - - // We do not enforce that new capacity >= heap->max_size_for(this). The maximum generation size is treated as a rule of thumb - // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions - // in place. - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_max_capacity + increment <= ShenandoahHeap::heap()->max_capacity()), "Generation cannot be larger than heap size"); - assert(increment % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size"); - _max_capacity += increment; - - // This detects arithmetic wraparound on _used - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (used_regions_size() >= used()), - "Affiliated regions must hold more than what is currently used"); - return _max_capacity; -} - -size_t ShenandoahGeneration::set_capacity(size_t byte_size) { - shenandoah_assert_heaplocked_or_safepoint(); - _max_capacity = byte_size; - return _max_capacity; -} - -size_t ShenandoahGeneration::decrease_capacity(size_t decrement) { - shenandoah_assert_heaplocked_or_safepoint(); - - // We do not enforce that new capacity >= heap->min_size_for(this). The minimum generation size is treated as a rule of thumb - // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions - // in place. - assert(decrement % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size"); - assert(_max_capacity >= decrement, "Generation capacity cannot be negative"); - - _max_capacity -= decrement; - - // This detects arithmetic wraparound on _used - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (used_regions_size() >= used()), - "Affiliated regions must hold more than what is currently used"); - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_used <= _max_capacity), "Cannot use more than capacity"); - assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || - (used_regions_size() <= _max_capacity), - "Cannot use more than capacity"); - return _max_capacity; + size_t in_use = used(); + size_t result = in_use > capacity ? 0 : capacity - in_use; +#ifdef KELVIN_DEBUG + log_info(gc)("available(_type: %d, capacity: %zu) returning %zu", _type, capacity, result); +#endif + return result; } void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 2b7aca342dad1..5bdb3c1d66aef 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/heuristics/shenandoahSpaceInfo.hpp" #include "gc/shenandoah/shenandoahAffiliation.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGenerationType.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahMarkingContext.hpp" @@ -69,9 +70,8 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { // Usage volatile size_t _used; - volatile size_t _bytes_allocated_since_gc_start; size_t _max_capacity; - + ShenandoahFreeSet* _free_set; ShenandoahHeuristics* _heuristics; private: @@ -99,6 +99,7 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { // to false. size_t select_aged_regions(size_t old_available); + // Return available assuming that we can allocate no more than capacity bytes within this generation. size_t available(size_t capacity) const; public: @@ -124,15 +125,43 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode); - size_t max_capacity() const override { return _max_capacity; } + virtual void post_initialize(ShenandoahHeap* heap); + + size_t max_capacity() const override; + virtual size_t used_regions() const; virtual size_t used_regions_size() const; virtual size_t free_unaffiliated_regions() const; - size_t used() const override { return Atomic::load(&_used); } + size_t used() const override { + size_t result; + switch (_type) { + case ShenandoahGenerationType::OLD: + result = _free_set->old_used(); + break; + case ShenandoahGenerationType::YOUNG: + result = _free_set->young_used(); + break; + case ShenandoahGenerationType::GLOBAL: + case ShenandoahGenerationType::NON_GEN: + default: + result = _free_set->global_used(); + break; + } +#undef KELVIN_DEBUG +#ifdef KELVIN_DEBUG + log_info(gc)("used(_type: %d) returning %zu", _type, result); +#endif + return result; + } + size_t available() const override; size_t available_with_reserve() const; size_t used_including_humongous_waste() const { - return used() + get_humongous_waste(); + // In the current implementation, used() includes humongous waste +#ifdef KELVIN_DEBUG + log_info(gc)("used_including_humongous_waste(_type: %d) returning %zu", _type, used()); +#endif + return used(); } // Returns the memory available based on the _soft_ max heap capacity (soft_max_heap - used). @@ -141,17 +170,28 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { // max heap size will cause the adaptive heuristic to run more frequent cycles. size_t soft_available() const override; - size_t bytes_allocated_since_gc_start() const override; - void reset_bytes_allocated_since_gc_start(); - void increase_allocated(size_t bytes); - - // These methods change the capacity of the generation by adding or subtracting the given number of bytes from the current - // capacity, returning the capacity of the generation following the change. - size_t increase_capacity(size_t increment); - size_t decrease_capacity(size_t decrement); - - // Set the capacity of the generation, returning the value set - size_t set_capacity(size_t byte_size); + size_t bytes_allocated_since_gc_start() const override { + if (_type == ShenandoahGenerationType::YOUNG) { + size_t result = _free_set->get_bytes_allocated_since_gc_start(); +#ifdef KELVIN_DEBUG + log_info(gc)("bytes_allocated_since_gc_start(_type: %d) returning %zu", _type, result); +#endif + return result; + } else if (_type == ShenandoahGenerationType::NON_GEN) { + assert(!ShenandoahHeap::heap()->mode()->is_generational(), "NON_GEN implies not generational"); + size_t result = _free_set->get_bytes_allocated_since_gc_start(); +#ifdef KELVIN_DEBUG + log_info(gc)("bytes_allocated_since_gc_start(_type: %d) returning %zu", _type, result); +#endif + return result; + } else { + size_t result = 0; +#ifdef KELVIN_DEBUG + log_info(gc)("bytes_allocated_since_gc_start(_type: %d) returning %zu", _type, result); +#endif + return result; + } + } void log_status(const char* msg) const; @@ -212,28 +252,68 @@ class ShenandoahGeneration : public CHeapObj, public ShenandoahSpaceInfo { // Scan remembered set at start of concurrent young-gen marking. void scan_remembered_set(bool is_concurrent); - // Return the updated value of affiliated_region_count - size_t increment_affiliated_region_count(); - - // Return the updated value of affiliated_region_count - size_t decrement_affiliated_region_count(); - // Same as decrement_affiliated_region_count, but w/o the need to hold heap lock before being called. - size_t decrement_affiliated_region_count_without_lock(); - - // Return the updated value of affiliated_region_count - size_t increase_affiliated_region_count(size_t delta); - - // Return the updated value of affiliated_region_count - size_t decrease_affiliated_region_count(size_t delta); - - void establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste); + size_t get_affiliated_region_count() const { + size_t result; + switch (_type) { + case ShenandoahGenerationType::OLD: + result = _free_set->old_affiliated_regions(); + break; + case ShenandoahGenerationType::YOUNG: + result = _free_set->young_affiliated_regions(); + break; + case ShenandoahGenerationType::GLOBAL: + case ShenandoahGenerationType::NON_GEN: + default: + result = _free_set->global_affiliated_regions(); + break; + } +#ifdef KELVIN_DEBUG + log_info(gc)("get_affiliated_region_count(_type: %d) returning %zu", _type, result); +#endif + return result; + } - void increase_used(size_t bytes); - void decrease_used(size_t bytes); + size_t get_total_region_count() const { + size_t result; + switch (_type) { + case ShenandoahGenerationType::OLD: + result = _free_set->total_old_regions(); + break; + case ShenandoahGenerationType::YOUNG: + result = _free_set->total_young_regions(); + break; + case ShenandoahGenerationType::GLOBAL: + case ShenandoahGenerationType::NON_GEN: + default: + result = _free_set->total_global_regions(); + break; + } +#ifdef KELVIN_DEBUG + log_info(gc)("get_total_region_count(_type: %d) returning %zu", _type, result); +#endif + return result; + } - void increase_humongous_waste(size_t bytes); - void decrease_humongous_waste(size_t bytes); - size_t get_humongous_waste() const { return _humongous_waste; } + size_t get_humongous_waste() const { + size_t result; + switch (_type) { + case ShenandoahGenerationType::OLD: + result = _free_set->humongous_waste_in_old(); + break; + case ShenandoahGenerationType::YOUNG: + result = _free_set->humongous_waste_in_mutator(); + break; + case ShenandoahGenerationType::GLOBAL: + case ShenandoahGenerationType::NON_GEN: + default: + result = _free_set->total_humongous_waste(); + break; + } +#ifdef KELVIN_DEBUG + log_info(gc)("get_humongous_waste()(_type: %d) returning %zu", _type, result); +#endif + return result; + } virtual bool is_concurrent_mark_in_progress() = 0; void confirm_heuristics_mode(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp index 17f3d2f199f1a..640cf75ae41fe 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.cpp @@ -115,34 +115,6 @@ void ShenandoahGenerationSizer::heap_size_changed(size_t heap_size) { recalculate_min_max_young_length(heap_size / ShenandoahHeapRegion::region_size_bytes()); } -bool ShenandoahGenerationSizer::transfer_regions(ShenandoahGeneration* src, ShenandoahGeneration* dst, size_t regions) const { - const size_t bytes_to_transfer = regions * ShenandoahHeapRegion::region_size_bytes(); - - if (src->free_unaffiliated_regions() < regions) { - // Source does not have enough free regions for this transfer. The caller should have - // already capped the transfer based on available unaffiliated regions. - return false; - } - - if (dst->max_capacity() + bytes_to_transfer > max_size_for(dst)) { - // This transfer would cause the destination generation to grow above its configured maximum size. - return false; - } - - if (src->max_capacity() - bytes_to_transfer < min_size_for(src)) { - // This transfer would cause the source generation to shrink below its configured minimum size. - return false; - } - - src->decrease_capacity(bytes_to_transfer); - dst->increase_capacity(bytes_to_transfer); - const size_t new_size = dst->max_capacity(); - log_info(gc, ergo)("Transfer %zu region(s) from %s to %s, yielding increased size: " PROPERFMT, - regions, src->name(), dst->name(), PROPERFMTARGS(new_size)); - return true; -} - - size_t ShenandoahGenerationSizer::max_size_for(ShenandoahGeneration* generation) const { switch (generation->type()) { case YOUNG: @@ -171,34 +143,6 @@ size_t ShenandoahGenerationSizer::min_size_for(ShenandoahGeneration* generation) } } - -// Returns true iff transfer is successful -bool ShenandoahGenerationSizer::transfer_to_old(size_t regions) const { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - return transfer_regions(heap->young_generation(), heap->old_generation(), regions); -} - -// This is used when promoting humongous or highly utilized regular regions in place. It is not required in this situation -// that the transferred regions be unaffiliated. -void ShenandoahGenerationSizer::force_transfer_to_old(size_t regions) const { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - ShenandoahGeneration* old_gen = heap->old_generation(); - ShenandoahGeneration* young_gen = heap->young_generation(); - const size_t bytes_to_transfer = regions * ShenandoahHeapRegion::region_size_bytes(); - - young_gen->decrease_capacity(bytes_to_transfer); - old_gen->increase_capacity(bytes_to_transfer); - const size_t new_size = old_gen->max_capacity(); - log_info(gc, ergo)("Forcing transfer of %zu region(s) from %s to %s, yielding increased size: " PROPERFMT, - regions, young_gen->name(), old_gen->name(), PROPERFMTARGS(new_size)); -} - - -bool ShenandoahGenerationSizer::transfer_to_young(size_t regions) const { - ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); - return transfer_regions(heap->old_generation(), heap->young_generation(), regions); -} - size_t ShenandoahGenerationSizer::min_young_size() const { return min_young_regions() * ShenandoahHeapRegion::region_size_bytes(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp index 5752422bb7717..d1a4dff81f8c4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationSizer.hpp @@ -51,11 +51,8 @@ class ShenandoahGenerationSizer { // given the number of heap regions depending on the kind of sizing algorithm. void recalculate_min_max_young_length(size_t heap_region_count); - // This will attempt to transfer regions from the `src` generation to `dst` generation. - // If the transfer would violate the configured minimum size for the source or the configured - // maximum size of the destination, it will not perform the transfer and will return false. - // Returns true if the transfer is performed. - bool transfer_regions(ShenandoahGeneration* src, ShenandoahGeneration* dst, size_t regions) const; +public: + ShenandoahGenerationSizer(); // Return the configured maximum size in bytes for the given generation. size_t max_size_for(ShenandoahGeneration* generation) const; @@ -63,9 +60,6 @@ class ShenandoahGenerationSizer { // Return the configured minimum size in bytes for the given generation. size_t min_size_for(ShenandoahGeneration* generation) const; -public: - ShenandoahGenerationSizer(); - // Calculate the maximum length of the young gen given the number of regions // depending on the sizing algorithm. void heap_size_changed(size_t heap_size); @@ -81,13 +75,6 @@ class ShenandoahGenerationSizer { size_t max_young_regions() const { return _max_desired_young_regions; } - - // True if transfer succeeds, else false. See transfer_regions. - bool transfer_to_young(size_t regions) const; - bool transfer_to_old(size_t regions) const; - - // force transfer is used when we promote humongous objects. May violate min/max limits on generation sizes - void force_transfer_to_old(size_t regions) const; }; #endif //SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONSIZER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp index 1cf8b78ef1aae..6c1b498919017 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalControlThread.cpp @@ -269,11 +269,11 @@ void ShenandoahGenerationalControlThread::run_gc_cycle(const ShenandoahGCRequest if (!_heap->cancelled_gc()) { notify_gc_waiters(); notify_alloc_failure_waiters(); + // Report current free set state at the end of cycle if normal completion. + // Do not report if cancelled, since we may not have rebuilt free set and content is unreliable. + _heap->free_set()->log_status_under_lock(); } - // Report current free set state at the end of cycle, whether - // it is a normal completion, or the abort. - _heap->free_set()->log_status_under_lock(); // Notify Universe about new heap usage. This has implications for // global soft refs policy, and we better report it every time heap diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp index 29fd3258b6ca7..0456bc552e731 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp @@ -168,9 +168,10 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion assert(!_heap->gc_generation()->is_old(), "Sanity check"); ShenandoahMarkingContext* const marking_context = _heap->young_generation()->complete_marking_context(); HeapWord* const tams = marking_context->top_at_mark_start(region); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); { - const size_t old_garbage_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold) / 100; + const size_t old_garbage_threshold = (region_size_bytes * ShenandoahOldGarbageThreshold) / 100; shenandoah_assert_generations_reconciled(); assert(!_heap->is_concurrent_old_mark_in_progress(), "Cannot promote in place during old marking"); assert(region->garbage_before_padded_for_promote() < old_garbage_threshold, "Region %zu has too much garbage for promotion", region->index()); @@ -220,12 +221,19 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion ShenandoahHeapLocker locker(_heap->lock()); HeapWord* update_watermark = region->get_update_watermark(); + // pip_unpadded is memory too small to be filled above original top + size_t pip_unpadded = (region->end() - region->top()) * HeapWordSize; + assert((region->top() == region->end()) + || (pip_unpadded == (size_t) ((region->end() - region->top()) * HeapWordSize)), "Invariant"); + assert(pip_unpadded < ShenandoahHeap::min_fill_size() * HeapWordSize, "Sanity"); + size_t pip_pad_bytes = (region->top() - region->get_top_before_promote()) * HeapWordSize; + assert((pip_unpadded == 0) || (pip_pad_bytes == 0), "Only one of pip_unpadded and pip_pad_bytes is non-zero"); // Now that this region is affiliated with old, we can allow it to receive allocations, though it may not be in the // is_collector_free range. region->restore_top_before_promote(); - - size_t region_used = region->used(); + size_t region_to_be_used_in_old = region->used(); + assert(region_to_be_used_in_old + pip_pad_bytes + pip_unpadded == region_size_bytes, "invariant"); // The update_watermark was likely established while we had the artificially high value of top. Make it sane now. assert(update_watermark >= region->top(), "original top cannot exceed preserved update_watermark"); @@ -237,18 +245,17 @@ void ShenandoahGenerationalEvacuationTask::promote_in_place(ShenandoahHeapRegion // However, if we do not transfer the capacities, we end up reducing the amount of memory that would have // otherwise been available to hold old evacuations, because old available is max_capacity - used and now // we would be trading a fully empty region for a partially used region. - young_gen->decrease_used(region_used); - young_gen->decrement_affiliated_region_count(); - - // transfer_to_old() increases capacity of old and decreases capacity of young - _heap->generation_sizer()->force_transfer_to_old(1); - region->set_affiliation(OLD_GENERATION); - old_gen->increment_affiliated_region_count(); - old_gen->increase_used(region_used); + size_t available_in_region = region->free(); + size_t plab_min_size_in_bytes = _heap->plab_min_size() * HeapWordSize; + if (available_in_region < plab_min_size_in_bytes) { + // The available memory in young had been retired. Retire it in old also. + region_to_be_used_in_old += available_in_region; + } // add_old_collector_free_region() increases promoted_reserve() if available space exceeds plab_min_size() _heap->free_set()->add_promoted_in_place_region_to_old_collector(region); + region->set_affiliation(OLD_GENERATION); } } @@ -264,7 +271,8 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio const size_t used_bytes = obj->size() * HeapWordSize; const size_t spanned_regions = ShenandoahHeapRegion::required_regions(used_bytes); - const size_t humongous_waste = spanned_regions * ShenandoahHeapRegion::region_size_bytes() - obj->size() * HeapWordSize; + const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + const size_t humongous_waste = spanned_regions * region_size_bytes - obj->size() * HeapWordSize; const size_t index_limit = region->index() + spanned_regions; ShenandoahOldGeneration* const old_gen = _heap->old_generation(); @@ -278,13 +286,6 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio // usage totals, including humongous waste, after evacuation is done. log_debug(gc)("promoting humongous region %zu, spanning %zu", region->index(), spanned_regions); - young_gen->decrease_used(used_bytes); - young_gen->decrease_humongous_waste(humongous_waste); - young_gen->decrease_affiliated_region_count(spanned_regions); - - // transfer_to_old() increases capacity of old and decreases capacity of young - _heap->generation_sizer()->force_transfer_to_old(spanned_regions); - // For this region and each humongous continuation region spanned by this humongous object, change // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory // in the last humongous region that is not spanned by obj is currently not used. @@ -296,9 +297,8 @@ void ShenandoahGenerationalEvacuationTask::promote_humongous(ShenandoahHeapRegio r->set_affiliation(OLD_GENERATION); } - old_gen->increase_affiliated_region_count(spanned_regions); - old_gen->increase_used(used_bytes); - old_gen->increase_humongous_waste(humongous_waste); + ShenandoahFreeSet* freeset = _heap->free_set(); + freeset->transfer_humongous_regions_from_mutator_to_old_collector(spanned_regions, humongous_waste); } // Since this region may have served previously as OLD, it may hold obsolete object range info. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp index e2e3f0a467744..3b43433e22d54 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.cpp @@ -105,33 +105,6 @@ void ShenandoahGenerationalFullGC::rebuild_remembered_set(ShenandoahHeap* heap) heap->old_generation()->set_parsable(true); } -void ShenandoahGenerationalFullGC::balance_generations_after_gc(ShenandoahHeap* heap) { - ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::cast(heap); - ShenandoahOldGeneration* const old_gen = gen_heap->old_generation(); - - size_t old_usage = old_gen->used_regions_size(); - size_t old_capacity = old_gen->max_capacity(); - - assert(old_usage % ShenandoahHeapRegion::region_size_bytes() == 0, "Old usage must align with region size"); - assert(old_capacity % ShenandoahHeapRegion::region_size_bytes() == 0, "Old capacity must align with region size"); - - if (old_capacity > old_usage) { - size_t excess_old_regions = (old_capacity - old_usage) / ShenandoahHeapRegion::region_size_bytes(); - gen_heap->generation_sizer()->transfer_to_young(excess_old_regions); - } else if (old_capacity < old_usage) { - size_t old_regions_deficit = (old_usage - old_capacity) / ShenandoahHeapRegion::region_size_bytes(); - gen_heap->generation_sizer()->force_transfer_to_old(old_regions_deficit); - } - - log_info(gc, ergo)("FullGC done: young usage: " PROPERFMT ", old usage: " PROPERFMT, - PROPERFMTARGS(gen_heap->young_generation()->used()), - PROPERFMTARGS(old_gen->used())); -} - -ShenandoahGenerationalHeap::TransferResult ShenandoahGenerationalFullGC::balance_generations_after_rebuilding_free_set() { - return ShenandoahGenerationalHeap::heap()->balance_generations(); -} - void ShenandoahGenerationalFullGC::log_live_in_old(ShenandoahHeap* heap) { LogTarget(Debug, gc) lt; if (lt.is_enabled()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp index 9240a056105fc..da1f6db01cf06 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalFullGC.hpp @@ -45,25 +45,11 @@ class ShenandoahGenerationalFullGC { // Records end of cycle for young and old and establishes size of live bytes in old static void handle_completion(ShenandoahHeap* heap); - // Full GC may have promoted regions and may have temporarily violated constraints on the usage and - // capacity of the old generation. This method will balance the accounting of regions between the - // young and old generations. This is somewhat vestigial, but the outcome of this method is used - // when rebuilding the free sets. - static void balance_generations_after_gc(ShenandoahHeap* heap); - // This will compute the target size for the old generation. It will be expressed in terms of // a region surplus and deficit, which will be redistributed accordingly after rebuilding the // free set. static void compute_balances(); - // Rebuilding the free set may have resulted in regions being pulled in to the old generation - // evacuation reserve. For this reason, we must update the usage and capacity of the generations - // again. In the distant past, the free set did not know anything about generations, so we had - // a layer built above it to represent how much young/old memory was available. This layer is - // redundant and adds complexity. We would like to one day remove it. Until then, we must keep it - // synchronized with the free set's view of things. - static ShenandoahGenerationalHeap::TransferResult balance_generations_after_rebuilding_free_set(); - // Logs the number of live bytes marked in the old generation. This is _not_ the same // value used as the baseline for the old generation _after_ the full gc is complete. // The value reported in the logs does not include objects and regions that may be diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp index d05ae713645af..6124c8c54ea73 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/shenandoahClosures.inline.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahGenerationalControlThread.hpp" #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp" #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" @@ -119,6 +120,12 @@ void ShenandoahGenerationalHeap::initialize_heuristics() { _old_generation->initialize_heuristics(mode()); } +void ShenandoahGenerationalHeap::post_initialize_heuristics() { + ShenandoahHeap::post_initialize_heuristics(); + _young_generation->post_initialize(this); + _old_generation->post_initialize(this); +} + void ShenandoahGenerationalHeap::initialize_serviceability() { assert(mode()->is_generational(), "Only for the generational mode"); _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this); @@ -576,39 +583,13 @@ void ShenandoahGenerationalHeap::retire_plab(PLAB* plab) { retire_plab(plab, thread); } -ShenandoahGenerationalHeap::TransferResult ShenandoahGenerationalHeap::balance_generations() { - shenandoah_assert_heaplocked_or_safepoint(); - - ShenandoahOldGeneration* old_gen = old_generation(); - const ssize_t old_region_balance = old_gen->get_region_balance(); - old_gen->set_region_balance(0); - - if (old_region_balance > 0) { - const auto old_region_surplus = checked_cast(old_region_balance); - const bool success = generation_sizer()->transfer_to_young(old_region_surplus); - return TransferResult { - success, old_region_surplus, "young" - }; - } - - if (old_region_balance < 0) { - const auto old_region_deficit = checked_cast(-old_region_balance); - const bool success = generation_sizer()->transfer_to_old(old_region_deficit); - if (!success) { - old_gen->handle_failed_transfer(); - } - return TransferResult { - success, old_region_deficit, "old" - }; - } - - return TransferResult {true, 0, "none"}; -} - // Make sure old-generation is large enough, but no larger than is necessary, to hold mixed evacuations // and promotions, if we anticipate either. Any deficit is provided by the young generation, subject to // xfer_limit, and any surplus is transferred to the young generation. -// xfer_limit is the maximum we're able to transfer from young to old. +// +// xfer_limit is the maximum we're able to transfer from young to old based on either: +// 1. an assumption that we will be able to replenish memory "borrowed" from young at the end of collection, or +// 2. there is sufficient excess in the allocation runway during GC idle cycles void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_limit, size_t old_cset_regions) { // We can limit the old reserve to the size of anticipated promotions: @@ -636,9 +617,9 @@ void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_ // In the case that ShenandoahOldEvacRatioPercent equals 100, max_old_reserve is limited only by xfer_limit. const double bound_on_old_reserve = old_available + old_xfer_limit + young_reserve; - const double max_old_reserve = (ShenandoahOldEvacRatioPercent == 100)? - bound_on_old_reserve: MIN2(double(young_reserve * ShenandoahOldEvacRatioPercent) / double(100 - ShenandoahOldEvacRatioPercent), - bound_on_old_reserve); + const double max_old_reserve = ((ShenandoahOldEvacRatioPercent == 100)? bound_on_old_reserve: + MIN2(double(young_reserve * ShenandoahOldEvacRatioPercent) + / double(100 - ShenandoahOldEvacRatioPercent), bound_on_old_reserve)); const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); @@ -647,10 +628,12 @@ void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_ if (old_generation()->has_unprocessed_collection_candidates()) { // We want this much memory to be unfragmented in order to reliably evacuate old. This is conservative because we // may not evacuate the entirety of unprocessed candidates in a single mixed evacuation. - const double max_evac_need = (double(old_generation()->unprocessed_collection_candidates_live_memory()) * ShenandoahOldEvacWaste); + const double max_evac_need = + (double(old_generation()->unprocessed_collection_candidates_live_memory()) * ShenandoahOldEvacWaste); assert(old_available >= old_generation()->free_unaffiliated_regions() * region_size_bytes, "Unaffiliated available must be less than total available"); - const double old_fragmented_available = double(old_available - old_generation()->free_unaffiliated_regions() * region_size_bytes); + const double old_fragmented_available = + double(old_available - old_generation()->free_unaffiliated_regions() * region_size_bytes); reserve_for_mixed = max_evac_need + old_fragmented_available; if (reserve_for_mixed > max_old_reserve) { reserve_for_mixed = max_old_reserve; @@ -697,6 +680,7 @@ void ShenandoahGenerationalHeap::compute_old_generation_balance(size_t old_xfer_ } void ShenandoahGenerationalHeap::reset_generation_reserves() { + ShenandoahHeapLocker locker(lock()); young_generation()->set_evacuation_reserve(0); old_generation()->set_evacuation_reserve(0); old_generation()->set_promoted_reserve(0); @@ -1057,15 +1041,6 @@ void ShenandoahGenerationalHeap::complete_degenerated_cycle() { // a more detailed explanation. old_generation()->transfer_pointers_from_satb(); } - - // We defer generation resizing actions until after cset regions have been recycled. - TransferResult result = balance_generations(); - LogTarget(Info, gc, ergo) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - result.print_on("Degenerated GC", &ls); - } - // In case degeneration interrupted concurrent evacuation or update references, we need to clean up // transient state. Otherwise, these actions have no effect. reset_generation_reserves(); @@ -1087,20 +1062,7 @@ void ShenandoahGenerationalHeap::complete_concurrent_cycle() { // throw off the heuristics. entry_global_coalesce_and_fill(); } - - TransferResult result; - { - ShenandoahHeapLocker locker(lock()); - - result = balance_generations(); - reset_generation_reserves(); - } - - LogTarget(Info, gc, ergo) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - result.print_on("Concurrent GC", &ls); - } + reset_generation_reserves(); } void ShenandoahGenerationalHeap::entry_global_coalesce_and_fill() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp index f23e49735e9f6..a9a8d23d34fdf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.hpp @@ -42,6 +42,7 @@ class ShenandoahGenerationalHeap : public ShenandoahHeap { explicit ShenandoahGenerationalHeap(ShenandoahCollectorPolicy* policy); void post_initialize() override; void initialize_heuristics() override; + void post_initialize_heuristics() override; static ShenandoahGenerationalHeap* heap() { shenandoah_assert_generational(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 2baf7e25cbabb..71f85faed08ba 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -404,7 +404,6 @@ jint ShenandoahHeap::initialize() { { ShenandoahHeapLocker locker(lock()); - _free_set = new ShenandoahFreeSet(this, _num_regions); for (size_t i = 0; i < _num_regions; i++) { HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i; bool is_committed = i < num_committed_regions; @@ -419,12 +418,21 @@ jint ShenandoahHeap::initialize() { _affiliations[i] = ShenandoahAffiliation::FREE; } + _free_set = new ShenandoahFreeSet(this, _num_regions); - size_t young_cset_regions, old_cset_regions; + post_initialize_heuristics(); // We are initializing free set. We ignore cset region tallies. - size_t first_old, last_old, num_old; + size_t young_cset_regions, old_cset_regions, first_old, last_old, num_old; _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); + if (mode()->is_generational()) { + ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); + // We cannot call + // gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions) + // until after the heap is fully initialized. So we make up a safe value here. + size_t allocation_runway = InitialHeapSize / 2; + gen_heap->compute_old_generation_balance(allocation_runway, old_cset_regions); + } _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old); } @@ -522,6 +530,10 @@ void ShenandoahHeap::initialize_heuristics() { _global_generation->initialize_heuristics(mode()); } +void ShenandoahHeap::post_initialize_heuristics() { + _global_generation->post_initialize(this); +} + #ifdef _MSC_VER #pragma warning( push ) #pragma warning( disable:4355 ) // 'this' : used in base member initializer list @@ -737,46 +749,8 @@ void ShenandoahHeap::increase_used(const ShenandoahAllocRequest& req) { if (req.is_gc_alloc()) { assert(wasted_bytes == 0 || req.type() == ShenandoahAllocRequest::_alloc_plab, "Only PLABs have waste"); - increase_used(generation, actual_bytes + wasted_bytes); } else { assert(req.is_mutator_alloc(), "Expected mutator alloc here"); - // padding and actual size both count towards allocation counter - generation->increase_allocated(actual_bytes + wasted_bytes); - - // only actual size counts toward usage for mutator allocations - increase_used(generation, actual_bytes); - - if (wasted_bytes > 0 && ShenandoahHeapRegion::requires_humongous(req.actual_size())) { - increase_humongous_waste(generation,wasted_bytes); - } - } -} - -void ShenandoahHeap::increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes) { - generation->increase_humongous_waste(bytes); - if (!generation->is_global()) { - global_generation()->increase_humongous_waste(bytes); - } -} - -void ShenandoahHeap::decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes) { - generation->decrease_humongous_waste(bytes); - if (!generation->is_global()) { - global_generation()->decrease_humongous_waste(bytes); - } -} - -void ShenandoahHeap::increase_used(ShenandoahGeneration* generation, size_t bytes) { - generation->increase_used(bytes); - if (!generation->is_global()) { - global_generation()->increase_used(bytes); - } -} - -void ShenandoahHeap::decrease_used(ShenandoahGeneration* generation, size_t bytes) { - generation->decrease_used(bytes); - if (!generation->is_global()) { - global_generation()->decrease_used(bytes); } } @@ -2311,12 +2285,9 @@ address ShenandoahHeap::in_cset_fast_test_addr() { } void ShenandoahHeap::reset_bytes_allocated_since_gc_start() { - if (mode()->is_generational()) { - young_generation()->reset_bytes_allocated_since_gc_start(); - old_generation()->reset_bytes_allocated_since_gc_start(); - } - - global_generation()->reset_bytes_allocated_since_gc_start(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahHeapLocker locker(heap->lock()); + free_set()->reset_bytes_allocated_since_gc_start(); } void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index bec1235e94140..28c7d8ba43e4d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -203,6 +203,7 @@ class ShenandoahHeap : public CollectedHeap { void post_initialize() override; void initialize_mode(); virtual void initialize_heuristics(); + virtual void post_initialize_heuristics(); virtual void print_init_logger() const; void initialize_serviceability() override; @@ -709,8 +710,6 @@ class ShenandoahHeap : public CollectedHeap { size_t size, Metaspace::MetadataType mdtype) override; - void notify_mutator_alloc_words(size_t words, size_t waste); - HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) override; size_t tlab_capacity(Thread *thr) const override; size_t unsafe_max_tlab_alloc(Thread *thread) const override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 3eb7ccba35af3..e22baf5095340 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -578,16 +578,16 @@ void ShenandoahHeapRegion::recycle_internal() { set_affiliation(FREE); } +// Upon return, this region has been recycled. We try to recycle it. +// We may fail if some other thread recycled it before we do. void ShenandoahHeapRegion::try_recycle_under_lock() { shenandoah_assert_heaplocked(); if (is_trash() && _recycling.try_set()) { if (is_trash()) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahGeneration* generation = heap->generation_for(affiliation()); - - heap->decrease_used(generation, used()); - generation->decrement_affiliated_region_count(); - + // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as + // part of capacity, as empty, as fully available, and as unaffiliated. This provides short-lived optimism + // for triggering heuristics. It greatly simplifies and reduces the locking overhead required + // by more time-precise accounting of these details. recycle_internal(); } _recycling.unset(); @@ -608,11 +608,10 @@ void ShenandoahHeapRegion::try_recycle() { if (is_trash() && _recycling.try_set()) { // Double check region state after win the race to set recycling flag if (is_trash()) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahGeneration* generation = heap->generation_for(affiliation()); - heap->decrease_used(generation, used()); - generation->decrement_affiliated_region_count_without_lock(); - + // At freeset rebuild time, which precedes recycling of collection set, we treat all cset regions as + // part of capacity, as empty, as fully available, and as unaffiliated. This provides short-lived optimism + // for triggering and pacing heuristics. It greatly simplifies and reduces the locking overhead required + // by more time-precise accounting of these details. recycle_internal(); } _recycling.unset(); @@ -900,12 +899,11 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation heap->set_affiliation(this, new_affiliation); } -void ShenandoahHeapRegion::decrement_humongous_waste() const { +void ShenandoahHeapRegion::decrement_humongous_waste() { assert(is_humongous(), "Should only use this for humongous regions"); size_t waste_bytes = free(); if (waste_bytes > 0) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahGeneration* generation = heap->generation_for(affiliation()); - heap->decrease_humongous_waste(generation, waste_bytes); + heap->free_set()->decrease_humongous_waste_for_regular_bypass(this, waste_bytes); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 4c99364bc6ed4..a56efcc4ebae9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -366,6 +366,9 @@ class ShenandoahHeapRegion { // Allocation (return nullptr if full) inline HeapWord* allocate(size_t word_size, const ShenandoahAllocRequest& req); + // Allocate fill after top + inline HeapWord* allocate_fill(size_t word_size); + inline void clear_live_data(); void set_live_data(size_t s); @@ -492,7 +495,7 @@ class ShenandoahHeapRegion { } private: - void decrement_humongous_waste() const; + void decrement_humongous_waste(); void do_commit(); void do_uncommit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 53f86c8cc58b6..232a1c8d2d8ed 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -87,6 +87,23 @@ HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocReq } } +HeapWord* ShenandoahHeapRegion::allocate_fill(size_t size) { + shenandoah_assert_heaplocked_or_safepoint(); + assert(is_object_aligned(size), "alloc size breaks alignment: %zu", size); + assert(size >= ShenandoahHeap::min_fill_size(), "Cannot fill unless min fill size"); + + HeapWord* obj = top(); + HeapWord* new_top = obj + size; + ShenandoahHeap::fill_with_object(obj, size); + set_top(new_top); + + assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top)); + assert(is_object_aligned(obj), "obj is not aligned: " PTR_FORMAT, p2i(obj)); + + return obj; +} + + HeapWord* ShenandoahHeapRegion::allocate(size_t size, const ShenandoahAllocRequest& req) { shenandoah_assert_heaplocked_or_safepoint(); assert(is_object_aligned(size), "alloc size breaks alignment: %zu", size); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp index ebfe5267160fb..93d40b839e93d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp @@ -63,7 +63,6 @@ MemoryUsage ShenandoahMemoryPool::get_memory_usage() { // to make sense under the race. See JDK-8207200. committed = MAX2(used, committed); assert(used <= committed, "used: %zu, committed: %zu", used, committed); - return MemoryUsage(initial, used, committed, max); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 1724fc2849f76..ed24a6e671992 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -144,21 +144,7 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // collection. heap->concurrent_final_roots(); - // We do not rebuild_free following increments of old marking because memory has not been reclaimed. However, we may - // need to transfer memory to OLD in order to efficiently support the mixed evacuations that might immediately follow. size_t allocation_runway = heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(0); heap->compute_old_generation_balance(allocation_runway, 0); - - ShenandoahGenerationalHeap::TransferResult result; - { - ShenandoahHeapLocker locker(heap->lock()); - result = heap->balance_generations(); - } - - LogTarget(Info, gc, ergo) lt; - if (lt.is_enabled()) { - LogStream ls(lt); - result.print_on("Old Mark", &ls); - } return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 5cccd395d3819..8e868b7c9b29f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -511,12 +511,20 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); ShenandoahHeapLocker locker(heap->lock()); - size_t cset_young_regions, cset_old_regions; + size_t young_trash_regions, old_trash_regions; size_t first_old, last_old, num_old; - heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old); - // This is just old-gen completion. No future budgeting required here. The only reason to rebuild the freeset here - // is in case there was any immediate old garbage identified. - heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old); + heap->free_set()->prepare_to_rebuild(young_trash_regions, old_trash_regions, first_old, last_old, num_old); + // At the end of old-gen, we may find that we have reclaimed immediate garbage, allowing a longer allocation runway. + // We may also find that we have accumulated canddiate regions for mixed evacuation. If so, we will want to expand + // the OldCollector reserve in order to make room for these mixed evacuations. + assert(ShenandoahHeap::heap()->mode()->is_generational(), "sanity"); + assert(young_trash_regions == 0, "sanity"); + ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap(); + size_t allocation_runway = + gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_trash_regions); + gen_heap->compute_old_generation_balance(allocation_runway, old_trash_regions); + + heap->free_set()->finish_rebuild(young_trash_regions, old_trash_regions, num_old); } } @@ -717,11 +725,6 @@ void ShenandoahOldGeneration::handle_evacuation(HeapWord* obj, size_t words, boo // do this in batch, in a background GC thread than to try to carefully dirty only cards // that hold interesting pointers right now. _card_scan->mark_range_as_dirty(obj, words); - - if (promotion) { - // This evacuation was a promotion, track this as allocation against old gen - increase_allocated(words * HeapWordSize); - } } bool ShenandoahOldGeneration::has_unprocessed_collection_candidates() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index abc865c31cd1e..edbe7b3538911 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -100,6 +100,11 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { return _old_heuristics; } + // Use this only for unit testing. Do not use for production. + inline void set_capacity(size_t bytes) { + ShenandoahHeap::heap()->free_set()->resize_old_collector_capacity(bytes / ShenandoahHeapRegion::region_size_bytes()); + } + // See description in field declaration void set_promoted_reserve(size_t new_val); size_t get_promoted_reserve() const; @@ -135,7 +140,9 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { void configure_plab_for_current_thread(const ShenandoahAllocRequest &req); // See description in field declaration - void set_region_balance(ssize_t balance) { _region_balance = balance; } + void set_region_balance(ssize_t balance) { + _region_balance = balance; + } ssize_t get_region_balance() const { return _region_balance; } // See description in field declaration void set_promotion_potential(size_t val) { _promotion_potential = val; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp index 3f4bbafb755e3..eb55755b1711f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.cpp @@ -25,7 +25,7 @@ #include "gc/shenandoah/shenandoahSimpleBitMap.inline.hpp" -ShenandoahSimpleBitMap::ShenandoahSimpleBitMap(size_t num_bits) : +ShenandoahSimpleBitMap::ShenandoahSimpleBitMap(index_type num_bits) : _num_bits(num_bits), _num_words(align_up(num_bits, BitsPerWord) / BitsPerWord), _bitmap(NEW_C_HEAP_ARRAY(uintx, _num_words, mtGC)) @@ -39,7 +39,7 @@ ShenandoahSimpleBitMap::~ShenandoahSimpleBitMap() { } } -size_t ShenandoahSimpleBitMap::count_leading_ones(idx_t start_idx) const { +size_t ShenandoahSimpleBitMap::count_leading_ones(index_type start_idx) const { assert((start_idx >= 0) && (start_idx < _num_bits), "precondition"); size_t array_idx = start_idx >> LogBitsPerWord; uintx element_bits = _bitmap[array_idx]; @@ -66,7 +66,7 @@ size_t ShenandoahSimpleBitMap::count_leading_ones(idx_t start_idx) const { return counted_ones + count_trailing_zeros(complement); } -size_t ShenandoahSimpleBitMap::count_trailing_ones(idx_t last_idx) const { +size_t ShenandoahSimpleBitMap::count_trailing_ones(index_type last_idx) const { assert((last_idx >= 0) && (last_idx < _num_bits), "precondition"); size_t array_idx = last_idx >> LogBitsPerWord; uintx element_bits = _bitmap[array_idx]; @@ -93,11 +93,11 @@ size_t ShenandoahSimpleBitMap::count_trailing_ones(idx_t last_idx) const { return counted_ones + count_leading_zeros(complement); } -bool ShenandoahSimpleBitMap::is_forward_consecutive_ones(idx_t start_idx, idx_t count) const { +bool ShenandoahSimpleBitMap::is_forward_consecutive_ones(index_type start_idx, index_type count) const { while (count > 0) { assert((start_idx >= 0) && (start_idx < _num_bits), "precondition: start_idx: %zd, count: %zd", start_idx, count); - assert(start_idx + count <= (idx_t) _num_bits, "precondition"); + assert(start_idx + count <= (index_type) _num_bits, "precondition"); size_t array_idx = start_idx >> LogBitsPerWord; uintx bit_number = start_idx & (BitsPerWord - 1); uintx element_bits = _bitmap[array_idx]; @@ -123,7 +123,7 @@ bool ShenandoahSimpleBitMap::is_forward_consecutive_ones(idx_t start_idx, idx_t return true; } -bool ShenandoahSimpleBitMap::is_backward_consecutive_ones(idx_t last_idx, idx_t count) const { +bool ShenandoahSimpleBitMap::is_backward_consecutive_ones(index_type last_idx, index_type count) const { while (count > 0) { assert((last_idx >= 0) && (last_idx < _num_bits), "precondition"); assert(last_idx - count >= -1, "precondition"); @@ -152,11 +152,11 @@ bool ShenandoahSimpleBitMap::is_backward_consecutive_ones(idx_t last_idx, idx_t return true; } -idx_t ShenandoahSimpleBitMap::find_first_consecutive_set_bits(idx_t beg, idx_t end, size_t num_bits) const { +index_type ShenandoahSimpleBitMap::find_first_consecutive_set_bits(index_type beg, index_type end, size_t num_bits) const { assert((beg >= 0) && (beg < _num_bits), "precondition"); // Stop looking if there are not num_bits remaining in probe space. - idx_t start_boundary = end - num_bits; + index_type start_boundary = end - num_bits; if (beg > start_boundary) { return end; } @@ -231,12 +231,12 @@ idx_t ShenandoahSimpleBitMap::find_first_consecutive_set_bits(idx_t beg, idx_t e } } -idx_t ShenandoahSimpleBitMap::find_last_consecutive_set_bits(const idx_t beg, idx_t end, const size_t num_bits) const { +index_type ShenandoahSimpleBitMap::find_last_consecutive_set_bits(const index_type beg, index_type end, const size_t num_bits) const { assert((end >= 0) && (end < _num_bits), "precondition"); // Stop looking if there are not num_bits remaining in probe space. - idx_t last_boundary = beg + num_bits; + index_type last_boundary = beg + num_bits; if (end < last_boundary) { return beg; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp index 3a4cb8cf742fc..7f86220d38e1f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.hpp @@ -35,29 +35,29 @@ // 1. Allow searches from high to low memory (when biasing allocations towards the top of the heap) // 2. Allow searches for clusters of contiguous set bits (to expedite allocation for humongous objects) // -// idx_t is defined here as ssize_t. In src/hotspot/share/utiliities/bitMap.hpp, idx is defined as size_t. +// index_type is defined here as ssize_t. In src/hotspot/share/utiliities/bitMap.hpp, idx is defined as size_t. // This is a significant incompatibility. // -// The API and internal implementation of ShenandoahSimpleBitMap and ShenandoahRegionPartitions use idx_t to +// The API and internal implementation of ShenandoahSimpleBitMap and ShenandoahRegionPartitions use index_type to // represent index, even though index is "inherently" unsigned. There are two reasons for this choice: // 1. We use -1 as a sentinel value to represent empty partitions. This same value may be used to represent // failure to find a previous set bit or previous range of set bits. -// 2. Certain loops are written most naturally if the iterator, which may hold the sentinel -1 value, can be +// 2. Certain loops are written most naturally if the induction variable, which may hold the sentinel -1 value, can be // declared as signed and the terminating condition can be < 0. -typedef ssize_t idx_t; +typedef ssize_t index_type; // ShenandoahSimpleBitMap resembles CHeapBitMap but adds missing support for find_first_consecutive_set_bits() and // find_last_consecutive_set_bits. An alternative refactoring of code would subclass CHeapBitMap, but this might // break abstraction rules, because efficient implementation requires assumptions about superclass internals that // might be violated through future software maintenance. class ShenandoahSimpleBitMap { - const idx_t _num_bits; + const index_type _num_bits; const size_t _num_words; uintx* const _bitmap; public: - ShenandoahSimpleBitMap(size_t num_bits); + ShenandoahSimpleBitMap(index_type num_bits); ~ShenandoahSimpleBitMap(); @@ -71,42 +71,42 @@ class ShenandoahSimpleBitMap { // Count consecutive ones in forward order, starting from start_idx. Requires that there is at least one zero // between start_idx and index value (_num_bits - 1), inclusive. - size_t count_leading_ones(idx_t start_idx) const; + size_t count_leading_ones(index_type start_idx) const; // Count consecutive ones in reverse order, starting from last_idx. Requires that there is at least one zero // between last_idx and index value zero, inclusive. - size_t count_trailing_ones(idx_t last_idx) const; + size_t count_trailing_ones(index_type last_idx) const; - bool is_forward_consecutive_ones(idx_t start_idx, idx_t count) const; - bool is_backward_consecutive_ones(idx_t last_idx, idx_t count) const; + bool is_forward_consecutive_ones(index_type start_idx, index_type count) const; + bool is_backward_consecutive_ones(index_type last_idx, index_type count) const; static inline uintx tail_mask(uintx bit_number); public: - inline idx_t aligned_index(idx_t idx) const { + inline index_type aligned_index(index_type idx) const { assert((idx >= 0) && (idx < _num_bits), "precondition"); - idx_t array_idx = idx & ~(BitsPerWord - 1); + index_type array_idx = idx & ~(BitsPerWord - 1); return array_idx; } - inline constexpr idx_t alignment() const { + inline constexpr index_type alignment() const { return BitsPerWord; } // For testing - inline idx_t size() const { + inline index_type size() const { return _num_bits; } // Return the word that holds idx bit and its neighboring bits. - inline uintx bits_at(idx_t idx) const { + inline uintx bits_at(index_type idx) const { assert((idx >= 0) && (idx < _num_bits), "precondition"); - idx_t array_idx = idx >> LogBitsPerWord; + index_type array_idx = idx >> LogBitsPerWord; return _bitmap[array_idx]; } - inline void set_bit(idx_t idx) { + inline void set_bit(index_type idx) { assert((idx >= 0) && (idx < _num_bits), "precondition"); size_t array_idx = idx >> LogBitsPerWord; uintx bit_number = idx & (BitsPerWord - 1); @@ -114,18 +114,16 @@ class ShenandoahSimpleBitMap { _bitmap[array_idx] |= the_bit; } - inline void clear_bit(idx_t idx) { + inline void clear_bit(index_type idx) { assert((idx >= 0) && (idx < _num_bits), "precondition"); - assert(idx >= 0, "precondition"); size_t array_idx = idx >> LogBitsPerWord; uintx bit_number = idx & (BitsPerWord - 1); uintx the_bit = nth_bit(bit_number); _bitmap[array_idx] &= ~the_bit; } - inline bool is_set(idx_t idx) const { + inline bool is_set(index_type idx) const { assert((idx >= 0) && (idx < _num_bits), "precondition"); - assert(idx >= 0, "precondition"); size_t array_idx = idx >> LogBitsPerWord; uintx bit_number = idx & (BitsPerWord - 1); uintx the_bit = nth_bit(bit_number); @@ -134,39 +132,39 @@ class ShenandoahSimpleBitMap { // Return the index of the first set bit in the range [beg, size()), or size() if none found. // precondition: beg and end form a valid range for the bitmap. - inline idx_t find_first_set_bit(idx_t beg) const; + inline index_type find_first_set_bit(index_type beg) const; // Return the index of the first set bit in the range [beg, end), or end if none found. // precondition: beg and end form a valid range for the bitmap. - inline idx_t find_first_set_bit(idx_t beg, idx_t end) const; + inline index_type find_first_set_bit(index_type beg, index_type end) const; // Return the index of the last set bit in the range (-1, end], or -1 if none found. // precondition: beg and end form a valid range for the bitmap. - inline idx_t find_last_set_bit(idx_t end) const; + inline index_type find_last_set_bit(index_type end) const; // Return the index of the last set bit in the range (beg, end], or beg if none found. // precondition: beg and end form a valid range for the bitmap. - inline idx_t find_last_set_bit(idx_t beg, idx_t end) const; + inline index_type find_last_set_bit(index_type beg, index_type end) const; // Return the start index of the first run of consecutive set bits for which the first set bit is within // the range [beg, size()), or size() if the run of is not found within this range. // precondition: beg is within the valid range for the bitmap. - inline idx_t find_first_consecutive_set_bits(idx_t beg, size_t num_bits) const; + inline index_type find_first_consecutive_set_bits(index_type beg, size_t num_bits) const; // Return the start index of the first run of consecutive set bits for which the first set bit is within // the range [beg, end), or end if the run of is not found within this range. // precondition: beg and end form a valid range for the bitmap. - idx_t find_first_consecutive_set_bits(idx_t beg, idx_t end, size_t num_bits) const; + index_type find_first_consecutive_set_bits(index_type beg, index_type end, size_t num_bits) const; // Return the start index of the last run of consecutive set bits for which the entire run of set bits is within // the range (-1, end], or -1 if the run of is not found within this range. // precondition: end is within the valid range for the bitmap. - inline idx_t find_last_consecutive_set_bits(idx_t end, size_t num_bits) const; + inline index_type find_last_consecutive_set_bits(index_type end, size_t num_bits) const; // Return the start index of the first run of consecutive set bits for which the entire run of set bits is within // the range (beg, end], or beg if the run of is not found within this range. // precondition: beg and end form a valid range for the bitmap. - idx_t find_last_consecutive_set_bits(idx_t beg, idx_t end, size_t num_bits) const; + index_type find_last_consecutive_set_bits(index_type beg, index_type end, size_t num_bits) const; }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSIMPLEBITMAP_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp index 4582ab9a781dd..423c876a880d0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSimpleBitMap.inline.hpp @@ -34,7 +34,7 @@ inline uintx ShenandoahSimpleBitMap::tail_mask(uintx bit_number) { return (uintx(1) << bit_number) - 1; } -inline idx_t ShenandoahSimpleBitMap::find_first_set_bit(idx_t beg, idx_t end) const { +inline index_type ShenandoahSimpleBitMap::find_first_set_bit(index_type beg, index_type end) const { assert((beg >= 0) && (beg < _num_bits), "precondition"); assert((end > beg) && (end <= _num_bits), "precondition"); do { @@ -49,7 +49,7 @@ inline idx_t ShenandoahSimpleBitMap::find_first_set_bit(idx_t beg, idx_t end) co // The next set bit is here. Find first set bit >= bit_number; uintx aligned = element_bits >> bit_number; uintx first_set_bit = count_trailing_zeros(aligned); - idx_t candidate_result = (array_idx * BitsPerWord) + bit_number + first_set_bit; + index_type candidate_result = (array_idx * BitsPerWord) + bit_number + first_set_bit; return (candidate_result < end)? candidate_result: end; } else { // Next bit is not here. Try the next array element @@ -59,16 +59,16 @@ inline idx_t ShenandoahSimpleBitMap::find_first_set_bit(idx_t beg, idx_t end) co return end; } -inline idx_t ShenandoahSimpleBitMap::find_first_set_bit(idx_t beg) const { +inline index_type ShenandoahSimpleBitMap::find_first_set_bit(index_type beg) const { assert((beg >= 0) && (beg < size()), "precondition"); return find_first_set_bit(beg, size()); } -inline idx_t ShenandoahSimpleBitMap::find_last_set_bit(idx_t beg, idx_t end) const { +inline index_type ShenandoahSimpleBitMap::find_last_set_bit(index_type beg, index_type end) const { assert((end >= 0) && (end < _num_bits), "precondition"); assert((beg >= -1) && (beg < end), "precondition"); do { - idx_t array_idx = end >> LogBitsPerWord; + index_type array_idx = end >> LogBitsPerWord; uint8_t bit_number = end & (BitsPerWord - 1); uintx element_bits = _bitmap[array_idx]; if (bit_number < BitsPerWord - 1){ @@ -79,7 +79,7 @@ inline idx_t ShenandoahSimpleBitMap::find_last_set_bit(idx_t beg, idx_t end) con // The prev set bit is here. Find the first set bit <= bit_number uintx aligned = element_bits << (BitsPerWord - (bit_number + 1)); uintx first_set_bit = count_leading_zeros(aligned); - idx_t candidate_result = array_idx * BitsPerWord + (bit_number - first_set_bit); + index_type candidate_result = array_idx * BitsPerWord + (bit_number - first_set_bit); return (candidate_result > beg)? candidate_result: beg; } else { // Next bit is not here. Try the previous array element @@ -89,19 +89,19 @@ inline idx_t ShenandoahSimpleBitMap::find_last_set_bit(idx_t beg, idx_t end) con return beg; } -inline idx_t ShenandoahSimpleBitMap::find_last_set_bit(idx_t end) const { +inline index_type ShenandoahSimpleBitMap::find_last_set_bit(index_type end) const { assert((end >= 0) && (end < _num_bits), "precondition"); return find_last_set_bit(-1, end); } -inline idx_t ShenandoahSimpleBitMap::find_first_consecutive_set_bits(idx_t beg, size_t num_bits) const { +inline index_type ShenandoahSimpleBitMap::find_first_consecutive_set_bits(index_type beg, size_t num_bits) const { assert((beg >= 0) && (beg < _num_bits), "precondition"); return find_first_consecutive_set_bits(beg, size(), num_bits); } -inline idx_t ShenandoahSimpleBitMap::find_last_consecutive_set_bits(idx_t end, size_t num_bits) const { +inline index_type ShenandoahSimpleBitMap::find_last_consecutive_set_bits(index_type end, size_t num_bits) const { assert((end >= 0) && (end < _num_bits), "precondition"); - return find_last_consecutive_set_bits((idx_t) -1, end, num_bits); + return find_last_consecutive_set_bits((index_type) -1, end, num_bits); } #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSIMPLEBITMAP_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 33b8744be3d88..9e9129ee112cc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -375,30 +375,57 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { // a subset (e.g. the young generation or old generation) of the total heap. class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { private: - size_t _used, _committed, _garbage, _regions, _humongous_waste, _trashed_regions; + size_t _used, _committed, _garbage, _regions, _humongous_waste, _trashed_regions, _trashed_used; + size_t _region_size_bytes, _min_free_size; public: ShenandoahCalculateRegionStatsClosure() : - _used(0), _committed(0), _garbage(0), _regions(0), _humongous_waste(0), _trashed_regions(0) {}; + _used(0), _committed(0), _garbage(0), _regions(0), _humongous_waste(0), _trashed_regions(0), _trashed_used(0) + { + _region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + // Retired regions are not necessarily filled, thouugh their remnant memory is considered used. + _min_free_size = PLAB::min_size() * HeapWordSize; + }; void heap_region_do(ShenandoahHeapRegion* r) override { - _used += r->used(); - _garbage += r->garbage(); - _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; - if (r->is_humongous()) { - _humongous_waste += r->free(); - } - if (r->is_trash()) { - _trashed_regions++; + if (r->is_cset() || r->is_trash()) { + // Count the entire cset or trashed (formerly cset) region as used + // Note: Immediate garbage trash regions were never in the cset. + _used += _region_size_bytes; + _garbage += _region_size_bytes - r->get_live_data_bytes(); + if (r->is_trash()) { + _trashed_regions++; + _trashed_used += _region_size_bytes; + } + } else { + if (r->is_humongous()) { + _used += _region_size_bytes; + _garbage += _region_size_bytes - r->get_live_data_bytes(); + _humongous_waste += r->free(); + } else { + size_t alloc_capacity = r->free(); + if (alloc_capacity < _min_free_size) { + // this region has been retired already, count it as entirely consumed + alloc_capacity = 0; + } + size_t bytes_used_in_region = _region_size_bytes - alloc_capacity; + size_t bytes_garbage_in_region = bytes_used_in_region - r->get_live_data_bytes(); + size_t waste_bytes = r->free(); + _used += bytes_used_in_region; + _garbage += bytes_garbage_in_region; + } } + _committed += r->is_committed() ? _region_size_bytes : 0; _regions++; log_debug(gc)("ShenandoahCalculateRegionStatsClosure: adding %zu for %s Region %zu, yielding: %zu", r->used(), (r->is_humongous() ? "humongous" : "regular"), r->index(), _used); } size_t used() const { return _used; } + size_t used_after_recycle() const { return _used - _trashed_used; } size_t committed() const { return _committed; } size_t garbage() const { return _garbage; } size_t regions() const { return _regions; } + size_t trashed_regions() const { return _trashed_regions; } size_t waste() const { return _humongous_waste; } // span is the total memory affiliated with these stats (some of which is in use and other is available) @@ -408,21 +435,21 @@ class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { public: - ShenandoahCalculateRegionStatsClosure old; - ShenandoahCalculateRegionStatsClosure young; - ShenandoahCalculateRegionStatsClosure global; + ShenandoahCalculateRegionStatsClosure _old; + ShenandoahCalculateRegionStatsClosure _young; + ShenandoahCalculateRegionStatsClosure _global; void heap_region_do(ShenandoahHeapRegion* r) override { switch (r->affiliation()) { case FREE: return; case YOUNG_GENERATION: - young.heap_region_do(r); - global.heap_region_do(r); + _young.heap_region_do(r); + _global.heap_region_do(r); break; case OLD_GENERATION: - old.heap_region_do(r); - global.heap_region_do(r); + _old.heap_region_do(r); + _global.heap_region_do(r); break; default: ShouldNotReachHere(); @@ -436,23 +463,22 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); } - static void validate_usage(const bool adjust_for_padding, + static void validate_usage(const bool adjust_for_padding, const bool adjust_for_trash, const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { ShenandoahHeap* heap = ShenandoahHeap::heap(); size_t generation_used = generation->used(); size_t generation_used_regions = generation->used_regions(); - if (adjust_for_padding && (generation->is_young() || generation->is_global())) { - size_t pad = heap->old_generation()->get_pad_for_promote_in_place(); - generation_used += pad; - } - guarantee(stats.used() == generation_used, + size_t stats_used = adjust_for_trash? stats.used_after_recycle(): stats.used(); + guarantee(stats_used == generation_used, "%s: generation (%s) used size must be consistent: generation-used: " PROPERFMT ", regions-used: " PROPERFMT, - label, generation->name(), PROPERFMTARGS(generation_used), PROPERFMTARGS(stats.used())); + label, generation->name(), PROPERFMTARGS(generation_used), PROPERFMTARGS(stats_used)); - guarantee(stats.regions() == generation_used_regions, - "%s: generation (%s) used regions (%zu) must equal regions that are in use (%zu)", - label, generation->name(), generation->used_regions(), stats.regions()); + size_t stats_regions = adjust_for_trash? stats.regions() - stats.trashed_regions(): stats.regions(); + guarantee(stats_regions == generation_used_regions, + "%s: generation (%s) used regions (%zu) must equal regions that are in use (%zu)%s", + label, generation->name(), generation->used_regions(), stats_regions, + adjust_for_trash? " (after adjusting for trash)": ""); size_t generation_capacity = generation->max_capacity(); guarantee(stats.non_trashed_span() <= generation_capacity, @@ -469,15 +495,15 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { }; class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { -private: + private: ShenandoahHeap* _heap; const char* _phase; ShenandoahVerifier::VerifyRegions _regions; -public: + public: ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) : - _heap(ShenandoahHeap::heap()), - _phase(phase), - _regions(regions) {}; + _heap(ShenandoahHeap::heap()), + _phase(phase), + _regions(regions) {}; void print_failure(ShenandoahHeapRegion* r, const char* label) { ResourceMark rm; @@ -564,7 +590,7 @@ class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { }; class ShenandoahVerifierReachableTask : public WorkerTask { -private: + private: const char* _label; ShenandoahVerifier::VerifyOptions _options; ShenandoahHeap* _heap; @@ -572,18 +598,18 @@ class ShenandoahVerifierReachableTask : public WorkerTask { MarkBitMap* _bitmap; volatile size_t _processed; -public: + public: ShenandoahVerifierReachableTask(MarkBitMap* bitmap, ShenandoahLivenessData* ld, const char* label, ShenandoahVerifier::VerifyOptions options) : - WorkerTask("Shenandoah Verifier Reachable Objects"), - _label(label), - _options(options), - _heap(ShenandoahHeap::heap()), - _ld(ld), - _bitmap(bitmap), - _processed(0) {}; + WorkerTask("Shenandoah Verifier Reachable Objects"), + _label(label), + _options(options), + _heap(ShenandoahHeap::heap()), + _ld(ld), + _bitmap(bitmap), + _processed(0) {}; size_t processed() const { return _processed; @@ -599,14 +625,14 @@ class ShenandoahVerifierReachableTask : public WorkerTask { // extended parallelism would buy us out. if (((ShenandoahVerifyLevel == 2) && (worker_id == 0)) || (ShenandoahVerifyLevel >= 3)) { - ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, - ShenandoahMessageBuffer("%s, Roots", _label), - _options); - if (_heap->unload_classes()) { - ShenandoahRootVerifier::strong_roots_do(&cl); - } else { - ShenandoahRootVerifier::roots_do(&cl); - } + ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, + ShenandoahMessageBuffer("%s, Roots", _label), + _options); + if (_heap->unload_classes()) { + ShenandoahRootVerifier::strong_roots_do(&cl); + } else { + ShenandoahRootVerifier::roots_do(&cl); + } } size_t processed = 0; @@ -627,7 +653,7 @@ class ShenandoahVerifierReachableTask : public WorkerTask { }; class ShenandoahVerifyNoIncompleteSatbBuffers : public ThreadClosure { -public: + public: void do_thread(Thread* thread) override { SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); if (!queue.is_empty()) { @@ -637,7 +663,7 @@ class ShenandoahVerifyNoIncompleteSatbBuffers : public ThreadClosure { }; class ShenandoahVerifierMarkedRegionTask : public WorkerTask { -private: + private: const char* _label; ShenandoahVerifier::VerifyOptions _options; ShenandoahHeap *_heap; @@ -647,20 +673,20 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { volatile size_t _processed; ShenandoahGeneration* _generation; -public: + public: ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap, ShenandoahLivenessData* ld, const char* label, ShenandoahVerifier::VerifyOptions options) : - WorkerTask("Shenandoah Verifier Marked Objects"), - _label(label), - _options(options), - _heap(ShenandoahHeap::heap()), - _bitmap(bitmap), - _ld(ld), - _claimed(0), - _processed(0), - _generation(nullptr) { + WorkerTask("Shenandoah Verifier Marked Objects"), + _label(label), + _options(options), + _heap(ShenandoahHeap::heap()), + _bitmap(bitmap), + _ld(ld), + _claimed(0), + _processed(0), + _generation(nullptr) { if (_heap->mode()->is_generational()) { _generation = _heap->gc_generation(); assert(_generation != nullptr, "Expected active generation in this mode."); @@ -770,11 +796,11 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { }; class VerifyThreadGCState : public ThreadClosure { -private: + private: const char* const _label; - char const _expected; + char const _expected; -public: + public: VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {} void do_thread(Thread* t) override { char actual = ShenandoahThreadLocalData::gc_state(t); @@ -875,16 +901,18 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, size_t heap_used; if (_heap->mode()->is_generational() && (sizeness == _verify_size_adjusted_for_padding)) { // Prior to evacuation, regular regions that are to be evacuated in place are padded to prevent further allocations - heap_used = _heap->used() + _heap->old_generation()->get_pad_for_promote_in_place(); + // but this padding is already represented in _heap->used() + heap_used = _heap->used(); } else if (sizeness != _verify_size_disable) { heap_used = _heap->used(); } if (sizeness != _verify_size_disable) { - guarantee(cl.used() == heap_used, + size_t cl_size = (sizeness == _verify_size_exact_including_trash)? cl.used(): cl.used_after_recycle(); + guarantee(cl_size == heap_used, "%s: heap used size must be consistent: heap-used = %zu%s, regions-used = %zu%s", label, byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used), - byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used())); + byte_size_in_proper_unit(cl_size), proper_unit_for_byte_size(cl_size)); } size_t heap_committed = _heap->committed(); guarantee(cl.committed() == heap_committed, @@ -931,18 +959,19 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, _heap->heap_region_iterate(&cl); if (LogTarget(Debug, gc)::is_enabled()) { - ShenandoahGenerationStatsClosure::log_usage(_heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::log_usage(_heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::log_usage(_heap->global_generation(), cl.global); + ShenandoahGenerationStatsClosure::log_usage(_heap->old_generation(), cl._old); + ShenandoahGenerationStatsClosure::log_usage(_heap->young_generation(), cl._young); + ShenandoahGenerationStatsClosure::log_usage(_heap->global_generation(), cl._global); } if (sizeness == _verify_size_adjusted_for_padding) { - ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::validate_usage(true, label, _heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::validate_usage(true, label, _heap->global_generation(), cl.global); - } else if (sizeness == _verify_size_exact) { - ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->global_generation(), cl.global); + ShenandoahGenerationStatsClosure::validate_usage(false, true, label, _heap->old_generation(), cl._old); + ShenandoahGenerationStatsClosure::validate_usage(true, true, label, _heap->young_generation(), cl._young); + ShenandoahGenerationStatsClosure::validate_usage(true, true, label, _heap->global_generation(), cl._global); + } else if (sizeness == _verify_size_exact || sizeness == _verify_size_exact_including_trash) { + bool adjust_trash = (sizeness == _verify_size_exact); + ShenandoahGenerationStatsClosure::validate_usage(false, adjust_trash, label, _heap->old_generation(), cl._old); + ShenandoahGenerationStatsClosure::validate_usage(false, adjust_trash, label, _heap->young_generation(), cl._young); + ShenandoahGenerationStatsClosure::validate_usage(false, adjust_trash, label, _heap->global_generation(), cl._global); } // else: sizeness must equal _verify_size_disable } @@ -1158,7 +1187,8 @@ void ShenandoahVerifier::verify_after_update_refs() { _verify_cset_none, // no cset references, all updated _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_nocset, // no cset regions, trash regions have appeared - _verify_size_exact, // expect generation and heap sizes to match exactly + // expect generation and heap sizes to match exactly, including trash + _verify_size_exact_including_trash, _verify_gcstate_stable // update refs had cleaned up forwarded objects ); } @@ -1422,7 +1452,7 @@ void ShenandoahVerifier::verify_before_rebuilding_free_set() { ShenandoahGenerationStatsClosure cl; _heap->heap_region_iterate(&cl); - ShenandoahGenerationStatsClosure::validate_usage(false, "Before free set rebuild", _heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::validate_usage(false, "Before free set rebuild", _heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::validate_usage(false, "Before free set rebuild", _heap->global_generation(), cl.global); + ShenandoahGenerationStatsClosure::validate_usage(false, true, "Before free set rebuild", _heap->old_generation(), cl._old); + ShenandoahGenerationStatsClosure::validate_usage(false, true, "Before free set rebuild", _heap->young_generation(), cl._young); + ShenandoahGenerationStatsClosure::validate_usage(false, true, "Before free set rebuild", _heap->global_generation(), cl._global); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index aba6379e0223c..c4b1d6f313cee 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -155,7 +155,10 @@ class ShenandoahVerifier : public CHeapObj { _verify_size_exact, // Expect promote-in-place adjustments: padding inserted to temporarily prevent further allocation in regular regions - _verify_size_adjusted_for_padding + _verify_size_adjusted_for_padding, + + // Expected heap size should not include + _verify_size_exact_including_trash } VerifySize; typedef enum { diff --git a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java index 37359b038b358..dd2c4adf2e5d0 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java @@ -144,12 +144,25 @@ * @requires vm.gc.Shenandoah * @library /test/lib * - * @run main/othervm/timeout=240 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm/timeout=480 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+UseShenandoahGC * -XX:-UseTLAB -XX:+ShenandoahVerify * TestSieveObjects */ +/* + * @test id=no-tlab-genshen + * @summary Acceptance tests: collector can deal with retained objects + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm/timeout=480 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:-UseTLAB -XX:+ShenandoahVerify + * TestSieveObjects + */ + import java.util.Random; import jdk.test.lib.Utils; diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java index bd3c4508c7b84..7c4fdf28e8f52 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java @@ -159,7 +159,8 @@ public void handleNotification(Notification n, Object o) { final int size = 100_000; long count = TARGET_MB * 1024 * 1024 / (16 + 4 * size); - long mem = count * (16 + 4 * size); + long anticipated_humongous_waste_per_array = 124_272; + long mem = count * (16 + 4 * size + anticipated_humongous_waste_per_array); for (int c = 0; c < count; c++) { sink = new int[size];