diff --git a/Makefile b/Makefile index b059fe772f..18f857dbf4 100644 --- a/Makefile +++ b/Makefile @@ -2171,9 +2171,18 @@ gc-obj-budget-test: $(SPINEL) $(SP_RT_LIB) $(SPINEL_TIMEOUT) # gate has to decline, which is read against `walk`. The collection counts are # compared across arms: they are counts, so a busy machine gives the same # answer as an idle one. +# The gate reads the mark share the last collection MEASURED, not what +# SPINEL_GC_OBJ_BUDGET made of it for the object budget, and four more runs +# hold that: `obj` there must not take the share off the first program, `walk` +# must not put it on the second, and neither must `fixed` (with the object +# floor raised, so that budget is held where the string trigger still decides +# when to collect), read against `walk` under the same held budget. +# gc_str_budget_kept is the reading itself: a sweep that reclaims nothing puts +# no share in the trigger, so the last `[gc]` line has to say zero -- on a +# program whose first half did carry it, which the count against `str` shows. gc-str-budget-test: $(SPINEL) $(SP_RT_LIB) $(SPINEL_TIMEOUT) @tmp=$$(mktemp -d /tmp/spinel-gcstr.XXXXXX); ok=1; \ - for prog in gc_str_budget_objects gc_str_budget_still; do \ + for prog in gc_str_budget_objects gc_str_budget_still gc_str_budget_kept; do \ $(SPINEL) test/$$prog.rb -o "$$tmp/$$prog" >"$$tmp/cc.out" 2>&1; rc=$$?; \ [ $$rc -eq 0 ] || \ { echo "gc-str-budget-test: FAIL ($$prog: compile, exit $$rc)"; tail -5 "$$tmp/cc.out"; ok=0; continue; }; \ @@ -2186,6 +2195,17 @@ gc-str-budget-test: $(SPINEL) $(SP_RT_LIB) $(SPINEL_TIMEOUT) done; \ unset SPINEL_GC_STR_BUDGET; \ done; \ + held="SPINEL_GC_OBJ_BUDGET=fixed SPINEL_GC_THRESHOLD_OBJ_KB=65536"; \ + for arm in "gc_str_budget_objects objpin SPINEL_GC_OBJ_BUDGET=obj" \ + "gc_str_budget_still walkpin SPINEL_GC_OBJ_BUDGET=walk" \ + "gc_str_budget_still held $$held" \ + "gc_str_budget_still heldwalk $$held SPINEL_GC_STR_BUDGET=walk"; do \ + set -- $$arm; prog=$$1; tag=$$2; shift 2; \ + [ -x "$$tmp/$$prog" ] || continue; \ + env "$$@" SPINEL_GC_STATS=1 $(TIMEOUT60) "$$tmp/$$prog" > "$$tmp/$$prog.$$tag.out" 2> "$$tmp/$$prog.$$tag.err"; \ + cmp -s "$$tmp/$$prog.$$tag.out" test/$$prog.rb.expected || \ + { echo "gc-str-budget-test: FAIL ($$prog: $$* changed the answer)"; ok=0; }; \ + done; \ n_of() { sed -n 's/^\[gc\] \([0-9]*\) collections.*/\1/p' "$$1" | tail -1; }; \ obj_of() { sed -n 's/.*live \([0-9.]*\) MB obj.*/\1/p' "$$1" | tail -1; }; \ share_of() { sed -n 's/.*old obj in str trigger \([0-9.]*\) MB.*/\1/p' "$$1" | tail -1; }; \ @@ -2197,7 +2217,15 @@ gc-str-budget-test: $(SPINEL) $(SP_RT_LIB) $(SPINEL_TIMEOUT) sh=$$(share_of "$$tmp/gc_str_budget_objects.str.err"); \ gn=$$(n_of "$$tmp/gc_str_budget_still.default.err"); \ gw=$$(n_of "$$tmp/gc_str_budget_still.walk.err"); \ - for v in "$$on" "$$ow" "$$os" "$$ol" "$$oh" "$$sh" "$$gn" "$$gw"; do \ + pl=$$(obj_of "$$tmp/gc_str_budget_objects.objpin.err"); \ + ph=$$(share_of "$$tmp/gc_str_budget_objects.objpin.err"); \ + wn=$$(n_of "$$tmp/gc_str_budget_still.walkpin.err"); \ + hn=$$(n_of "$$tmp/gc_str_budget_still.held.err"); \ + hw=$$(n_of "$$tmp/gc_str_budget_still.heldwalk.err"); \ + kn=$$(n_of "$$tmp/gc_str_budget_kept.default.err"); \ + ks=$$(n_of "$$tmp/gc_str_budget_kept.str.err"); \ + kh=$$(share_of "$$tmp/gc_str_budget_kept.default.err"); \ + for v in "$$on" "$$ow" "$$os" "$$ol" "$$oh" "$$sh" "$$gn" "$$gw" "$$pl" "$$ph" "$$wn" "$$hn" "$$hw" "$$kn" "$$ks" "$$kh"; do \ [ -n "$$v" ] || { echo "gc-str-budget-test: FAIL (no [gc] line with the share on it)"; ok=0; break; }; \ done; \ awk -v s="$$sh" 'BEGIN{exit !(s == 0)}' || \ @@ -2214,6 +2242,16 @@ gc-str-budget-test: $(SPINEL) $(SP_RT_LIB) $(SPINEL_TIMEOUT) { echo "gc-str-budget-test: FAIL (the gate fell short of walk on a mark-bound program: $$on collections vs walk $$ow)"; ok=0; }; \ awk -v d="$$gn" -v w="$$gw" 'BEGIN{exit !(d * 2 > w * 3)}' || \ { echo "gc-str-budget-test: FAIL (the gate carried the share on a sweep-bound program: $$gn collections vs walk $$gw)"; ok=0; }; \ + awk -v s="$$ph" -v o="$$pl" 'BEGIN{exit !(s > o / 8)}' || \ + { echo "gc-str-budget-test: FAIL (SPINEL_GC_OBJ_BUDGET=obj took the share off the string budget: $$ph MB against $$pl MB live)"; ok=0; }; \ + awk -v d="$$wn" -v w="$$gw" 'BEGIN{exit !(d * 2 > w * 3)}' || \ + { echo "gc-str-budget-test: FAIL (SPINEL_GC_OBJ_BUDGET=walk opened the string budget's gate: $$wn collections vs walk $$gw)"; ok=0; }; \ + awk -v d="$$hn" -v w="$$hw" 'BEGIN{exit !(d * 2 > w * 3)}' || \ + { echo "gc-str-budget-test: FAIL (SPINEL_GC_OBJ_BUDGET=fixed opened the string budget's gate: $$hn collections vs walk $$hw)"; ok=0; }; \ + awk -v d="$$kn" -v s="$$ks" 'BEGIN{exit !(d * 2 < s)}' || \ + { echo "gc-str-budget-test: FAIL (gc_str_budget_kept never carried the share: $$ks -> $$kn collections)"; ok=0; }; \ + awk -v s="$$kh" 'BEGIN{exit !(s == 0)}' || \ + { echo "gc-str-budget-test: FAIL (the reading kept a share the last retune did not put in: $$kh MB)"; ok=0; }; \ rm -rf "$$tmp"; \ if [ $$ok -eq 1 ]; then echo "gc-str-budget-test: pass"; else exit 1; fi diff --git a/docs/thread.md b/docs/thread.md index 8188395cd4..614d2f8218 100644 --- a/docs/thread.md +++ b/docs/thread.md @@ -157,7 +157,7 @@ These are deliberate consequences of real parallelism, listed in | `SPINEL_GC_THRESHOLD_OBJ_KB` | the same budget for the OBJECT heap alone, overriding the pair above | | `SPINEL_GC_THRESHOLD_STR_KB` | the same for the STRING heap alone | | `SPINEL_GC_OBJ_BUDGET` | the default GATES the widening on what the last collection cost. `obj` pins it off (the object heap alone, as spinel did before 2026-09-09), `walk` pins it on (everything a mark walks). `fixed` is a separate axis: it stops re-aiming the budget after each collection and holds it at its floor | -| `SPINEL_GC_STR_BUDGET` | by default the string budget also carries a share of the object old generation, gated by the same `mark share`. `str` pins that off (the string heap alone, as spinel did before the share). `fixed` does for the STRING budget what it does for the object one | +| `SPINEL_GC_STR_BUDGET` | by default the string budget also carries a share of the object old generation, gated by the same `mark share` as it was measured (`SPINEL_GC_OBJ_BUDGET` pins the object budget, not this one). `str` pins that off (the string heap alone, as spinel did before the share). `fixed` does for the STRING budget what it does for the object one | | `SPINEL_GC_SLAB` | `0` turns the slab allocator off: every object and heap string is then its own malloc, which is what ASAN needs to see a use-after-free (the slab hides one). On by default whatever the process's malloc. It used to default off under jemalloc (linked or preloaded), whose thread caches did what the slab's free lists did and beside which the free-list slab measured 8% slower; the bitmap sweep is what jemalloc's caches cannot do, and with it the slab under jemalloc answers the same requests a second on 17% less CPU and 13% less memory | | `SPINEL_SLAB_HUGE` | `0` stops the slab asking for transparent huge pages on its arenas (`madvise(MADV_HUGEPAGE)`, effective where `transparent_hugepage` is `madvise` or `always`). On by default: an arena faults in as two 2 MB pages instead of a thousand 4 KB ones, and a list benchmark spent a third of its time in those faults | | `SPINEL_GC_CONC` | `0` sweeps under the stop-the-world barrier instead of beside the program, which is also what the verifiers (`SPINEL_GC_VERIFY`, `SPINEL_GC_VERIFY_GEN`) and aging (`SPINEL_GC_AGE`) do on their own, since they read the heap the sweep is rewriting. `SPINEL_GC_VERIFY` also checks the slab's bitmaps against their invariants at every barrier and after every chunk's sweep | @@ -286,9 +286,12 @@ the heap. So the string budget carries a share of the object OLD generation: one eighth of it, times the same `alpha`. `SPINEL_GC_STATS=1` reports what that put into -the trigger as `old obj in str trigger` on the `[gc]` line. +the trigger as `old obj in str trigger` on the `[gc]` line, zero after a sweep +that reclaimed too little to carry it. `SPINEL_GC_STR_BUDGET=str` pins the share off and `walk` pins it on, the two -ends `SPINEL_GC_OBJ_BUDGET` has. +ends `SPINEL_GC_OBJ_BUDGET` has. Each knob pins its own budget: the string +gate reads `alpha` as the last collection measured it, whatever +`SPINEL_GC_OBJ_BUDGET` made of it for the object budget, `fixed` included. Wall time and peak resident set on one laptop core, the same binary with `str` and without: diff --git a/lib/sp_alloc.c b/lib/sp_alloc.c index 4c0b9a058b..536e67944e 100644 --- a/lib/sp_alloc.c +++ b/lib/sp_alloc.c @@ -53,6 +53,10 @@ int sp_gc_obj_budget_mode = 2; /* The last gate decision, in 1024ths, so the stats line can report it and a test can read it. 1024 is `walk`, 0 is `obj`. */ size_t sp_gc_obj_alpha1024 = 1024; +/* The MEASUREMENT that decision starts from -- the mark's share of the last + collection -- before SPINEL_GC_OBJ_BUDGET has pinned it at either end. The + string budget's gate reads this one: its own knob is what pins it. */ +static size_t sp_gc_mark_share1024 = 1024; /* SPINEL_GC_OBJ_BUDGET=fixed / SPINEL_GC_STR_BUDGET=fixed: hold that heap's budget at its floor instead of re-aiming it after every collection. Read once beside the other boot-time GC modes; see the comment there. */ @@ -67,7 +71,8 @@ int sp_gc_str_budget_mode = 2; /* That share: one part in this many. An ORDER, like the gate's coefficients. */ #define SP_STR_BUDGET_OBJ_DIV 8 /* What the last string retune put into the per-worker trigger for the object - old generation, so the stats line can report it and a test can read it. */ + old generation, so the stats line can report it and a test can read it. + Zero after a retune that put none in. */ static size_t sp_str_budget_obj_bytes = 0; /* SPINEL_GC_STR_MAJOR=fixed: hold the string old generation's gate at its floor instead of re-aiming it, adapting nothing. */ @@ -489,9 +494,12 @@ static void sp_gc_stats_emit(void) { sp_gc_ph_mk_globals, sp_gc_ph_mk_scan); } +/* Re-aim the object trigger from what a collection left, and take the + measurement both budgets' gates start from: the mark's share of that + collection. The reasoning is inline, beside each step. */ void sp_gc_retune_object(size_t before) { sp_gc_stats_report(); - if (sp_gc_stress_pin || sp_gc_obj_budget_fixed) { sp_gc_threshold = sp_gc_threshold_init; return; } + if (sp_gc_stress_pin) { sp_gc_threshold = sp_gc_threshold_init; return; } size_t live = sp_gc_bytes; /* The budget is what may be ALLOCATED before the next collection, and what pays for it is what that collection COSTS. A collection marks BOTH heaps, @@ -563,6 +571,12 @@ void sp_gc_retune_object(size_t before) { if (alpha > 1024) alpha = 1024; } } + /* What was measured, kept before the knobs below decide what the OBJECT + budget makes of it. `fixed` leaves here and not at the top for the same + reason: a budget held at its floor still has a collection to measure, and + the string budget's gate (sp_str_retune) reads the measurement. */ + sp_gc_mark_share1024 = alpha; + if (sp_gc_obj_budget_fixed) { sp_gc_threshold = sp_gc_threshold_init; return; } if (sp_gc_obj_budget_mode == 0) alpha = 0; else if (sp_gc_obj_budget_mode == 1) alpha = 1024; sp_gc_obj_alpha1024 = alpha; @@ -644,6 +658,9 @@ static void sp_str_retune(size_t before, size_t promoted) { serially, before the slab promotions are folded in when the workers sweep (so one cycle behind there), at the next barrier for the concurrent sweep. alpha is one cycle behind everywhere -- the object retune runs after this. + It is the share that retune MEASURED, not what SPINEL_GC_OBJ_BUDGET made + of it: `obj`, `walk` and `fixed` pin the object budget, and this budget + has its own knob for the same three ends. One part in SP_STR_BUDGET_OBJ_DIV, and not the whole of it, because the whole is mostly memory. Measured from 1 to 1/32: on the Array above and @@ -661,7 +678,7 @@ static void sp_str_retune(size_t before, size_t promoted) { size_t share = 0; if (sp_gc_str_budget_mode) { size_t obj = sp_gc_old_bytes; - size_t alpha = sp_gc_str_budget_mode == 1 ? 1024 : sp_gc_obj_alpha1024; + size_t alpha = sp_gc_str_budget_mode == 1 ? 1024 : sp_gc_mark_share1024; share = (obj / 1024) * alpha + ((obj % 1024) * alpha) / 1024; #ifdef SP_THREADS share /= (size_t)nw; /* per worker, as `after` is */ @@ -672,7 +689,7 @@ static void sp_str_retune(size_t before, size_t promoted) { keeps the collections it buys from piling up: the share goes into the other case only. That case is also the empty heap now -- no live string and no share reads as zero and takes the floor. */ - if (freed < before / 4) { sp_str_threshold = sp_gc_sat_mul(before, 2); } + if (freed < before / 4) { sp_str_threshold = sp_gc_sat_mul(before, 2); sp_str_budget_obj_bytes = 0; } else { sp_str_threshold = sp_gc_sat_mul(after + share, 2); if (sp_str_threshold < sp_str_threshold_init) sp_str_threshold = sp_str_threshold_init; diff --git a/test/gc_str_budget_kept.rb b/test/gc_str_budget_kept.rb new file mode 100644 index 0000000000..00f59dc9b4 --- /dev/null +++ b/test/gc_str_budget_kept.rb @@ -0,0 +1,50 @@ +# The `[gc]` line reports what the last string retune put into the trigger for +# the object old generation (`old obj in str trigger`). A sweep that reclaims +# under a quarter of the string heap puts none in -- it doubles the budget as +# it stands -- so the reading after one has to be zero, not the figure the last +# productive sweep left there. +# +# This program does both in turn. First the shape gc_str_budget_objects.rb +# has: records kept in one Array, every String thrown away, so the sweeps are +# productive and the budget carries the share. Then it keeps every String it +# makes, one a turn with no temporary beside it and each past the size the +# slab takes (2 KB), so a sweep finds the whole young list still live, reclaims +# nothing, and the budget doubles on its own. +# +# The gc-str-budget-test leg reads two arms of the same binary. The default has +# to collect less than half as often as SPINEL_GC_STR_BUDGET=str, which is the +# share at work in the first half; and its last `[gc]` line has to read zero, +# which is the second half taking it back out. It fails with the share never +# carried and with the reading left where the first half put it. +# +# Single-threaded on purpose, as the two siblings are. +class Rec + attr_reader :id, :qty + def initialize(id, qty) + @id = id + @qty = qty + end +end + +recs = [] +i = 0 +while i < 150000 + line = "#{i},name-#{i % 1000},#{i * 7}" + parts = line.split(",") + recs << Rec.new(parts[0].to_i, parts[2].to_i) + i += 1 +end +kept = [] +k = 0 +while k < 8000 + kept << ("ab" * 1500) + k += 1 +end +total = 0 +recs.each { |r| total = (total + r.qty - r.id) % 1000003 } +bytes = 0 +kept.each { |s| bytes += s.size } +puts recs.size +puts total +puts kept.size +puts bytes diff --git a/test/gc_str_budget_kept.rb.expected b/test/gc_str_budget_kept.rb.expected new file mode 100644 index 0000000000..f13c7e833d --- /dev/null +++ b/test/gc_str_budget_kept.rb.expected @@ -0,0 +1,4 @@ +150000 +347503 +8000 +24000000