[2026.10.7] lock: steal a free locker from another stripe before failing (recd015)

src/lock/lock_id.c

@@ -550,8 +550,50 @@ __lock_getlocker_int(lt, locker, create, retp)
 			F_CLR(&lt->reginfo, REGION_TRACKED);
 			LOCK_REGION_UNLOCK(lt->env);
 			LOCK_LOCKERS(env, region);
-			if (nlockers == 0)
-				return (__lock_nomem(env, "locker entries"));
+			if (nlockers == 0) {
+				/*
+				 * The region cannot grow st_lockers any further
+				 * (it has reached st_maxlockers = lk_max_lockers),
+				 * yet this stripe's free list is empty.  Under P12's
+				 * per-stripe free lists the global cap can be
+				 * unreached while THIS stripe is dry and OTHER
+				 * stripes still hold idle free lockers -- the id hash
+				 * can pile creates onto a few stripes (recovery's
+				 * backward pass makes one locker per prepared txn,
+				 * recd015).  Before failing, steal a free locker from
+				 * another stripe onto this one.
+				 *
+				 * Safe because this path holds LOCK_LOCKERS (all
+				 * stripes, exclusive), so every stripe's free_lockers
+				 * is ours to touch.  The stolen locker's bucket (and
+				 * therefore its stripe) is recomputed from its id
+				 * below, which is THIS stripe, so it is inserted on
+				 * this stripe's bucket chain and ulinks list and, when
+				 * freed, returns to this stripe's free list
+				 * (__lock_freelocker_int derives the stripe the same
+				 * way) -- a one-way migration, never a cross-stripe
+				 * leak.  The retained mtx_locker travels with it; a
+				 * logical-lock mutex carries no stripe identity.
+				 */
+				for (s = 0; s < LOCK_LOCKER_STRIPES; s++) {
+					if (s == LOCK_LOCKER_ALLOC_STRIPE(region,
+					    LOCK_LOCKER_STRIPE(indx)))
+						continue;
+					if ((sh_locker = SH_TAILQ_FIRST(&region->
+					    locker_stripe[s].free_lockers,
+					    __db_locker)) == NULL)
+						continue;
+					SH_TAILQ_REMOVE(&region->
+					    locker_stripe[s].free_lockers,
+					    sh_locker, links, __db_locker);
+					SH_TAILQ_INSERT_HEAD(&ls->free_lockers,
+					    sh_locker, links, __db_locker);
+					break;
+				}
+				if (s == LOCK_LOCKER_STRIPES)
+					return (__lock_nomem(env,
+					    "locker entries"));
+			}
 			/*
 			 * Spread the new lockers across ALL stripes' free lists,
 			 * round-robin starting at this one.

test/MANIFEST

@@ -466,6 +466,7 @@ db	run_recd_compact
 db	run_recd_handlers
 db	run_upgrade
 db	run_lock_priority_nullderef
+db	run_lock_stripe_exhaustion
 db	run_txn_remevent_leak
 db	run_compact_subdb_remove
 db	run_log_emfile_nopanic

test/db/lock_stripe_exhaustion.c

@@ -0,0 +1,180 @@
+/*-
+ * Copyright (c) 2026 libdb contributors.  All rights reserved.
+ *
+ * SPDX-License-Identifier: Sleepycat
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+/*-
+ * Regression test for the P12 per-stripe locker-allocation exhaustion
+ * (recd015 "out of available locker entries" on a backward recovery pass).
+ *
+ * P12 (2d8397dfe) gave each of the 64 locker stripes its own free list and
+ * refilled an empty stripe with a batch spread round-robin across ALL stripes,
+ * capped by st_maxlockers - st_lockers (= lk_max_lockers - in use).  Once
+ * st_lockers reaches st_maxlockers the cap is zero, so a create that lands on
+ * a stripe whose free list happens to be empty fails with
+ * "Lock table is out of available locker entries" -- even though OTHER stripes
+ * still hold idle free lockers and the number of lockers actually in use is
+ * well below lk_max_lockers.  Recovery's backward pass creates one locker per
+ * prepared txn; the id hash concentrates those creates on a few stripes, so it
+ * hit the dry-stripe case and panicked the recovery.
+ *
+ * This drives the bug through the public API only (lock_id / lock_id_free):
+ *
+ *   1. lk_max_lockers = MAXL.  The locker hash table size is a prime >= MAXL
+ *      (__db_tablesize), so for every id we allocate (all < MAXL < table size)
+ *      the bucket is id itself and the stripe is id & 63.  We therefore know
+ *      each id's stripe exactly.
+ *   2. Allocate ids with lock_id() until the pool is at its limit, recording
+ *      each id's stripe.  Pick a TARGET stripe that received several ids.
+ *   3. Free every allocated id that is NOT on the target stripe.  Freed lockers
+ *      return to their own stripe's free list, so now every stripe except the
+ *      target has free lockers and the target stripe's free list is empty,
+ *      while st_lockers is still at st_maxlockers (nothing was returned to the
+ *      region).
+ *   4. Allocate ids until one lands on the target stripe.  That create finds an
+ *      empty free list and a full region: before the fix it fails with ENOMEM
+ *      (the recd015 panic); with the steal-from-another-stripe fix it succeeds.
+ *
+ * Before the fix the test fails at step 4; with the fix it passes.
+ */
+#include <sys/types.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <db.h>
+
+#define	MAXL		320		/* lk_max_lockers */
+#define	STRIPES		64		/* LOCK_LOCKER_STRIPES (dbinc/lock.h) */
+
+static int failures = 0;
+static int checks = 0;
+
+#define CHECK(cond, ...) do {						\
+	checks++;							\
+	if (!(cond)) {							\
+		failures++;						\
+		printf("  FAIL: ");					\
+		printf(__VA_ARGS__);					\
+		printf("\n");						\
+	}								\
+} while (0)
+
+int
+main(int argc, char *argv[])
+{
+	DB_ENV *dbenv;
+	u_int32_t id, ids[MAXL + 1], target, newid;
+	const char *home = "LOCK_STRIPE_TESTDIR";
+	int i, n, ret, per_stripe[STRIPES], freed, got_target;
+
+	(void)argc; (void)argv;
+
+	if ((ret = db_env_create(&dbenv, 0)) != 0) {
+		fprintf(stderr, "db_env_create: %s\n", db_strerror(ret));
+		return (EXIT_FAILURE);
+	}
+	dbenv->set_errpfx(dbenv, "lock_stripe_exhaustion");
+	(void)dbenv->set_lk_max_lockers(dbenv, MAXL);
+	if ((ret = dbenv->open(dbenv, home, DB_CREATE | DB_INIT_LOCK |
+	    DB_INIT_MPOOL, 0600)) != 0) {
+		fprintf(stderr, "env open %s: %s\n", home, db_strerror(ret));
+		return (EXIT_FAILURE);
+	}
+
+	/*
+	 * 2: allocate until the pool is exhausted, recording each id's stripe.
+	 * The library seeds its own initial free list and may hold a handful of
+	 * internal lockers, so we simply allocate until lock_id returns ENOMEM
+	 * rather than assuming an exact count.
+	 */
+	memset(per_stripe, 0, sizeof(per_stripe));
+	for (n = 0; n < MAXL; n++) {
+		if ((ret = dbenv->lock_id(dbenv, &id)) != 0) {
+			if (ret == ENOMEM)
+				break;
+			fprintf(stderr, "lock_id #%d: %s\n", n,
+			    db_strerror(ret));
+			return (EXIT_FAILURE);
+		}
+		ids[n] = id;
+		per_stripe[id & (STRIPES - 1)]++;
+	}
+	printf("  allocated %d lockers before the pool filled\n", n);
+	CHECK(n > STRIPES,
+	    "allocated only %d lockers; expected more than %d so several share "
+	    "a stripe", n, STRIPES);
+
+	/* Pick a target stripe that several ids landed on. */
+	target = 0;
+	for (i = 0; i < STRIPES; i++)
+		if (per_stripe[i] > per_stripe[target])
+			target = i;
+	printf("  target stripe %u holds %d allocated lockers\n",
+	    target, per_stripe[target]);
+	CHECK(per_stripe[target] >= 1,
+	    "no allocated locker landed on the target stripe");
+
+	/*
+	 * 3: free every allocated id NOT on the target stripe.  Those lockers
+	 * go back to their own stripes' free lists; the target stripe keeps its
+	 * lockers in use and its free list empty.  st_lockers stays at the max.
+	 */
+	freed = 0;
+	for (i = 0; i < n; i++)
+		if ((ids[i] & (STRIPES - 1)) != target) {
+			if ((ret = dbenv->lock_id_free(dbenv, ids[i])) != 0) {
+				fprintf(stderr, "lock_id_free %u: %s\n",
+				    ids[i], db_strerror(ret));
+				return (EXIT_FAILURE);
+			}
+			freed++;
+		}
+	printf("  freed %d lockers on other stripes; target stripe stays dry\n",
+	    freed);
+	CHECK(freed > 0, "freed nothing; cannot create the dry-stripe case");
+
+	/*
+	 * 4: allocate until one id lands on the dry target stripe.  That is the
+	 * create that panicked recd015.  With the fix it steals a free locker
+	 * from one of the stripes we emptied in step 3 and succeeds.
+	 */
+	got_target = 0;
+	newid = 0;
+	for (i = 0; i < 4 * STRIPES; i++) {
+		ret = dbenv->lock_id(dbenv, &id);
+		if (ret == ENOMEM) {
+			printf("  lock_id returned ENOMEM "
+			    "(out of locker entries) at try %d\n", i);
+			break;
+		}
+		if (ret != 0) {
+			fprintf(stderr, "lock_id (step 4) #%d: %s\n", i,
+			    db_strerror(ret));
+			return (EXIT_FAILURE);
+		}
+		if ((id & (STRIPES - 1)) == target) {
+			got_target = 1;
+			newid = id;
+			break;
+		}
+		/* Free non-target ids again so the pool stays full. */
+		(void)dbenv->lock_id_free(dbenv, id);
+	}
+	CHECK(got_target == 1,
+	    "could not allocate a locker on the dry target stripe %u: "
+	    "the create failed (recd015 regression)", target);
+	if (got_target)
+		printf("  allocated locker id %u on the dry target stripe %u\n",
+		    newid, target);
+
+	(void)dbenv->close(dbenv, 0);
+
+	printf("lock_stripe_exhaustion: %d checks, %d failures\n",
+	    checks, failures);
+	printf("lock_stripe_exhaustion: %s\n", failures == 0 ? "PASS" : "FAIL");
+	return (failures == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
+}

test/db/run_lock_stripe_exhaustion.sh

@@ -0,0 +1,79 @@
+#!/bin/sh -
+#
+# $Id$
+#
+# run_lock_stripe_exhaustion.sh --
+#	Build and run lock_stripe_exhaustion.c, the regression test for the P12
+#	per-stripe locker-allocation exhaustion that made a TCL recd015 backward
+#	recovery pass fail with "BDB2055 Lock table is out of available locker
+#	entries" (first broken by 2d8397dfe, shipped in v2026.10.5).
+#
+#	P12 gave each of the 64 locker stripes its own free list and refills an
+#	empty stripe with a batch capped by st_maxlockers - st_lockers.  Once
+#	st_lockers reaches st_maxlockers (= lk_max_lockers) the cap is zero, so a
+#	create landing on a stripe whose free list is empty failed even though
+#	other stripes still held idle free lockers.  __lock_getlocker_int now
+#	steals a free locker from another stripe before failing.
+#
+# Exits non-zero on failure or hang.
+
+set -e
+
+BUILD=${BUILD:-.}
+_HERE=$(cd "$(dirname "$0")" && pwd)
+SRC=${SRC:-$_HERE/lock_stripe_exhaustion.c}
+HOME_DIR=${HOME_DIR:-LOCK_STRIPE_TESTDIR}
+TIMEOUT=${TIMEOUT:-180}
+
+# Prefer the STATIC library (see run_lock_priority_nullderef.sh for why).
+LIB=""
+LIBRPATH=""
+for cand in "$BUILD"/libdb.a "$BUILD"/.libs/libdb-5.3.a "$BUILD"/.libs/libdb-*.a; do
+	if [ -f "$cand" ]; then LIB="$cand"; break; fi
+done
+if [ -z "$LIB" ]; then
+	for cand in "$BUILD"/.libs/libdb-5.3.so "$BUILD"/.libs/libdb-5.3.dylib \
+	    "$BUILD"/.libs/libdb-*.so "$BUILD"/.libs/libdb-*.dylib; do
+		if [ -f "$cand" ]; then LIB="$cand"; break; fi
+	done
+	[ -n "$LIB" ] && LIBRPATH="-Wl,-rpath,$(cd "$BUILD/.libs" && pwd)"
+fi
+[ -n "$LIB" ] || { echo "FAIL: no libdb library (static or shared) found under $BUILD"; exit 1; }
+
+# A static libdb needs its transitive deps named explicitly.
+EXTRALIBS=""
+for l in $(pkg-config --libs liburing 2>/dev/null); do EXTRALIBS="$EXTRALIBS $l"; done
+
+echo "Compiling lock_stripe_exhaustion against $LIB"
+"${CC:-cc}" -g -O1 ${CFLAGS:-} -I"$BUILD" "$SRC" "$LIB" \
+    -lpthread $LIBRPATH $EXTRALIBS \
+    -o "$BUILD/lock_stripe_exhaustion"
+
+# Start from an EMPTY home, regions included (see the priority runner note).
+[ -d "$HOME_DIR" ] && find "$HOME_DIR" -mindepth 1 -delete 2>/dev/null
+mkdir -p "$HOME_DIR"
+
+if command -v timeout >/dev/null 2>&1; then
+	TIMEOUT_CMD="timeout"
+elif command -v gtimeout >/dev/null 2>&1; then
+	TIMEOUT_CMD="gtimeout"
+else
+	TIMEOUT_CMD=""
+fi
+run_with_timeout() {
+	if [ -n "$TIMEOUT_CMD" ]; then
+		"$TIMEOUT_CMD" "$@"
+	else
+		shift	# drop the seconds argument
+		"$@"
+	fi
+}
+echo "Running lock_stripe_exhaustion (timeout ${TIMEOUT}s)"
+if run_with_timeout "$TIMEOUT" "$BUILD/lock_stripe_exhaustion"; then
+	echo "run_lock_stripe_exhaustion.sh: PASS"
+	exit 0
+else
+	rc=$?
+	echo "run_lock_stripe_exhaustion.sh: FAIL (rc=$rc)"
+	exit $rc
+fi