[make-initrd] [PATCH v2] runtime: Freeze rootdelay countdown while paused

[Alexey Gladkov <gladkov.alexey@gmail.com>]

The rootdelay service tracks device discovery timeouts against an
absolute deadline timestamp. When waiting for blocking operations such
as filesystem checks, the deadline was left unchanged while boottime
continued to advance. When unpaused, the elapsed waiting time was
counted against the rootdelay allowance, often causing premature timeout
expiration.

Furthermore, the existing pause mechanism relied on a single global
directory flag, making it incapable of handling concurrent pauses from
parallel scripts (where one script unpausing would prematurely unpause
the system for other running tasks) and prone to indefinite stalls if a
paused process terminated unexpectedly.

Redesign rootdelay pausing around a shared inhibitor lock using flock:
- Multiple concurrent tasks acquire a shared lock (flock -s) on a pause
  lockfile, ensuring the system remains paused until the last holder
  releases its lock.
- If a process crashes or is killed, the kernel automatically closes its
  file descriptor and drops the shared lock, preventing stale pauses.
- Introduce run_rootdelay_paused to wrap commands under a shared lock,
  and update fsck 040-mountdev to use it.
- Keep rootdelay_pause and rootdelay_unpause backward-compatible by
  holding the shared lock on an open descriptor within the process.
- Upon release of the final shared lock, adjust the deadline forward by
  the total elapsed pause duration.
- Track timer resets occurring during a pause to prevent adding
  redundant delay on unpause.
- Unify boottime retrieval across rootdelay helpers and rootdelayd via a
  current_seconds helper.

Signed-off-by: Alexey Gladkov <gladkov.alexey@gmail.com>
---
 .../data/lib/uevent/handlers/040-mountdev     |   7 +-
 .../kickstart/data/etc/rc.d/init.d/kickstart  |   5 +-
 features/runtime/data/bin/initrd-sh-functions | 126 +++++++++++++-
 features/runtime/data/sbin/rootdelayd         |   9 +-
 testing/tests/ts0047-rootdelay-pause/expect   |  18 ++
 testing/tests/ts0047-rootdelay-pause/run      | 156 ++++++++++++++++++
 6 files changed, 302 insertions(+), 19 deletions(-)
 create mode 100644 testing/tests/ts0047-rootdelay-pause/expect
 create mode 100755 testing/tests/ts0047-rootdelay-pause/run

diff --git a/features/fsck/data/lib/uevent/handlers/040-mountdev b/features/fsck/data/lib/uevent/handlers/040-mountdev
index 78bfd165..84fa6503 100755
--- a/features/fsck/data/lib/uevent/handlers/040-mountdev
+++ b/features/fsck/data/lib/uevent/handlers/040-mountdev
@@ -26,14 +26,11 @@ handler()
 
 	local rc=0
 
-	rootdelay_pause
-
-	action "Checking filesystem [$MOUNTDEV]:" fsck -t noloop,noswap -T "$MOUNTDEV" -- ${FSCK_ARGS-} ||
+	run_rootdelay_paused \
+		action "Checking filesystem [$MOUNTDEV]:" fsck -t noloop,noswap -T "$MOUNTDEV" -- ${FSCK_ARGS-} ||
 		rc=$?
 
 	if [ "$rc" = 0 ] || [ "$rc" = 1 ]; then
-		rootdelay_unpause
-
 		majmin="$(get_majmin "$MOUNTDEV")" &&
 			echo 'done' | statefs_write "/devices/block/$majmin/fsck" ||:
 
diff --git a/features/kickstart/data/etc/rc.d/init.d/kickstart b/features/kickstart/data/etc/rc.d/init.d/kickstart
index 3ccf2203..0527c20b 100755
--- a/features/kickstart/data/etc/rc.d/init.d/kickstart
+++ b/features/kickstart/data/etc/rc.d/init.d/kickstart
@@ -25,18 +25,15 @@ if [ -s "$KSFILE" ]; then
 	. uevent-sh-functions
 	. initrd-sh-functions
 
-	rootdelay_pause
-
 	method=
 	{ read -r method < /etc/initrd/method; } >/dev/null 2>&1 ||:
 	echo kickstart > /etc/initrd/method
 
-	kickstart -v "$KSFILE"
+	run_rootdelay_paused kickstart -v "$KSFILE"
 
 	echo "$method" > /etc/initrd/method
 
 	rootdelay_reset_timer
-	rootdelay_unpause
 
 	rmdir -- "$uevent_confdb/queue/pause/udev"
 	rmdir -- "$uevent_confdb/queue/pause/md-raid-member"
diff --git a/features/runtime/data/bin/initrd-sh-functions b/features/runtime/data/bin/initrd-sh-functions
index a69eabee..1266ea29 100644
--- a/features/runtime/data/bin/initrd-sh-functions
+++ b/features/runtime/data/bin/initrd-sh-functions
@@ -126,22 +126,126 @@ write_atomic()
 	fi
 }
 
-_rootdelay_pause=/.initrd/rootdelay/pause
-_rootdelay_timestamp=/.initrd/rootdelay/deadline
+current_seconds()
+{
+	local __v=
+	__v="$("${TIMESTAMP:-timestamp}" boottime raw 2>/dev/null)" ||:
+	__v="${__v%.*}"
+	eval "$1=\"\$__v\""
+}
+
+rootdelay_pause_lock="${rootdelay_pause_lock:-/.initrd/rootdelay/pause.lock}"
+_rootdelay_pause_time="${_rootdelay_pause_time:-/.initrd/rootdelay/pause.time}"
+_rootdelay_reset="${_rootdelay_reset:-/.initrd/rootdelay/reset}"
+_rootdelay_timestamp="${_rootdelay_timestamp:-/.initrd/rootdelay/deadline}"
+
+_rootdelay_finalize_pause()
+{
+	local chk_fd pause_start reset_time now paused_duration deadline
+
+	[ -e "$rootdelay_pause_lock" ] || return 0
+
+	exec {chk_fd}<"$rootdelay_pause_lock"
+	if ! flock -n -x "$chk_fd"; then
+		exec {chk_fd}<&-
+		return 1
+	fi
+
+	if [ -f "$_rootdelay_pause_time" ]; then
+		readline pause_start "$_rootdelay_pause_time"
+
+		if [ -f "$_rootdelay_reset" ]; then
+			readline reset_time "$_rootdelay_reset"
+			rm -f "$_rootdelay_reset"
+
+			if [ -n "$reset_time" ] && [ -n "$pause_start" ] && [ "$reset_time" -gt "$pause_start" ]; then
+				pause_start="$reset_time"
+			fi
+		fi
+
+		if [ -n "$pause_start" ] && [ -f "$_rootdelay_timestamp" ]; then
+			current_seconds now
+			paused_duration=$(( $now - $pause_start ))
+
+			if [ "$paused_duration" -gt 0 ]; then
+				readline deadline "$_rootdelay_timestamp"
+
+				if [ -n "$deadline" ] && [ "$deadline" -gt 0 ]; then
+					rootdelay_set_deadline "$(( $deadline + $paused_duration ))"
+				fi
+			fi
+		fi
+
+		rm -f "$_rootdelay_pause_time"
+	fi
+
+	exec {chk_fd}<&-
+}
+
+run_rootdelay_paused()
+{
+	local fd rc=0 now
+
+	mkdir -p -- "${rootdelay_pause_lock%/*}"
+	touch "$rootdelay_pause_lock"
+
+	exec {fd}<"$rootdelay_pause_lock"
+	flock -s "$fd"
+
+	if [ ! -f "$_rootdelay_pause_time" ]; then
+		current_seconds now
+		write_atomic "$_rootdelay_pause_time" printf '%s\n' "$now"
+	fi
+
+	"$@" {fd}<&- || rc=$?
+	exec {fd}<&-
+
+	_rootdelay_finalize_pause
+
+	return $rc
+}
 
 rootdelay_pause()
 {
-	mkdir -p "$_rootdelay_pause"
+	local now
+
+	[ -z "${rootdelay_pause_fd-}" ] || return 0
+
+	mkdir -p -- "${rootdelay_pause_lock%/*}"
+	touch "$rootdelay_pause_lock"
+
+	exec {rootdelay_pause_fd}<"$rootdelay_pause_lock"
+	flock -s "$rootdelay_pause_fd"
+
+	if [ ! -f "$_rootdelay_pause_time" ]; then
+		current_seconds now
+		write_atomic "$_rootdelay_pause_time" printf '%s\n' "$now"
+	fi
 }
 
 rootdelay_unpause()
 {
-	rm -rf "$_rootdelay_pause"
+	[ -n "${rootdelay_pause_fd-}" ] || return 0
+
+	exec {rootdelay_pause_fd}<&-
+	rootdelay_pause_fd=
+
+	_rootdelay_finalize_pause
 }
 
 rootdelay_paused()
 {
-	[ -d "$_rootdelay_pause" ]
+	local fd rc=0
+
+	[ -z "${rootdelay_pause_fd-}" ] || return 0
+	[ -e "$rootdelay_pause_lock"  ] || return 1
+
+	exec {fd}<"$rootdelay_pause_lock"
+
+	! flock -n -x "$fd" || rc=1
+	exec {fd}<&-
+
+	return $rc
 }
 
 rootdelay_set_deadline()
@@ -153,11 +257,17 @@ rootdelay_set_deadline()
 rootdelay_reset_timer()
 {
 	local now="${1-}"
+
 	if [ -z "$now" ]; then
-		now="$(timestamp boottime raw)"
-		now="${now%.*}"
+		current_seconds now
+	fi
+
+	rootdelay_set_deadline "$(( $now + ${ROOTDELAY:-0} ))"
+
+	if rootdelay_paused; then
+		mkdir -p -- "${_rootdelay_reset%/*}"
+		write_atomic "$_rootdelay_reset" printf '%s\n' "$now"
 	fi
-	rootdelay_set_deadline "$(( $now + $ROOTDELAY ))"
 }
 
 initrd_features=/.initrd/features
diff --git a/features/runtime/data/sbin/rootdelayd b/features/runtime/data/sbin/rootdelayd
index 5fa21f07..cdd8fcf8 100755
--- a/features/runtime/data/sbin/rootdelayd
+++ b/features/runtime/data/sbin/rootdelayd
@@ -47,6 +47,11 @@ while :; do
 		continue
 	fi
 
+	if [ -f "$_rootdelay_pause_time" ]; then
+		_rootdelay_finalize_pause || sleep 0.1
+		continue
+	fi
+
 	if [ -z "${__initrd_initenv-}" ] && [ -f /.initrd/initenv ]; then
 		. /.initrd/initenv
 
@@ -71,9 +76,9 @@ while :; do
 		continue
 	fi
 
+	current_seconds now
+
 	first_iter=
-	now="$(timestamp boottime raw)"
-	now="${now%.*}"
 	consmsg="/.initrd/rootdelay/message"
 
 	if [ ! -f "$_rootdelay_timestamp" ]; then
diff --git a/testing/tests/ts0047-rootdelay-pause/expect b/testing/tests/ts0047-rootdelay-pause/expect
new file mode 100644
index 00000000..7788a3a8
--- /dev/null
+++ b/testing/tests/ts0047-rootdelay-pause/expect
@@ -0,0 +1,18 @@
+initial unpause ok
+deadline after init: 160
+paused at 120
+unpaused at 150
+deadline after unpause: 190
+remaining delay: 40
+pause before deadline ok
+deadline reset: 360
+deadline after reset during pause: 410
+remaining delay: 60
+deadline 0 preserved: 0
+run_rootdelay_paused: inside paused ok
+run_rootdelay_paused: outside unpaused ok
+B unpause: still paused because A holds lock ok
+A unpause: unpaused ok
+concurrent deadline: 800
+concurrent remaining delay: 50
+rc=0
diff --git a/testing/tests/ts0047-rootdelay-pause/run b/testing/tests/ts0047-rootdelay-pause/run
new file mode 100755
index 00000000..6d32632a
--- /dev/null
+++ b/testing/tests/ts0047-rootdelay-pause/run
@@ -0,0 +1,156 @@
+#!/bin/bash -efu
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+cwd="${0%/*}"
+root="$cwd/root"
+
+rm -rf -- "$root"
+mkdir -p -- "$root/bin" "$root/rootdelay"
+
+mock_time="$root/time"
+echo "100" > "$mock_time"
+
+cat > "$root/bin/timestamp" <<EOF
+#!/bin/bash -efu
+[ "\$1" = "boottime" ] && [ "\$2" = "raw" ] || exit 1
+cat "$mock_time"
+EOF
+chmod +x "$root/bin/timestamp"
+
+export PATH="$PWD/.build/dest/usr/bin:$PATH"
+export TIMESTAMP="$root/bin/timestamp"
+export rootdelay_pause_lock="$root/rootdelay/pause.lock"
+export _rootdelay_pause_time="$root/rootdelay/pause.time"
+export _rootdelay_reset="$root/rootdelay/reset"
+export _rootdelay_timestamp="$root/rootdelay/deadline"
+export ROOTDELAY=60
+
+. features/runtime/data/bin/initrd-sh-functions
+
+# 1. Initially not paused, unpause is a no-op
+if rootdelay_paused; then
+	echo "unexpectedly paused initially"
+fi
+rootdelay_unpause
+echo "initial unpause ok"
+
+# 2. Reset timer at t=100 with ROOTDELAY=60 -> deadline=160
+rootdelay_reset_timer
+readline deadline "$_rootdelay_timestamp"
+echo "deadline after init: $deadline"
+
+# 3. Pause at t=120 (remaining: 40)
+echo "120" > "$mock_time"
+rootdelay_pause
+if rootdelay_paused; then
+	echo "paused at 120"
+fi
+
+# 4. Redundant pause at t=130 should not overwrite pause start
+echo "130" > "$mock_time"
+rootdelay_pause
+
+# 5. Unpause at t=150 (30 seconds paused). Deadline should be 160 + 30 = 190.
+echo "150" > "$mock_time"
+rootdelay_unpause
+if ! rootdelay_paused; then
+	echo "unpaused at 150"
+fi
+readline deadline "$_rootdelay_timestamp"
+echo "deadline after unpause: $deadline"
+echo "remaining delay: $(( deadline - 150 ))"
+
+# 6. Pause before deadline exists
+rm -f -- "$_rootdelay_timestamp"
+echo "200" > "$mock_time"
+rootdelay_pause
+echo "220" > "$mock_time"
+rootdelay_unpause
+if [ ! -f "$_rootdelay_timestamp" ]; then
+	echo "pause before deadline ok"
+fi
+
+# 7. Timer reset during pause
+echo "300" > "$mock_time"
+rootdelay_reset_timer
+readline deadline "$_rootdelay_timestamp"
+echo "deadline reset: $deadline"
+echo "320" > "$mock_time"
+rootdelay_pause
+echo "340" > "$mock_time"
+rootdelay_reset_timer
+echo "350" > "$mock_time"
+rootdelay_unpause
+readline deadline "$_rootdelay_timestamp"
+echo "deadline after reset during pause: $deadline"
+echo "remaining delay: $(( deadline - 350 ))"
+
+# 8. Deadline 0 is preserved
+rootdelay_set_deadline 0
+echo "500" > "$mock_time"
+rootdelay_pause
+echo "520" > "$mock_time"
+rootdelay_unpause
+readline deadline "$_rootdelay_timestamp"
+echo "deadline 0 preserved: $deadline"
+
+# 9. run_rootdelay_paused wrapper
+echo "600" > "$mock_time"
+rootdelay_reset_timer
+run_rootdelay_paused bash -c '
+	. features/runtime/data/bin/initrd-sh-functions
+	rootdelay_paused && echo "run_rootdelay_paused: inside paused ok"
+'
+if ! rootdelay_paused; then
+	echo "run_rootdelay_paused: outside unpaused ok"
+fi
+
+# 10. Concurrent / overlapping pauses in background subshells
+echo "700" > "$mock_time"
+rootdelay_reset_timer
+
+fifo1="$root/fifo1"
+fifo2="$root/fifo2"
+mkfifo "$fifo1" "$fifo2"
+
+# Subshell 1 (Process A)
+bash -c '
+	. features/runtime/data/bin/initrd-sh-functions
+	rootdelay_pause
+	echo "A_paused" > "'"$fifo1"'"
+	read -r _ < "'"$fifo2"'"
+	rootdelay_unpause
+' &
+pid_a=$!
+
+# Wait for A to pause at 710
+echo "710" > "$mock_time"
+read -r _ < "$fifo1"
+
+# Subshell 2 (Process B) pauses at 720
+echo "720" > "$mock_time"
+bash -c '
+	. features/runtime/data/bin/initrd-sh-functions
+	rootdelay_pause
+	# B unpauses at 730
+	echo "730" > "'"$mock_time"'"
+	rootdelay_unpause
+	if rootdelay_paused; then
+		echo "B unpause: still paused because A holds lock ok"
+	fi
+'
+
+# Now unblock A at 750
+echo "750" > "$mock_time"
+echo "go" > "$fifo2"
+wait "$pid_a"
+
+if ! rootdelay_paused; then
+	echo "A unpause: unpaused ok"
+fi
+readline deadline "$_rootdelay_timestamp"
+# Deadline was 700 + 60 = 760. Total pause was from 710 to 750 = 40s. New deadline: 760 + 40 = 800.
+echo "concurrent deadline: $deadline"
+echo "concurrent remaining delay: $(( deadline - 750 ))"
+
+rm -rf -- "$root"
-- 
2.56.0