Merge branch into tip/master: 'core/core'

 # New commits in core/core:
    908c121 ("Revert "timekeeping: Use time_after() in timekeeping_check_update()"")
    42db2c2 ("timekeeping: Use time_after() in timekeeping_check_update()")
    ed4fb6d ("hrtimer: Use and report correct timerslack values for realtime tasks")
    aef6987 ("sched/eevdf: Propagate min_slice up the cgroup hierarchy")
    857b158 ("sched/eevdf: Use sched_attr::sched_runtime to set request/slice suggestion")
    85e511d ("sched/eevdf: Allow shorter slices to wakeup-preempt")
    82e9d04 ("sched/fair: Avoid re-setting virtual deadline on 'migrations'")
    fc1892b ("sched/eevdf: Fixup PELT vs DELAYED_DEQUEUE")
    54a58a7 ("sched/fair: Implement DELAY_ZERO")
    152e11f ("sched/fair: Implement delayed dequeue")
    e1459a5 ("sched: Teach dequeue_task() about special task states")
    a1c4466 ("sched,freezer: Mark TASK_FROZEN special")
    781773e ("sched/fair: Implement ENQUEUE_DELAYED")
    f12e148 ("sched/fair: Prepare pick_next_task() for delayed dequeue")
    2e0199d ("sched/fair: Prepare exit/cleanup paths for delayed_dequeue")
    e28b5f8 ("sched/fair: Assert {set_next,put_prev}_entity() are properly balanced")
    dfa0a57 ("sched/uclamg: Handle delayed dequeue")
    abc158c ("sched: Prepare generic code for delayed dequeue")
    e890106 ("sched: Split DEQUEUE_SLEEP from deactivate_task()")
    fab4a80 ("sched/fair: Re-organize dequeue_task_fair()")
    863ccdb ("sched: Allow sched_class::dequeue_task() to fail")
    3b3dd89 ("sched/fair: Unify pick_{,next_}_task_fair()")
    c97f54f ("sched/fair: Cleanup pick_task_fair()'s curr")
    8e2e13a ("sched/fair: Cleanup pick_task_fair() vs throttle")
    949090e ("sched/eevdf: Remove min_vruntime_copy")
    f25b7b3 ("sched/eevdf: Add feature comments")
    330dd6d ("hrtimer: Annotate hrtimer_cpu_base_.*_expiry() for sparse.")
    38cd4ce ("timers: Add sparse annotation for timer_sync_wait_running().")
    ae04f69 ("sched/rt: Rename realtime_{prio, task}() to rt_or_dl_{prio, task}()")
    b166af3 ("sched/rt, dl: Convert functions to return bool")
    130fd05 ("sched/rt: Clean up usage of rt_task()")
    4ae0c2b ("sched/debug: Fix fair_server_period_max value")
    924e290 ("sched/fair: Make balance_fair() test sched_fair_runnable() instead of rq->nr_running")
    7f8af7b ("signal: Replace BUG_ON()s")
    a2b80ce ("signal: Remove task argument from dequeue_signal()")
    566e2d8 ("posix-timers: Consolidate signal queueing")
    24aea4c ("posix-cpu-timers: Make k_itimer::it_active consistent")
    20f1338 ("posix-timers: Consolidate timer setup")
    52dea0a ("posix-timers: Convert timer list to hlist")
    aca1dc0 ("posix-timers: Clear overrun in common_timer_set()")
    bfa408f ("posix-timers: Retrieve interval in common timer_settime() code")
    c20b99e ("posix-cpu-timers: Simplify posix_cpu_timer_set()")
    286bfac ("posix-cpu-timers: Remove incorrect comment in posix_cpu_timer_set()")
    c444626 ("posix-cpu-timers: Use @now instead of @Val for clarity")
    bd29d77 ("posix-cpu-timers: Do not arm SIGEV_NONE timers")
    d471ff3 ("posix-cpu-timers: Replace old expiry retrieval in posix_cpu_timer_set()")
    5f9d4a1 ("posix-cpu-timers: Handle SIGEV_NONE timers correctly in timer_set()")
    d786b8b ("posix-cpu-timers: Handle SIGEV_NONE timers correctly in timer_get()")
    1c50284 ("posix-cpu-timers: Handle interval timers correctly in timer_get()")
    b3e866b ("posix-cpu-timers: Save interval only for armed timers")
    d859704 ("posix-cpu-timers: Split up posix_cpu_timer_get()")
    73339b8 ("selftests/timers/posix-timers: Validate overrun after unblock")
    f924f86 ("selftests/timers/posix-timers: Validate timer_gettime()")
    2c2b561 ("selftests/timers/posix-timers: Validate SIGEV_NONE")
    e65bb03 ("selftests/timers/posix_timers: Validate signal rules")
    45c4225 ("selftests/timers/posix_timers: Add SIG_IGN test")
    0af02a8 ("selftests/timers/posix_timers: Simplify error handling")
    cea5a34 ("sched/fair: Cleanup fair_server")
    5f6bd38 ("sched/rt: Remove default bandwidth control")
    c8a8539 ("sched/core: Fix picking of tasks for core scheduling with DL server")
    4b26cfd ("sched/core: Fix priority checking for DL server picks")
    d741f29 ("sched/fair: Fair server interface")
    a110a81 ("sched/deadline: Deferrable dl server")
    557a6bf ("sched/fair: Add trivial fair server")
    a741b82 ("sched/core: Clear prev->dl_server in CFS pick fast path")
    c245910 ("sched/core: Add clearing of ->dl_server in put_prev_task_balance()")
    f23c042 ("sched/deadline: Comment sched_dl_entity::dl_server variable")
    faa42d2 ("sched/fair: Make SCHED_IDLE entity be preempted in strict hierarchy")
    a58501f ("sched: remove HZ_BW feature hedge")
    2c2d962 ("sched/fair: Remove cfs_rq::nr_spread_over and cfs_rq::exec_clock")
    0ec8d5a ("sched/core: Add WARN_ON_ONCE() to check overflow for migrate_disable()")
    c40dd90 ("sched: Initialize the vruntime of a new task when it is first enqueued")

Signed-off-by: Ingo Molnar <[email protected]>

fs/bcachefs/six.c

@@ -335,7 +335,7 @@ static inline bool six_owner_running(struct six_lock *lock)
 	 */
 	rcu_read_lock();
 	struct task_struct *owner = READ_ONCE(lock->owner);
-	bool ret = owner ? owner_on_cpu(owner) : !rt_task(current);
+	bool ret = owner ? owner_on_cpu(owner) : !rt_or_dl_task(current);
 	rcu_read_unlock();
 
 	return ret;

fs/proc/base.c

@@ -2456,13 +2456,13 @@ static void *timers_start(struct seq_file *m, loff_t *pos)
 	if (!tp->sighand)
 		return ERR_PTR(-ESRCH);
 
-	return seq_list_start(&tp->task->signal->posix_timers, *pos);
+	return seq_hlist_start(&tp->task->signal->posix_timers, *pos);
 }
 
 static void *timers_next(struct seq_file *m, void *v, loff_t *pos)
 {
 	struct timers_private *tp = m->private;
-	return seq_list_next(v, &tp->task->signal->posix_timers, pos);
+	return seq_hlist_next(v, &tp->task->signal->posix_timers, pos);
 }
 
 static void timers_stop(struct seq_file *m, void *v)
@@ -2491,7 +2491,7 @@ static int show_timer(struct seq_file *m, void *v)
 		[SIGEV_THREAD] = "thread",
 	};
 
-	timer = list_entry((struct list_head *)v, struct k_itimer, list);
+	timer = hlist_entry((struct hlist_node *)v, struct k_itimer, list);
 	notify = timer->it_sigev_notify;
 
 	seq_printf(m, "ID: %d\n", timer->it_id);
@@ -2569,10 +2569,11 @@ static ssize_t timerslack_ns_write(struct file *file, const char __user *buf,
 	}
 
 	task_lock(p);
-	if (slack_ns == 0)
-		p->timer_slack_ns = p->default_timer_slack_ns;
-	else
-		p->timer_slack_ns = slack_ns;
+	if (rt_or_dl_task_policy(p))
+		slack_ns = 0;
+	else if (slack_ns == 0)
+		slack_ns = p->default_timer_slack_ns;
+	p->timer_slack_ns = slack_ns;
 	task_unlock(p);
 
 out:

fs/select.c

@@ -77,19 +77,16 @@ u64 select_estimate_accuracy(struct timespec64 *tv)
 {
 	u64 ret;
 	struct timespec64 now;
+	u64 slack = current->timer_slack_ns;
 
-	/*
-	 * Realtime tasks get a slack of 0 for obvious reasons.
-	 */
-
-	if (rt_task(current))
+	if (slack == 0)
 		return 0;
 
 	ktime_get_ts64(&now);
 	now = timespec64_sub(*tv, now);
 	ret = __estimate_accuracy(&now);
-	if (ret < current->timer_slack_ns)
-		return current->timer_slack_ns;
+	if (ret < slack)
+		return slack;
 	return ret;
 }
 

fs/signalfd.c

@@ -159,7 +159,7 @@ static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info
 	DECLARE_WAITQUEUE(wait, current);
 
 	spin_lock_irq(&current->sighand->siglock);
-	ret = dequeue_signal(current, &ctx->sigmask, info, &type);
+	ret = dequeue_signal(&ctx->sigmask, info, &type);
 	switch (ret) {
 	case 0:
 		if (!nonblock)
@@ -174,7 +174,7 @@ static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info
 	add_wait_queue(&current->sighand->signalfd_wqh, &wait);
 	for (;;) {
 		set_current_state(TASK_INTERRUPTIBLE);
-		ret = dequeue_signal(current, &ctx->sigmask, info, &type);
+		ret = dequeue_signal(&ctx->sigmask, info, &type);
 		if (ret != 0)
 			break;
 		if (signal_pending(current)) {

include/linux/ioprio.h

@@ -40,7 +40,7 @@ static inline int task_nice_ioclass(struct task_struct *task)
 {
 	if (task->policy == SCHED_IDLE)
 		return IOPRIO_CLASS_IDLE;
-	else if (task_is_realtime(task))
+	else if (rt_or_dl_task_policy(task))
 		return IOPRIO_CLASS_RT;
 	else
 		return IOPRIO_CLASS_BE;

include/linux/posix-timers.h

@@ -158,7 +158,7 @@ static inline void posix_cputimers_init_work(void) { }
  * @rcu:		RCU head for freeing the timer.
  */
 struct k_itimer {
-	struct list_head	list;
+	struct hlist_node	list;
 	struct hlist_node	t_hash;
 	spinlock_t		it_lock;
 	const struct k_clock	*kclock;

include/linux/sched.h

@@ -149,8 +149,9 @@ struct user_event_mm;
  * Special states are those that do not use the normal wait-loop pattern. See
  * the comment with set_special_state().
  */
-#define is_special_task_state(state)				\
-	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
+#define is_special_task_state(state)					\
+	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED |	\
+		    TASK_DEAD | TASK_FROZEN))
 
 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
 # define debug_normal_state_change(state_value)				\
@@ -541,9 +542,14 @@ struct sched_entity {
 	struct rb_node			run_node;
 	u64				deadline;
 	u64				min_vruntime;
+	u64				min_slice;
 
 	struct list_head		group_node;
-	unsigned int			on_rq;
+	unsigned char			on_rq;
+	unsigned char			sched_delayed;
+	unsigned char			rel_deadline;
+	unsigned char			custom_slice;
+					/* hole */
 
 	u64				exec_start;
 	u64				sum_exec_runtime;
@@ -639,12 +645,26 @@ struct sched_dl_entity {
 	 *
 	 * @dl_overrun tells if the task asked to be informed about runtime
 	 * overruns.
+	 *
+	 * @dl_server tells if this is a server entity.
+	 *
+	 * @dl_defer tells if this is a deferred or regular server. For
+	 * now only defer server exists.
+	 *
+	 * @dl_defer_armed tells if the deferrable server is waiting
+	 * for the replenishment timer to activate it.
+	 *
+	 * @dl_defer_running tells if the deferrable server is actually
+	 * running, skipping the defer phase.
 	 */
 	unsigned int			dl_throttled      : 1;
 	unsigned int			dl_yielded        : 1;
 	unsigned int			dl_non_contending : 1;
 	unsigned int			dl_overrun	  : 1;
 	unsigned int			dl_server         : 1;
+	unsigned int			dl_defer	  : 1;
+	unsigned int			dl_defer_armed	  : 1;
+	unsigned int			dl_defer_running  : 1;
 
 	/*
 	 * Bandwidth enforcement timer. Each -deadline task has its
@@ -672,7 +692,8 @@ struct sched_dl_entity {
 	 */
 	struct rq			*rq;
 	dl_server_has_tasks_f		server_has_tasks;
-	dl_server_pick_f		server_pick;
+	dl_server_pick_f		server_pick_next;
+	dl_server_pick_f		server_pick_task;
 
 #ifdef CONFIG_RT_MUTEXES
 	/*

include/linux/sched/deadline.h

@@ -10,16 +10,16 @@
 
 #include <linux/sched.h>
 
-#define MAX_DL_PRIO		0
-
-static inline int dl_prio(int prio)
+static inline bool dl_prio(int prio)
 {
-	if (unlikely(prio < MAX_DL_PRIO))
-		return 1;
-	return 0;
+	return unlikely(prio < MAX_DL_PRIO);
 }
 
-static inline int dl_task(struct task_struct *p)
+/*
+ * Returns true if a task has a priority that belongs to DL class. PI-boosted
+ * tasks will return true. Use dl_policy() to ignore PI-boosted tasks.
+ */
+static inline bool dl_task(struct task_struct *p)
 {
 	return dl_prio(p->prio);
 }

include/linux/sched/prio.h

@@ -14,6 +14,7 @@
  */
 
 #define MAX_RT_PRIO		100
+#define MAX_DL_PRIO		0
 
 #define MAX_PRIO		(MAX_RT_PRIO + NICE_WIDTH)
 #define DEFAULT_PRIO		(MAX_RT_PRIO + NICE_WIDTH / 2)

include/linux/sched/rt.h

@@ -6,19 +6,40 @@
 
 struct task_struct;
 
-static inline int rt_prio(int prio)
+static inline bool rt_prio(int prio)
 {
-	if (unlikely(prio < MAX_RT_PRIO))
-		return 1;
-	return 0;
+	return unlikely(prio < MAX_RT_PRIO && prio >= MAX_DL_PRIO);
 }
 
-static inline int rt_task(struct task_struct *p)
+static inline bool rt_or_dl_prio(int prio)
+{
+	return unlikely(prio < MAX_RT_PRIO);
+}
+
+/*
+ * Returns true if a task has a priority that belongs to RT class. PI-boosted
+ * tasks will return true. Use rt_policy() to ignore PI-boosted tasks.
+ */
+static inline bool rt_task(struct task_struct *p)
 {
 	return rt_prio(p->prio);
 }
 
-static inline bool task_is_realtime(struct task_struct *tsk)
+/*
+ * Returns true if a task has a priority that belongs to RT or DL classes.
+ * PI-boosted tasks will return true. Use rt_or_dl_task_policy() to ignore
+ * PI-boosted tasks.
+ */
+static inline bool rt_or_dl_task(struct task_struct *p)
+{
+	return rt_or_dl_prio(p->prio);
+}
+
+/*
+ * Returns true if a task has a policy that belongs to RT or DL classes.
+ * PI-boosted tasks will return false.
+ */
+static inline bool rt_or_dl_task_policy(struct task_struct *tsk)
 {
 	int policy = tsk->policy;
 

include/linux/sched/signal.h

@@ -137,7 +137,7 @@ struct signal_struct {
 
 	/* POSIX.1b Interval Timers */
 	unsigned int		next_posix_timer_id;
-	struct list_head	posix_timers;
+	struct hlist_head	posix_timers;
 
 	/* ITIMER_REAL timer for the process */
 	struct hrtimer real_timer;
@@ -276,8 +276,7 @@ static inline void signal_set_stop_flags(struct signal_struct *sig,
 extern void flush_signals(struct task_struct *);
 extern void ignore_signals(struct task_struct *);
 extern void flush_signal_handlers(struct task_struct *, int force_default);
-extern int dequeue_signal(struct task_struct *task, sigset_t *mask,
-			  kernel_siginfo_t *info, enum pid_type *type);
+extern int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type);
 
 static inline int kernel_dequeue_signal(void)
 {
@@ -287,7 +286,7 @@ static inline int kernel_dequeue_signal(void)
 	int ret;
 
 	spin_lock_irq(&task->sighand->siglock);
-	ret = dequeue_signal(task, &task->blocked, &__info, &__type);
+	ret = dequeue_signal(&task->blocked, &__info, &__type);
 	spin_unlock_irq(&task->sighand->siglock);
 
 	return ret;

init/init_task.c

@@ -29,7 +29,7 @@ static struct signal_struct init_signals = {
 	.cred_guard_mutex = __MUTEX_INITIALIZER(init_signals.cred_guard_mutex),
 	.exec_update_lock = __RWSEM_INITIALIZER(init_signals.exec_update_lock),
 #ifdef CONFIG_POSIX_TIMERS
-	.posix_timers = LIST_HEAD_INIT(init_signals.posix_timers),
+	.posix_timers	= HLIST_HEAD_INIT,
 	.cputimer	= {
 		.cputime_atomic	= INIT_CPUTIME_ATOMIC,
 	},

kernel/fork.c

@@ -1861,7 +1861,7 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
 	prev_cputime_init(&sig->prev_cputime);
 
 #ifdef CONFIG_POSIX_TIMERS
-	INIT_LIST_HEAD(&sig->posix_timers);
+	INIT_HLIST_HEAD(&sig->posix_timers);
 	hrtimer_init(&sig->real_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	sig->real_timer.function = it_real_fn;
 #endif

kernel/freezer.c

@@ -72,7 +72,7 @@ bool __refrigerator(bool check_kthr_stop)
 		bool freeze;
 
 		raw_spin_lock_irq(&current->pi_lock);
-		set_current_state(TASK_FROZEN);
+		WRITE_ONCE(current->__state, TASK_FROZEN);
 		/* unstale saved_state so that __thaw_task() will wake us up */
 		current->saved_state = TASK_RUNNING;
 		raw_spin_unlock_irq(&current->pi_lock);

kernel/locking/rtmutex.c

@@ -347,7 +347,7 @@ static __always_inline int __waiter_prio(struct task_struct *task)
 {
 	int prio = task->prio;
 
-	if (!rt_prio(prio))
+	if (!rt_or_dl_prio(prio))
 		return DEFAULT_PRIO;
 
 	return prio;
@@ -435,7 +435,7 @@ static inline bool rt_mutex_steal(struct rt_mutex_waiter *waiter,
 	 * Note that RT tasks are excluded from same priority (lateral)
 	 * steals to prevent the introduction of an unbounded latency.
 	 */
-	if (rt_prio(waiter->tree.prio) || dl_prio(waiter->tree.prio))
+	if (rt_or_dl_prio(waiter->tree.prio))
 		return false;
 
 	return rt_waiter_node_equal(&waiter->tree, &top_waiter->tree);

kernel/locking/rwsem.c

@@ -631,7 +631,7 @@ static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
 			 * if it is an RT task or wait in the wait queue
 			 * for too long.
 			 */
-			if (has_handoff || (!rt_task(waiter->task) &&
+			if (has_handoff || (!rt_or_dl_task(waiter->task) &&
 					    !time_after(jiffies, waiter->timeout)))
 				return false;
 
@@ -914,7 +914,7 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem)
 		if (owner_state != OWNER_WRITER) {
 			if (need_resched())
 				break;
-			if (rt_task(current) &&
+			if (rt_or_dl_task(current) &&
 			   (prev_owner_state != OWNER_WRITER))
 				break;
 		}

kernel/locking/ww_mutex.h

@@ -237,7 +237,7 @@ __ww_ctx_less(struct ww_acquire_ctx *a, struct ww_acquire_ctx *b)
 	int a_prio = a->task->prio;
 	int b_prio = b->task->prio;
 
-	if (rt_prio(a_prio) || rt_prio(b_prio)) {
+	if (rt_or_dl_prio(a_prio) || rt_or_dl_prio(b_prio)) {
 
 		if (a_prio > b_prio)
 			return true;