Commit c70055d8 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge tag 'sched_urgent_for_v6.1_rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull scheduler fixes from Borislav Petkov:

 - Adjust code to not trip up CFI

 - Fix sched group cookie matching

* tag 'sched_urgent_for_v6.1_rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  sched: Introduce struct balance_callback to avoid CFI mismatches
  sched/core: Fix comparison in sched_group_cookie_match()
parents 6204a81a 8e5bad7d

kernel/sched/core.c

+12 −12
Original line number Diff line number Diff line
@@ -4823,10 +4823,10 @@ static inline void finish_task(struct task_struct *prev) 


#ifdef CONFIG_SMP



static void do_balance_callbacks(struct rq *rq, struct callback_head *head)

static void do_balance_callbacks(struct rq *rq, struct balance_callback *head)

{

	void (*func)(struct rq *rq);

	struct callback_head *next;

	struct balance_callback *next;



	lockdep_assert_rq_held(rq);
@@ -4853,15 +4853,15 @@ static void balance_push(struct rq *rq); 
 * This abuse is tolerated because it places all the unlikely/odd cases behind

 * a single test, namely: rq->balance_callback == NULL.

 */

struct callback_head balance_push_callback = {

struct balance_callback balance_push_callback = {

	.next = NULL,

	.func = (void (*)(struct callback_head *))balance_push,

	.func = balance_push,

};



static inline struct callback_head *

static inline struct balance_callback *

__splice_balance_callbacks(struct rq *rq, bool split)

{

	struct callback_head *head = rq->balance_callback;

	struct balance_callback *head = rq->balance_callback;



	if (likely(!head))

		return NULL;
@@ -4883,7 +4883,7 @@ __splice_balance_callbacks(struct rq *rq, bool split) 
	return head;

}



static inline struct callback_head *splice_balance_callbacks(struct rq *rq)

static inline struct balance_callback *splice_balance_callbacks(struct rq *rq)

{

	return __splice_balance_callbacks(rq, true);

}
@@ -4893,7 +4893,7 @@ static void __balance_callbacks(struct rq *rq) 
	do_balance_callbacks(rq, __splice_balance_callbacks(rq, false));

}



static inline void balance_callbacks(struct rq *rq, struct callback_head *head)

static inline void balance_callbacks(struct rq *rq, struct balance_callback *head)

{

	unsigned long flags;
@@ -4910,12 +4910,12 @@ static inline void __balance_callbacks(struct rq *rq) 
{

}



static inline struct callback_head *splice_balance_callbacks(struct rq *rq)

static inline struct balance_callback *splice_balance_callbacks(struct rq *rq)

{

	return NULL;

}



static inline void balance_callbacks(struct rq *rq, struct callback_head *head)

static inline void balance_callbacks(struct rq *rq, struct balance_callback *head)

{

}
@@ -6188,7 +6188,7 @@ static void sched_core_balance(struct rq *rq) 
	preempt_enable();

}



static DEFINE_PER_CPU(struct callback_head, core_balance_head);

static DEFINE_PER_CPU(struct balance_callback, core_balance_head);



static void queue_core_balance(struct rq *rq)

{
@@ -7419,7 +7419,7 @@ static int __sched_setscheduler(struct task_struct *p, 
	int oldpolicy = -1, policy = attr->sched_policy;

	int retval, oldprio, newprio, queued, running;

	const struct sched_class *prev_class;

	struct callback_head *head;

	struct balance_callback *head;

	struct rq_flags rf;

	int reset_on_fork;

	int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;

kernel/sched/deadline.c

+2 −2
Original line number Diff line number Diff line
@@ -644,8 +644,8 @@ static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev) 
	return rq->online && dl_task(prev);

}



static DEFINE_PER_CPU(struct callback_head, dl_push_head);

static DEFINE_PER_CPU(struct callback_head, dl_pull_head);

static DEFINE_PER_CPU(struct balance_callback, dl_push_head);

static DEFINE_PER_CPU(struct balance_callback, dl_pull_head);



static void push_dl_tasks(struct rq *);

static void pull_dl_task(struct rq *);

kernel/sched/rt.c

+2 −2
Original line number Diff line number Diff line
@@ -410,8 +410,8 @@ static inline int has_pushable_tasks(struct rq *rq) 
	return !plist_head_empty(&rq->rt.pushable_tasks);

}



static DEFINE_PER_CPU(struct callback_head, rt_push_head);

static DEFINE_PER_CPU(struct callback_head, rt_pull_head);

static DEFINE_PER_CPU(struct balance_callback, rt_push_head);

static DEFINE_PER_CPU(struct balance_callback, rt_pull_head);



static void push_rt_tasks(struct rq *);

static void pull_rt_task(struct rq *);

kernel/sched/sched.h

+19 −13
Original line number Diff line number Diff line
@@ -938,6 +938,12 @@ struct uclamp_rq { 
DECLARE_STATIC_KEY_FALSE(sched_uclamp_used);

#endif /* CONFIG_UCLAMP_TASK */



struct rq;

struct balance_callback {

	struct balance_callback *next;

	void (*func)(struct rq *rq);

};



/*

 * This is the main, per-CPU runqueue data structure.

 *
@@ -1036,7 +1042,7 @@ struct rq { 
	unsigned long		cpu_capacity;

	unsigned long		cpu_capacity_orig;



	struct callback_head	*balance_callback;

	struct balance_callback *balance_callback;



	unsigned char		nohz_idle_balance;

	unsigned char		idle_balance;
@@ -1182,6 +1188,14 @@ static inline bool is_migration_disabled(struct task_struct *p) 
#endif

}



DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);



#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))

#define this_rq()		this_cpu_ptr(&runqueues)

#define task_rq(p)		cpu_rq(task_cpu(p))

#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

#define raw_rq()		raw_cpu_ptr(&runqueues)



struct sched_group;

#ifdef CONFIG_SCHED_CORE

static inline struct cpumask *sched_group_span(struct sched_group *sg);
@@ -1269,7 +1283,7 @@ static inline bool sched_group_cookie_match(struct rq *rq, 
		return true;



	for_each_cpu_and(cpu, sched_group_span(group), p->cpus_ptr) {

		if (sched_core_cookie_match(rq, p))

		if (sched_core_cookie_match(cpu_rq(cpu), p))

			return true;

	}

	return false;
@@ -1384,14 +1398,6 @@ static inline void update_idle_core(struct rq *rq) 
static inline void update_idle_core(struct rq *rq) { }

#endif



DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);



#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))

#define this_rq()		this_cpu_ptr(&runqueues)

#define task_rq(p)		cpu_rq(task_cpu(p))

#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

#define raw_rq()		raw_cpu_ptr(&runqueues)



#ifdef CONFIG_FAIR_GROUP_SCHED

static inline struct task_struct *task_of(struct sched_entity *se)

{
@@ -1544,7 +1550,7 @@ struct rq_flags { 
#endif

};



extern struct callback_head balance_push_callback;

extern struct balance_callback balance_push_callback;



/*

 * Lockdep annotation that avoids accidental unlocks; it's like a
@@ -1724,7 +1730,7 @@ init_numa_balancing(unsigned long clone_flags, struct task_struct *p) 


static inline void

queue_balance_callback(struct rq *rq,

		       struct callback_head *head,

		       struct balance_callback *head,

		       void (*func)(struct rq *rq))

{

	lockdep_assert_rq_held(rq);
@@ -1737,7 +1743,7 @@ queue_balance_callback(struct rq *rq, 
	if (unlikely(head->next || rq->balance_callback == &balance_push_callback))

		return;



	head->func = (void (*)(struct callback_head *))func;

	head->func = func;

	head->next = rq->balance_callback;

	rq->balance_callback = head;

}