Commit 2ec30311 authored by Paul E. McKenney's avatar Paul E. McKenney
Browse files

srcu: Dynamically allocate srcu_node array 



This commit shrinks the srcu_struct structure by converting its ->node
field from a fixed-size compile-time array to a pointer to a dynamically
allocated array.  In kernels built with large values of NR_CPUS that boot
on systems with smaller numbers of CPUs, this can save significant memory.

[ paulmck: Apply kernel test robot feedback. ]

Reported-by: A cast of thousands
Co-developed-by: default avatarNeeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: default avatarNeeraj Upadhyay <quic_neeraju@quicinc.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@kernel.org>
parent 994f7068

include/linux/srcutree.h

+1 −1
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ struct srcu_node { 
 * Per-SRCU-domain structure, similar in function to rcu_state.

 */

struct srcu_struct {

	struct srcu_node node[NUM_RCU_NODES];	/* Combining tree. */

	struct srcu_node *node;			/* Combining tree. */

	struct srcu_node *level[RCU_NUM_LVLS + 1];

						/* First node at each level. */

	int srcu_size_state;			/* Small-to-big transition state. */

kernel/rcu/srcutree.c

+50 −15
Original line number Diff line number Diff line
@@ -24,6 +24,7 @@ 
#include <linux/smp.h>

#include <linux/delay.h>

#include <linux/module.h>

#include <linux/slab.h>

#include <linux/srcu.h>



#include "rcu.h"
@@ -75,12 +76,42 @@ do { \ 
	spin_unlock_irqrestore(&ACCESS_PRIVATE(p, lock), flags)	\



/*

 * Initialize SRCU combining tree.  Note that statically allocated

 * Initialize SRCU per-CPU data.  Note that statically allocated

 * srcu_struct structures might already have srcu_read_lock() and

 * srcu_read_unlock() running against them.  So if the is_static parameter

 * is set, don't initialize ->srcu_lock_count[] and ->srcu_unlock_count[].

 */

static void init_srcu_struct_nodes(struct srcu_struct *ssp)

static void init_srcu_struct_data(struct srcu_struct *ssp)

{

	int cpu;

	struct srcu_data *sdp;



	/*

	 * Initialize the per-CPU srcu_data array, which feeds into the

	 * leaves of the srcu_node tree.

	 */

	WARN_ON_ONCE(ARRAY_SIZE(sdp->srcu_lock_count) !=

		     ARRAY_SIZE(sdp->srcu_unlock_count));

	for_each_possible_cpu(cpu) {

		sdp = per_cpu_ptr(ssp->sda, cpu);

		spin_lock_init(&ACCESS_PRIVATE(sdp, lock));

		rcu_segcblist_init(&sdp->srcu_cblist);

		sdp->srcu_cblist_invoking = false;

		sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq;

		sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq;

		sdp->mynode = NULL;

		sdp->cpu = cpu;

		INIT_WORK(&sdp->work, srcu_invoke_callbacks);

		timer_setup(&sdp->delay_work, srcu_delay_timer, 0);

		sdp->ssp = ssp;

	}

}



/*

 * Allocated and initialize SRCU combining tree.  Returns @true if

 * allocation succeeded and @false otherwise.

 */

static bool init_srcu_struct_nodes(struct srcu_struct *ssp)

{

	int cpu;

	int i;
@@ -92,6 +123,9 @@ static void init_srcu_struct_nodes(struct srcu_struct *ssp) 


	/* Initialize geometry if it has not already been initialized. */

	rcu_init_geometry();

	ssp->node = kcalloc(rcu_num_nodes, sizeof(*ssp->node), GFP_ATOMIC);

	if (!ssp->node)

		return false;



	/* Work out the overall tree geometry. */

	ssp->level[0] = &ssp->node[0];
@@ -129,30 +163,20 @@ static void init_srcu_struct_nodes(struct srcu_struct *ssp) 
	 * Initialize the per-CPU srcu_data array, which feeds into the

	 * leaves of the srcu_node tree.

	 */

	WARN_ON_ONCE(ARRAY_SIZE(sdp->srcu_lock_count) !=

		     ARRAY_SIZE(sdp->srcu_unlock_count));

	level = rcu_num_lvls - 1;

	snp_first = ssp->level[level];

	for_each_possible_cpu(cpu) {

		sdp = per_cpu_ptr(ssp->sda, cpu);

		spin_lock_init(&ACCESS_PRIVATE(sdp, lock));

		rcu_segcblist_init(&sdp->srcu_cblist);

		sdp->srcu_cblist_invoking = false;

		sdp->srcu_gp_seq_needed = ssp->srcu_gp_seq;

		sdp->srcu_gp_seq_needed_exp = ssp->srcu_gp_seq;

		sdp->mynode = &snp_first[cpu / levelspread[level]];

		for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) {

			if (snp->grplo < 0)

				snp->grplo = cpu;

			snp->grphi = cpu;

		}

		sdp->cpu = cpu;

		INIT_WORK(&sdp->work, srcu_invoke_callbacks);

		timer_setup(&sdp->delay_work, srcu_delay_timer, 0);

		sdp->ssp = ssp;

		sdp->grpmask = 1 << (cpu - sdp->mynode->grplo);

	}

	smp_store_release(&ssp->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);

	return true;

}



/*
@@ -163,6 +187,7 @@ static void init_srcu_struct_nodes(struct srcu_struct *ssp) 
static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)

{

	ssp->srcu_size_state = SRCU_SIZE_SMALL;

	ssp->node = NULL;

	mutex_init(&ssp->srcu_cb_mutex);

	mutex_init(&ssp->srcu_gp_mutex);

	ssp->srcu_idx = 0;
@@ -175,8 +200,16 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 
		ssp->sda = alloc_percpu(struct srcu_data);

	if (!ssp->sda)

		return -ENOMEM;

	init_srcu_struct_nodes(ssp);

	init_srcu_struct_data(ssp);

	if (!init_srcu_struct_nodes(ssp)) {

		if (!is_static) {

			free_percpu(ssp->sda);

			ssp->sda = NULL;

			return -ENOMEM;

		}

	} else {

		ssp->srcu_size_state = SRCU_SIZE_BIG;

	}

	ssp->srcu_gp_seq_needed_exp = 0;

	ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();

	smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */
@@ -393,6 +426,8 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) 
	}

	free_percpu(ssp->sda);

	ssp->sda = NULL;

	kfree(ssp->node);

	ssp->node = NULL;

	ssp->srcu_size_state = SRCU_SIZE_SMALL;

}

EXPORT_SYMBOL_GPL(cleanup_srcu_struct);