block: use bi_max_vecs to find the bvec pool (7a800a20) · Commits · jan.koester / Linux

block/bio-integrity.c

+4 −7

Original line number	Original line	Diff line number	Diff line
	@@ -28,7 +28,7 @@ static void __bio_integrity_free(struct bio_set *bs,
	if (bs && mempool_initialized(&bs->bio_integrity_pool)) {		if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
	if (bip->bip_vec)		if (bip->bip_vec)
	bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,		bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
	bip->bip_slab);		bip->bip_max_vcnt);
	mempool_free(bip, &bs->bio_integrity_pool);		mempool_free(bip, &bs->bio_integrity_pool);
	} else {		} else {
	kfree(bip);		kfree(bip);
	@@ -70,14 +70,11 @@ struct bio_integrity_payload bio_integrity_alloc(struct bio bio,
	memset(bip, 0, sizeof(*bip));		memset(bip, 0, sizeof(*bip));

	if (nr_vecs > inline_vecs) {		if (nr_vecs > inline_vecs) {
	unsigned long idx = 0;		bip->bip_max_vcnt = nr_vecs;
			bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,
	bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,		&bip->bip_max_vcnt, gfp_mask);
	&bs->bvec_integrity_pool);
	if (!bip->bip_vec)		if (!bip->bip_vec)
	goto err;		goto err;
	bip->bip_max_vcnt = bvec_nr_vecs(idx);
	bip->bip_slab = idx;
	} else {		} else {
	bip->bip_vec = bip->bip_inline_vecs;		bip->bip_vec = bip->bip_inline_vecs;
	bip->bip_max_vcnt = inline_vecs;		bip->bip_max_vcnt = inline_vecs;

block/bio.c

+43 −61

Original line number	Original line	Diff line number	Diff line
	@@ -36,6 +36,24 @@ static struct biovec_slab {
	{ .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },		{ .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },
	};		};

			static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
			{
			switch (nr_vecs) {
			/* smaller bios use inline vecs */
			case 5 ... 16:
			return &bvec_slabs[0];
			case 17 ... 64:
			return &bvec_slabs[1];
			case 65 ... 128:
			return &bvec_slabs[2];
			case 129 ... BIO_MAX_PAGES:
			return &bvec_slabs[3];
			default:
			BUG();
			return NULL;
			}
			}

	/*		/*
	* fs_bio_set is the bio_set containing bio and iovec memory pools used by		* fs_bio_set is the bio_set containing bio and iovec memory pools used by
	* IO code that does not need private memory pools.		* IO code that does not need private memory pools.
	@@ -131,26 +149,14 @@ static void bio_put_slab(struct bio_set *bs)
	mutex_unlock(&bio_slab_lock);		mutex_unlock(&bio_slab_lock);
	}		}

	unsigned int bvec_nr_vecs(unsigned short idx)		void bvec_free(mempool_t pool, struct bio_vec bv, unsigned short nr_vecs)
	{		{
	return bvec_slabs[--idx].nr_vecs;		BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES);
	}

	void bvec_free(mempool_t pool, struct bio_vec bv, unsigned int idx)
	{
	if (!idx)
	return;
	idx--;

	BIO_BUG_ON(idx >= BVEC_POOL_NR);		if (nr_vecs == BIO_MAX_PAGES)

	if (idx == BVEC_POOL_MAX) {
	mempool_free(bv, pool);		mempool_free(bv, pool);
	} else {		else if (nr_vecs > BIO_INLINE_VECS)
	struct biovec_slab *bvs = bvec_slabs + idx;		kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);

	kmem_cache_free(bvs->slab, bv);
	}
	}		}

	/*		/*
	@@ -163,48 +169,34 @@ static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
	__GFP_NOMEMALLOC \| __GFP_NORETRY \| __GFP_NOWARN;		__GFP_NOMEMALLOC \| __GFP_NORETRY \| __GFP_NOWARN;
	}		}

	struct bio_vec bvec_alloc(gfp_t gfp_mask, int nr, unsigned long idx,		struct bio_vec bvec_alloc(mempool_t pool, unsigned short *nr_vecs,
	mempool_t *pool)		gfp_t gfp_mask)
	{		{
			struct biovec_slab bvs = biovec_slab(nr_vecs);

			if (WARN_ON_ONCE(!bvs))
			return NULL;

	/*		/*
	* see comment near bvec_array define!		* Upgrade the nr_vecs request to take full advantage of the allocation.
			* We also rely on this in the bvec_free path.
	*/		*/
	switch (nr) {		*nr_vecs = bvs->nr_vecs;
	/* smaller bios use inline vecs */
	case 5 ... 16:
	*idx = 2;
	break;
	case 17 ... 64:
	*idx = 3;
	break;
	case 65 ... 128:
	*idx = 4;
	break;
	case 129 ... BIO_MAX_PAGES:
	*idx = 5;
	break;
	default:
	return NULL;
	}

	/*		/*
	* Try a slab allocation first for all smaller allocations. If that		* Try a slab allocation first for all smaller allocations. If that
	* fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.		* fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
	* The mempool is sized to handle up to BIO_MAX_PAGES entries.		* The mempool is sized to handle up to BIO_MAX_PAGES entries.
	*/		*/
	if (*idx < BVEC_POOL_MAX) {		if (*nr_vecs < BIO_MAX_PAGES) {
	struct biovec_slab bvs = bvec_slabs + idx;
	struct bio_vec *bvl;		struct bio_vec *bvl;

	bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));		bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
	if (likely(bvl) \|\| !(gfp_mask & __GFP_DIRECT_RECLAIM)) {		if (likely(bvl) \|\| !(gfp_mask & __GFP_DIRECT_RECLAIM))
	(*idx)++;
	return bvl;		return bvl;
	}		*nr_vecs = BIO_MAX_PAGES;
	*idx = BVEC_POOL_MAX;
	}		}

	(*idx)++;
	return mempool_alloc(pool, gfp_mask);		return mempool_alloc(pool, gfp_mask);
	}		}

	@@ -231,7 +223,7 @@ static void bio_free(struct bio *bio)
	bio_uninit(bio);		bio_uninit(bio);

	if (bs) {		if (bs) {
	bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));		bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);

	/*		/*
	* If we have front padding, adjust the bio pointer before freeing		* If we have front padding, adjust the bio pointer before freeing
	@@ -275,12 +267,8 @@ EXPORT_SYMBOL(bio_init);
	*/		*/
	void bio_reset(struct bio *bio)		void bio_reset(struct bio *bio)
	{		{
	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);

	bio_uninit(bio);		bio_uninit(bio);

	memset(bio, 0, BIO_RESET_BYTES);		memset(bio, 0, BIO_RESET_BYTES);
	bio->bi_flags = flags;
	atomic_set(&bio->__bi_remaining, 1);		atomic_set(&bio->__bi_remaining, 1);
	}		}
	EXPORT_SYMBOL(bio_reset);		EXPORT_SYMBOL(bio_reset);
	@@ -453,22 +441,18 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs,

	bio = p + bs->front_pad;		bio = p + bs->front_pad;
	if (nr_iovecs > BIO_INLINE_VECS) {		if (nr_iovecs > BIO_INLINE_VECS) {
	unsigned long idx = 0;
	struct bio_vec *bvl = NULL;		struct bio_vec *bvl = NULL;

	bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);		bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
	if (!bvl && gfp_mask != saved_gfp) {		if (!bvl && gfp_mask != saved_gfp) {
	punt_bios_to_rescuer(bs);		punt_bios_to_rescuer(bs);
	gfp_mask = saved_gfp;		gfp_mask = saved_gfp;
	bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx,		bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
	&bs->bvec_pool);
	}		}

	if (unlikely(!bvl))		if (unlikely(!bvl))
	goto err_free;		goto err_free;

	bio_init(bio, bvl, bvec_nr_vecs(idx));		bio_init(bio, bvl, nr_iovecs);
	bio->bi_flags \|= idx << BVEC_POOL_OFFSET;
	} else if (nr_iovecs) {		} else if (nr_iovecs) {
	bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS);		bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS);
	} else {		} else {
	@@ -644,7 +628,7 @@ EXPORT_SYMBOL(bio_put);
	*/		*/
	void __bio_clone_fast(struct bio bio, struct bio bio_src)		void __bio_clone_fast(struct bio bio, struct bio bio_src)
	{		{
	BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));		WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs);

	/*		/*
	* most users will be overriding ->bi_bdev with a new target,		* most users will be overriding ->bi_bdev with a new target,
	@@ -934,7 +918,7 @@ EXPORT_SYMBOL_GPL(bio_release_pages);

	static int bio_iov_bvec_set(struct bio bio, struct iov_iter iter)		static int bio_iov_bvec_set(struct bio bio, struct iov_iter iter)
	{		{
	WARN_ON_ONCE(BVEC_POOL_IDX(bio) != 0);		WARN_ON_ONCE(bio->bi_max_vecs);

	bio->bi_vcnt = iter->nr_segs;		bio->bi_vcnt = iter->nr_segs;
	bio->bi_io_vec = (struct bio_vec *)iter->bvec;		bio->bi_io_vec = (struct bio_vec *)iter->bvec;
	@@ -1495,7 +1479,7 @@ EXPORT_SYMBOL_GPL(bio_trim);
	*/		*/
	int biovec_init_pool(mempool_t *pool, int pool_entries)		int biovec_init_pool(mempool_t *pool, int pool_entries)
	{		{
	struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;		struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;

	return mempool_init_slab_pool(pool, pool_entries, bp->slab);		return mempool_init_slab_pool(pool, pool_entries, bp->slab);
	}		}
	@@ -1605,8 +1589,6 @@ static int __init init_bio(void)
	{		{
	int i;		int i;

	BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);

	bio_integrity_init();		bio_integrity_init();

	for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {		for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {

block/blk.h

+3 −3

Original line number	Original line	Diff line number	Diff line
	@@ -56,9 +56,9 @@ void blk_free_flush_queue(struct blk_flush_queue *q);
	void blk_freeze_queue(struct request_queue *q);		void blk_freeze_queue(struct request_queue *q);

	#define BIO_INLINE_VECS 4		#define BIO_INLINE_VECS 4
	struct bio_vec bvec_alloc(gfp_t, int, unsigned long , mempool_t *);		struct bio_vec bvec_alloc(mempool_t pool, unsigned short *nr_vecs,
	void bvec_free(mempool_t , struct bio_vec , unsigned int);		gfp_t gfp_mask);
	unsigned int bvec_nr_vecs(unsigned short idx);		void bvec_free(mempool_t pool, struct bio_vec bv, unsigned short nr_vecs);

	static inline bool biovec_phys_mergeable(struct request_queue *q,		static inline bool biovec_phys_mergeable(struct request_queue *q,
	struct bio_vec vec1, struct bio_vec vec2)		struct bio_vec vec1, struct bio_vec vec2)

include/linux/bio.h

+0 −1

Original line number	Original line	Diff line number	Diff line
	@@ -329,7 +329,6 @@ struct bio_integrity_payload {

	struct bvec_iter bip_iter;		struct bvec_iter bip_iter;

	unsigned short bip_slab; /* slab the bip came from */
	unsigned short bip_vcnt; /* # of integrity bio_vecs */		unsigned short bip_vcnt; /* # of integrity bio_vecs */
	unsigned short bip_max_vcnt; /* integrity bio_vec slots */		unsigned short bip_max_vcnt; /* integrity bio_vec slots */
	unsigned short bip_flags; /* control flags */		unsigned short bip_flags; /* control flags */

include/linux/blk_types.h

+1 −28

Original line number	Original line	Diff line number	Diff line
	@@ -227,7 +227,7 @@ struct bio {
	* top bits REQ_OP. Use		* top bits REQ_OP. Use
	* accessors.		* accessors.
	*/		*/
	unsigned short bi_flags; /* status, etc and bvec pool number */		unsigned short bi_flags; /* BIO_* below */
	unsigned short bi_ioprio;		unsigned short bi_ioprio;
	unsigned short bi_write_hint;		unsigned short bi_write_hint;
	blk_status_t bi_status;		blk_status_t bi_status;
	@@ -307,33 +307,6 @@ enum {
	BIO_FLAG_LAST		BIO_FLAG_LAST
	};		};

	/* See BVEC_POOL_OFFSET below before adding new flags */

	/*
	* We support 6 different bvec pools, the last one is magic in that it
	* is backed by a mempool.
	*/
	#define BVEC_POOL_NR 6
	#define BVEC_POOL_MAX (BVEC_POOL_NR - 1)

	/*
	* Top 3 bits of bio flags indicate the pool the bvecs came from. We add
	* 1 to the actual index so that 0 indicates that there are no bvecs to be
	* freed.
	*/
	#define BVEC_POOL_BITS (3)
	#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS)
	#define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET)
	#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)
	# error "BVEC_POOL_BITS is too small"
	#endif

	/*
	* Flags starting here get preserved by bio_reset() - this includes
	* only BVEC_POOL_IDX()
	*/
	#define BIO_RESET_BITS BVEC_POOL_OFFSET

	typedef __u32 __bitwise blk_mq_req_flags_t;		typedef __u32 __bitwise blk_mq_req_flags_t;

	/*		/*