Commit f4209439 authored by Maciej S. Szmigiero's avatar Maciej S. Szmigiero Committed by Paolo Bonzini
Browse files

KVM: Optimize gfn lookup in kvm_zap_gfn_range() 



Introduce a memslots gfn upper bound operation and use it to optimize
kvm_zap_gfn_range().
This way this handler can do a quick lookup for intersecting gfns and won't
have to do a linear scan of the whole memslot set.

Signed-off-by: default avatarMaciej S. Szmigiero <maciej.szmigiero@oracle.com>
Message-Id: <ef242146a87a335ee93b441dcf01665cb847c902.1638817641.git.maciej.szmigiero@oracle.com>
parent bcb63dcd

arch/x86/kvm/mmu/mmu.c

+9 −3
Original line number Diff line number Diff line
@@ -5728,19 +5728,22 @@ static bool __kvm_zap_rmaps(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) 
{

	const struct kvm_memory_slot *memslot;

	struct kvm_memslots *slots;

	struct kvm_memslot_iter iter;

	bool flush = false;

	gfn_t start, end;

	int i, bkt;

	int i;



	if (!kvm_memslots_have_rmaps(kvm))

		return flush;



	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {

		slots = __kvm_memslots(kvm, i);

		kvm_for_each_memslot(memslot, bkt, slots) {



		kvm_for_each_memslot_in_gfn_range(&iter, slots, gfn_start, gfn_end) {

			memslot = iter.slot;

			start = max(gfn_start, memslot->base_gfn);

			end = min(gfn_end, memslot->base_gfn + memslot->npages);

			if (start >= end)

			if (WARN_ON_ONCE(start >= end))

				continue;



			flush = slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
@@ -5761,6 +5764,9 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) 
	bool flush;

	int i;



	if (WARN_ON_ONCE(gfn_end <= gfn_start))

		return;



	write_lock(&kvm->mmu_lock);



	kvm_inc_notifier_count(kvm, gfn_start, gfn_end);

include/linux/kvm_host.h

+94 −0
Original line number Diff line number Diff line
@@ -846,6 +846,100 @@ struct kvm_memory_slot *id_to_memslot(struct kvm_memslots *slots, int id) 
	return NULL;

}



/* Iterator used for walking memslots that overlap a gfn range. */

struct kvm_memslot_iter {

	struct kvm_memslots *slots;

	struct rb_node *node;

	struct kvm_memory_slot *slot;

};



static inline void kvm_memslot_iter_next(struct kvm_memslot_iter *iter)

{

	iter->node = rb_next(iter->node);

	if (!iter->node)

		return;



	iter->slot = container_of(iter->node, struct kvm_memory_slot, gfn_node[iter->slots->node_idx]);

}



static inline void kvm_memslot_iter_start(struct kvm_memslot_iter *iter,

					  struct kvm_memslots *slots,

					  gfn_t start)

{

	int idx = slots->node_idx;

	struct rb_node *tmp;

	struct kvm_memory_slot *slot;



	iter->slots = slots;



	/*

	 * Find the so called "upper bound" of a key - the first node that has

	 * its key strictly greater than the searched one (the start gfn in our case).

	 */

	iter->node = NULL;

	for (tmp = slots->gfn_tree.rb_node; tmp; ) {

		slot = container_of(tmp, struct kvm_memory_slot, gfn_node[idx]);

		if (start < slot->base_gfn) {

			iter->node = tmp;

			tmp = tmp->rb_left;

		} else {

			tmp = tmp->rb_right;

		}

	}



	/*

	 * Find the slot with the lowest gfn that can possibly intersect with

	 * the range, so we'll ideally have slot start <= range start

	 */

	if (iter->node) {

		/*

		 * A NULL previous node means that the very first slot

		 * already has a higher start gfn.

		 * In this case slot start > range start.

		 */

		tmp = rb_prev(iter->node);

		if (tmp)

			iter->node = tmp;

	} else {

		/* a NULL node below means no slots */

		iter->node = rb_last(&slots->gfn_tree);

	}



	if (iter->node) {

		iter->slot = container_of(iter->node, struct kvm_memory_slot, gfn_node[idx]);



		/*

		 * It is possible in the slot start < range start case that the

		 * found slot ends before or at range start (slot end <= range start)

		 * and so it does not overlap the requested range.

		 *

		 * In such non-overlapping case the next slot (if it exists) will

		 * already have slot start > range start, otherwise the logic above

		 * would have found it instead of the current slot.

		 */

		if (iter->slot->base_gfn + iter->slot->npages <= start)

			kvm_memslot_iter_next(iter);

	}

}



static inline bool kvm_memslot_iter_is_valid(struct kvm_memslot_iter *iter, gfn_t end)

{

	if (!iter->node)

		return false;



	/*

	 * If this slot starts beyond or at the end of the range so does

	 * every next one

	 */

	return iter->slot->base_gfn < end;

}



/* Iterate over each memslot at least partially intersecting [start, end) range */

#define kvm_for_each_memslot_in_gfn_range(iter, slots, start, end)	\

	for (kvm_memslot_iter_start(iter, slots, start);		\

	     kvm_memslot_iter_is_valid(iter, end);			\

	     kvm_memslot_iter_next(iter))



/*

 * KVM_SET_USER_MEMORY_REGION ioctl allows the following operations:

 * - create a new memory slot