Commit 9f32624b authored by Joonsoo Kim's avatar Joonsoo Kim Committed by Linus Torvalds
Browse files

mm/rmap: use rmap_walk() in page_referenced() 



Now, we have an infrastructure in rmap_walk() to handle difference from
variants of rmap traversing functions.

So, just use it in page_referenced().

In this patch, I change following things.

1. remove some variants of rmap traversing functions.
	cf> page_referenced_ksm, page_referenced_anon,
	page_referenced_file

2. introduce new struct page_referenced_arg and pass it to
   page_referenced_one(), main function of rmap_walk, in order to count
   reference, to store vm_flags and to check finish condition.

3. mechanical change to use rmap_walk() in page_referenced().

[liwanp@linux.vnet.ibm.com: fix BUG at rmap_walk]
Signed-off-by: default avatarJoonsoo Kim <iamjoonsoo.kim@lge.com>
Reviewed-by: default avatarNaoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Hillf Danton <dhillf@gmail.com>
Signed-off-by: default avatarWanpeng Li <liwanp@linux.vnet.ibm.com>
Cc: Sasha Levin <sasha.levin@oracle.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent e8351ac9

include/linux/ksm.h

+0 −2
Original line number Original line Diff line number Diff line
@@ -73,8 +73,6 @@ static inline void set_page_stable_node(struct page *page, 
struct page *ksm_might_need_to_copy(struct page *page,

struct page *ksm_might_need_to_copy(struct page *page,

			struct vm_area_struct *vma, unsigned long address);

			struct vm_area_struct *vma, unsigned long address);





int page_referenced_ksm(struct page *page,

			struct mem_cgroup *memcg, unsigned long *vm_flags);

int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);

int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);

void ksm_migrate_page(struct page *newpage, struct page *oldpage);

void ksm_migrate_page(struct page *newpage, struct page *oldpage);

include/linux/rmap.h

+1 −1
Original line number Original line Diff line number Diff line
@@ -184,7 +184,7 @@ static inline void page_dup_rmap(struct page *page) 
int page_referenced(struct page *, int is_locked,

int page_referenced(struct page *, int is_locked,

			struct mem_cgroup *memcg, unsigned long *vm_flags);

			struct mem_cgroup *memcg, unsigned long *vm_flags);

int page_referenced_one(struct page *, struct vm_area_struct *,

int page_referenced_one(struct page *, struct vm_area_struct *,

	unsigned long address, unsigned int *mapcount, unsigned long *vm_flags);

	unsigned long address, void *arg);





#define TTU_ACTION(x) ((x) & TTU_ACTION_MASK)

#define TTU_ACTION(x) ((x) & TTU_ACTION_MASK)

mm/ksm.c

+5 −55
Original line number Original line Diff line number Diff line
@@ -1891,69 +1891,19 @@ struct page *ksm_might_need_to_copy(struct page *page, 
	return new_page;

	return new_page;

}

}





int page_referenced_ksm(struct page *page, struct mem_cgroup *memcg,

int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)

			unsigned long *vm_flags)

{

{

	struct stable_node *stable_node;

	struct stable_node *stable_node;

	struct rmap_item *rmap_item;

	struct rmap_item *rmap_item;

	unsigned int mapcount = page_mapcount(page);

	int ret = SWAP_AGAIN;

	int referenced = 0;

	int search_new_forks = 0;

	int search_new_forks = 0;





	VM_BUG_ON(!PageKsm(page));

	VM_BUG_ON(!PageKsm(page));

	VM_BUG_ON(!PageLocked(page));



	stable_node = page_stable_node(page);

	if (!stable_node)

		return 0;

again:

	hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {

		struct anon_vma *anon_vma = rmap_item->anon_vma;

		struct anon_vma_chain *vmac;

		struct vm_area_struct *vma;





		anon_vma_lock_read(anon_vma);

		anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,

					       0, ULONG_MAX) {

			vma = vmac->vma;

			if (rmap_item->address < vma->vm_start ||

			    rmap_item->address >= vma->vm_end)

				continue;

	/*

	/*

			 * Initially we examine only the vma which covers this

	 * Rely on the page lock to protect against concurrent modifications

			 * rmap_item; but later, if there is still work to do,

	 * to that page's node of the stable tree.

			 * we examine covering vmas in other mms: in case they

			 * were forked from the original since ksmd passed.

	 */

	 */

			if ((rmap_item->mm == vma->vm_mm) == search_new_forks)

				continue;



			if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))

				continue;



			referenced += page_referenced_one(page, vma,

				rmap_item->address, &mapcount, vm_flags);

			if (!search_new_forks || !mapcount)

				break;

		}

		anon_vma_unlock_read(anon_vma);

		if (!mapcount)

			goto out;

	}

	if (!search_new_forks++)

		goto again;

out:

	return referenced;

}



int rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc)

{

	struct stable_node *stable_node;

	struct rmap_item *rmap_item;

	int ret = SWAP_AGAIN;

	int search_new_forks = 0;



	VM_BUG_ON(!PageKsm(page));

	VM_BUG_ON(!PageLocked(page));

	VM_BUG_ON(!PageLocked(page));





	stable_node = page_stable_node(page);

	stable_node = page_stable_node(page);

mm/rmap.c

+74 −136
Original line number Original line Diff line number Diff line
@@ -660,17 +660,22 @@ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) 
	return 1;

	return 1;

}

}





struct page_referenced_arg {

	int mapcount;

	int referenced;

	unsigned long vm_flags;

	struct mem_cgroup *memcg;

};

/*

/*

 * Subfunctions of page_referenced: page_referenced_one called

 * arg: page_referenced_arg will be passed

 * repeatedly from either page_referenced_anon or page_referenced_file.

 */

 */

int page_referenced_one(struct page *page, struct vm_area_struct *vma,

int page_referenced_one(struct page *page, struct vm_area_struct *vma,

			unsigned long address, unsigned int *mapcount,

			unsigned long address, void *arg)

			unsigned long *vm_flags)

{

{

	struct mm_struct *mm = vma->vm_mm;

	struct mm_struct *mm = vma->vm_mm;

	spinlock_t *ptl;

	spinlock_t *ptl;

	int referenced = 0;

	int referenced = 0;

	struct page_referenced_arg *pra = arg;





	if (unlikely(PageTransHuge(page))) {

	if (unlikely(PageTransHuge(page))) {

		pmd_t *pmd;

		pmd_t *pmd;
@@ -682,13 +687,12 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma, 
		pmd = page_check_address_pmd(page, mm, address,

		pmd = page_check_address_pmd(page, mm, address,

					     PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl);

					     PAGE_CHECK_ADDRESS_PMD_FLAG, &ptl);

		if (!pmd)

		if (!pmd)

			goto out;

			return SWAP_AGAIN;





		if (vma->vm_flags & VM_LOCKED) {

		if (vma->vm_flags & VM_LOCKED) {

			spin_unlock(ptl);

			spin_unlock(ptl);

			*mapcount = 0;	/* break early from loop */

			pra->vm_flags |= VM_LOCKED;

			*vm_flags |= VM_LOCKED;

			return SWAP_FAIL; /* To break the loop */

			goto out;

		}

		}





		/* go ahead even if the pmd is pmd_trans_splitting() */

		/* go ahead even if the pmd is pmd_trans_splitting() */
@@ -704,13 +708,12 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma, 
		 */

		 */

		pte = page_check_address(page, mm, address, &ptl, 0);

		pte = page_check_address(page, mm, address, &ptl, 0);

		if (!pte)

		if (!pte)

			goto out;

			return SWAP_AGAIN;





		if (vma->vm_flags & VM_LOCKED) {

		if (vma->vm_flags & VM_LOCKED) {

			pte_unmap_unlock(pte, ptl);

			pte_unmap_unlock(pte, ptl);

			*mapcount = 0;	/* break early from loop */

			pra->vm_flags |= VM_LOCKED;

			*vm_flags |= VM_LOCKED;

			return SWAP_FAIL; /* To break the loop */

			goto out;

		}

		}





		if (ptep_clear_flush_young_notify(vma, address, pte)) {

		if (ptep_clear_flush_young_notify(vma, address, pte)) {
@@ -727,113 +730,27 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma, 
		pte_unmap_unlock(pte, ptl);

		pte_unmap_unlock(pte, ptl);

	}

	}





	(*mapcount)--;

	if (referenced) {



		pra->referenced++;

	if (referenced)

		pra->vm_flags |= vma->vm_flags;

		*vm_flags |= vma->vm_flags;

out:

	return referenced;

	}

	}





static int page_referenced_anon(struct page *page,

	pra->mapcount--;

				struct mem_cgroup *memcg,

	if (!pra->mapcount)

				unsigned long *vm_flags)

		return SWAP_SUCCESS; /* To break the loop */

{

	unsigned int mapcount;

	struct anon_vma *anon_vma;

	pgoff_t pgoff;

	struct anon_vma_chain *avc;

	int referenced = 0;



	anon_vma = page_lock_anon_vma_read(page);

	if (!anon_vma)

		return referenced;



	mapcount = page_mapcount(page);

	pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);

	anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) {

		struct vm_area_struct *vma = avc->vma;

		unsigned long address = vma_address(page, vma);

		/*

		 * If we are reclaiming on behalf of a cgroup, skip

		 * counting on behalf of references from different

		 * cgroups

		 */

		if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))

			continue;

		referenced += page_referenced_one(page, vma, address,

						  &mapcount, vm_flags);

		if (!mapcount)

			break;

	}





	page_unlock_anon_vma_read(anon_vma);

	return SWAP_AGAIN;

	return referenced;

}

}





/**

static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg)

 * page_referenced_file - referenced check for object-based rmap

 * @page: the page we're checking references on.

 * @memcg: target memory control group

 * @vm_flags: collect encountered vma->vm_flags who actually referenced the page

 *

 * For an object-based mapped page, find all the places it is mapped and

 * check/clear the referenced flag.  This is done by following the page->mapping

 * pointer, then walking the chain of vmas it holds.  It returns the number

 * of references it found.

 *

 * This function is only called from page_referenced for object-based pages.

 */

static int page_referenced_file(struct page *page,

				struct mem_cgroup *memcg,

				unsigned long *vm_flags)

{

{

	unsigned int mapcount;

	struct page_referenced_arg *pra = arg;

	struct address_space *mapping = page->mapping;

	struct mem_cgroup *memcg = pra->memcg;

	pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);

	struct vm_area_struct *vma;

	int referenced = 0;



	/*

	 * The caller's checks on page->mapping and !PageAnon have made

	 * sure that this is a file page: the check for page->mapping

	 * excludes the case just before it gets set on an anon page.

	 */

	BUG_ON(PageAnon(page));



	/*

	 * The page lock not only makes sure that page->mapping cannot

	 * suddenly be NULLified by truncation, it makes sure that the

	 * structure at mapping cannot be freed and reused yet,

	 * so we can safely take mapping->i_mmap_mutex.

	 */

	BUG_ON(!PageLocked(page));





	mutex_lock(&mapping->i_mmap_mutex);

	if (!mm_match_cgroup(vma->vm_mm, memcg))



		return true;

	/*

	 * i_mmap_mutex does not stabilize mapcount at all, but mapcount

	 * is more likely to be accurate if we note it after spinning.

	 */

	mapcount = page_mapcount(page);



	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {

		unsigned long address = vma_address(page, vma);

		/*

		 * If we are reclaiming on behalf of a cgroup, skip

		 * counting on behalf of references from different

		 * cgroups

		 */

		if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))

			continue;

		referenced += page_referenced_one(page, vma, address,

						  &mapcount, vm_flags);

		if (!mapcount)

			break;

	}





	mutex_unlock(&mapping->i_mmap_mutex);

	return false;

	return referenced;

}

}





/**

/**
@@ -851,32 +768,47 @@ int page_referenced(struct page *page, 
		    struct mem_cgroup *memcg,

		    struct mem_cgroup *memcg,

		    unsigned long *vm_flags)

		    unsigned long *vm_flags)

{

{

	int referenced = 0;

	int ret;

	int we_locked = 0;

	int we_locked = 0;

	struct page_referenced_arg pra = {

		.mapcount = page_mapcount(page),

		.memcg = memcg,

	};

	struct rmap_walk_control rwc = {

		.rmap_one = page_referenced_one,

		.arg = (void *)&pra,

		.anon_lock = page_lock_anon_vma_read,

	};





	*vm_flags = 0;

	*vm_flags = 0;

	if (page_mapped(page) && page_rmapping(page)) {

	if (!page_mapped(page))

		return 0;



	if (!page_rmapping(page))

		return 0;



	if (!is_locked && (!PageAnon(page) || PageKsm(page))) {

	if (!is_locked && (!PageAnon(page) || PageKsm(page))) {

		we_locked = trylock_page(page);

		we_locked = trylock_page(page);

			if (!we_locked) {

		if (!we_locked)

				referenced++;

			return 1;

				goto out;

	}

	}



	/*

	 * If we are reclaiming on behalf of a cgroup, skip

	 * counting on behalf of references from different

	 * cgroups

	 */

	if (memcg) {

		rwc.invalid_vma = invalid_page_referenced_vma;

	}

	}

		if (unlikely(PageKsm(page)))



			referenced += page_referenced_ksm(page, memcg,

	ret = rmap_walk(page, &rwc);

								vm_flags);

	*vm_flags = pra.vm_flags;

		else if (PageAnon(page))



			referenced += page_referenced_anon(page, memcg,

								vm_flags);

		else if (page->mapping)

			referenced += page_referenced_file(page, memcg,

								vm_flags);

	if (we_locked)

	if (we_locked)

		unlock_page(page);

		unlock_page(page);

	}



out:

	return pra.referenced;

	return referenced;

}

}





static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,

static int page_mkclean_one(struct page *page, struct vm_area_struct *vma,
@@ -1700,6 +1632,14 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc) 
	struct vm_area_struct *vma;

	struct vm_area_struct *vma;

	int ret = SWAP_AGAIN;

	int ret = SWAP_AGAIN;





	/*

	 * The page lock not only makes sure that page->mapping cannot

	 * suddenly be NULLified by truncation, it makes sure that the

	 * structure at mapping cannot be freed and reused yet,

	 * so we can safely take mapping->i_mmap_mutex.

	 */

	VM_BUG_ON(!PageLocked(page));



	if (!mapping)

	if (!mapping)

		return ret;

		return ret;

	mutex_lock(&mapping->i_mmap_mutex);

	mutex_lock(&mapping->i_mmap_mutex);
@@ -1731,8 +1671,6 @@ static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc) 




int rmap_walk(struct page *page, struct rmap_walk_control *rwc)

int rmap_walk(struct page *page, struct rmap_walk_control *rwc)

{

{

	VM_BUG_ON(!PageLocked(page));



	if (unlikely(PageKsm(page)))

	if (unlikely(PageKsm(page)))

		return rmap_walk_ksm(page, rwc);

		return rmap_walk_ksm(page, rwc);

	else if (PageAnon(page))

	else if (PageAnon(page))