Commit 19138349 authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle)
Browse files

mm/migrate: Add folio_migrate_flags() 



Turn migrate_page_states() into a wrapper around folio_migrate_flags().
Also convert two functions only called from folio_migrate_flags() to
be folio-based.  ksm_migrate_page() becomes folio_migrate_ksm() and
copy_page_owner() becomes folio_copy_owner().  folio_migrate_flags()
alone shrinks by two thirds -- 1967 bytes down to 642 bytes.

Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: default avatarZi Yan <ziy@nvidia.com>
Reviewed-by: default avatarDavid Howells <dhowells@redhat.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent 3417013e

include/linux/ksm.h

+2 −2
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ struct page *ksm_might_need_to_copy(struct page *page, 
			struct vm_area_struct *vma, unsigned long address);



void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);

void ksm_migrate_page(struct page *newpage, struct page *oldpage);

void folio_migrate_ksm(struct folio *newfolio, struct folio *folio);



#else  /* !CONFIG_KSM */
@@ -83,7 +83,7 @@ static inline void rmap_walk_ksm(struct page *page, 
{

}



static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)

static inline void folio_migrate_ksm(struct folio *newfolio, struct folio *old)

{

}

#endif /* CONFIG_MMU */

include/linux/migrate.h

+1 −0
Original line number Diff line number Diff line
@@ -57,6 +57,7 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping, 
				  struct page *newpage, struct page *page);

extern int migrate_page_move_mapping(struct address_space *mapping,

		struct page *newpage, struct page *page, int extra_count);

void folio_migrate_flags(struct folio *newfolio, struct folio *folio);

int folio_migrate_mapping(struct address_space *mapping,

		struct folio *newfolio, struct folio *folio, int extra_count);

#else

include/linux/page_owner.h

+4 −4
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ extern void __reset_page_owner(struct page *page, unsigned int order); 
extern void __set_page_owner(struct page *page,

			unsigned int order, gfp_t gfp_mask);

extern void __split_page_owner(struct page *page, unsigned int nr);

extern void __copy_page_owner(struct page *oldpage, struct page *newpage);

extern void __folio_copy_owner(struct folio *newfolio, struct folio *old);

extern void __set_page_owner_migrate_reason(struct page *page, int reason);

extern void __dump_page_owner(const struct page *page);

extern void pagetypeinfo_showmixedcount_print(struct seq_file *m,
@@ -36,10 +36,10 @@ static inline void split_page_owner(struct page *page, unsigned int nr) 
	if (static_branch_unlikely(&page_owner_inited))

		__split_page_owner(page, nr);

}

static inline void copy_page_owner(struct page *oldpage, struct page *newpage)

static inline void folio_copy_owner(struct folio *newfolio, struct folio *old)

{

	if (static_branch_unlikely(&page_owner_inited))

		__copy_page_owner(oldpage, newpage);

		__folio_copy_owner(newfolio, old);

}

static inline void set_page_owner_migrate_reason(struct page *page, int reason)

{
@@ -63,7 +63,7 @@ static inline void split_page_owner(struct page *page, 
			unsigned int order)

{

}

static inline void copy_page_owner(struct page *oldpage, struct page *newpage)

static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio)

{

}

static inline void set_page_owner_migrate_reason(struct page *page, int reason)

mm/folio-compat.c

+6 −0
Original line number Diff line number Diff line
@@ -58,4 +58,10 @@ int migrate_page_move_mapping(struct address_space *mapping, 
					page_folio(page), extra_count);

}

EXPORT_SYMBOL(migrate_page_move_mapping);



void migrate_page_states(struct page *newpage, struct page *page)

{

	folio_migrate_flags(page_folio(newpage), page_folio(page));

}

EXPORT_SYMBOL(migrate_page_states);

#endif

mm/ksm.c

+18 −13
Original line number Diff line number Diff line
@@ -751,7 +751,7 @@ static struct page *get_ksm_page(struct stable_node *stable_node, 
	/*

	 * We come here from above when page->mapping or !PageSwapCache

	 * suggests that the node is stale; but it might be under migration.

	 * We need smp_rmb(), matching the smp_wmb() in ksm_migrate_page(),

	 * We need smp_rmb(), matching the smp_wmb() in folio_migrate_ksm(),

	 * before checking whether node->kpfn has been changed.

	 */

	smp_rmb();
@@ -852,9 +852,14 @@ static int unmerge_ksm_pages(struct vm_area_struct *vma, 
	return err;

}



static inline struct stable_node *folio_stable_node(struct folio *folio)

{

	return folio_test_ksm(folio) ? folio_raw_mapping(folio) : NULL;

}



static inline struct stable_node *page_stable_node(struct page *page)

{

	return PageKsm(page) ? page_rmapping(page) : NULL;

	return folio_stable_node(page_folio(page));

}



static inline void set_page_stable_node(struct page *page,
@@ -2659,26 +2664,26 @@ void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc) 
}



#ifdef CONFIG_MIGRATION

void ksm_migrate_page(struct page *newpage, struct page *oldpage)

void folio_migrate_ksm(struct folio *newfolio, struct folio *folio)

{

	struct stable_node *stable_node;



	VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);

	VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);

	VM_BUG_ON_PAGE(newpage->mapping != oldpage->mapping, newpage);

	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);

	VM_BUG_ON_FOLIO(!folio_test_locked(newfolio), newfolio);

	VM_BUG_ON_FOLIO(newfolio->mapping != folio->mapping, newfolio);



	stable_node = page_stable_node(newpage);

	stable_node = folio_stable_node(folio);

	if (stable_node) {

		VM_BUG_ON_PAGE(stable_node->kpfn != page_to_pfn(oldpage), oldpage);

		stable_node->kpfn = page_to_pfn(newpage);

		VM_BUG_ON_FOLIO(stable_node->kpfn != folio_pfn(folio), folio);

		stable_node->kpfn = folio_pfn(newfolio);

		/*

		 * newpage->mapping was set in advance; now we need smp_wmb()

		 * newfolio->mapping was set in advance; now we need smp_wmb()

		 * to make sure that the new stable_node->kpfn is visible

		 * to get_ksm_page() before it can see that oldpage->mapping

		 * has gone stale (or that PageSwapCache has been cleared).

		 * to get_ksm_page() before it can see that folio->mapping

		 * has gone stale (or that folio_test_swapcache has been cleared).

		 */

		smp_wmb();

		set_page_stable_node(oldpage, NULL);

		set_page_stable_node(&folio->page, NULL);

	}

}

#endif /* CONFIG_MIGRATION */