Commit 56ae0bb3 authored by Kefeng Wang's avatar Kefeng Wang Committed by Andrew Morton
Browse files

mm: compaction: convert to use a folio in isolate_migratepages_block() 

Directly use a folio instead of page_folio() when page successfully
isolated (hugepage and movable page) and after folio_get_nontail_page(),
which removes several calls to compound_head().

Link: https://lkml.kernel.org/r/20230619110718.65679-1-wangkefeng.wang@huawei.com


Signed-off-by: default avatarKefeng Wang <wangkefeng.wang@huawei.com>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: James Gowans <jgowans@amazon.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 18a93707

mm/compaction.c

+44 −40
Original line number Original line Diff line number Diff line
@@ -822,6 +822,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
	struct lruvec *lruvec;

	struct lruvec *lruvec;

	unsigned long flags = 0;

	unsigned long flags = 0;

	struct lruvec *locked = NULL;

	struct lruvec *locked = NULL;

	struct folio *folio = NULL;

	struct page *page = NULL, *valid_page = NULL;

	struct page *page = NULL, *valid_page = NULL;

	struct address_space *mapping;

	struct address_space *mapping;

	unsigned long start_pfn = low_pfn;

	unsigned long start_pfn = low_pfn;
@@ -918,7 +919,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
		if (!valid_page && pageblock_aligned(low_pfn)) {

		if (!valid_page && pageblock_aligned(low_pfn)) {

			if (!isolation_suitable(cc, page)) {

			if (!isolation_suitable(cc, page)) {

				low_pfn = end_pfn;

				low_pfn = end_pfn;

				page = NULL;

				folio = NULL;

				goto isolate_abort;

				goto isolate_abort;

			}

			}

			valid_page = page;

			valid_page = page;
@@ -950,7 +951,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
				 * Hugepage was successfully isolated and placed

				 * Hugepage was successfully isolated and placed

				 * on the cc->migratepages list.

				 * on the cc->migratepages list.

				 */

				 */

				low_pfn += compound_nr(page) - 1;

				folio = page_folio(page);

				low_pfn += folio_nr_pages(folio) - 1;

				goto isolate_success_no_list;

				goto isolate_success_no_list;

			}

			}
@@ -1018,9 +1020,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
					locked = NULL;

					locked = NULL;

				}

				}





				if (isolate_movable_page(page, mode))

				if (isolate_movable_page(page, mode)) {

					folio = page_folio(page);

					goto isolate_success;

					goto isolate_success;

				}

				}

			}





			goto isolate_fail;

			goto isolate_fail;

		}

		}
@@ -1030,7 +1034,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
		 * sure the page is not being freed elsewhere -- the

		 * sure the page is not being freed elsewhere -- the

		 * page release code relies on it.

		 * page release code relies on it.

		 */

		 */

		if (unlikely(!get_page_unless_zero(page)))

		folio = folio_get_nontail_page(page);

		if (unlikely(!folio))

			goto isolate_fail;

			goto isolate_fail;





		/*

		/*
@@ -1038,8 +1043,8 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
		 * so avoid taking lru_lock and isolating it unnecessarily in an

		 * so avoid taking lru_lock and isolating it unnecessarily in an

		 * admittedly racy check.

		 * admittedly racy check.

		 */

		 */

		mapping = page_mapping(page);

		mapping = folio_mapping(folio);

		if (!mapping && (page_count(page) - 1) > total_mapcount(page))

		if (!mapping && (folio_ref_count(folio) - 1) > folio_mapcount(folio))

			goto isolate_fail_put;

			goto isolate_fail_put;





		/*

		/*
@@ -1050,11 +1055,11 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
			goto isolate_fail_put;

			goto isolate_fail_put;





		/* Only take pages on LRU: a check now makes later tests safe */

		/* Only take pages on LRU: a check now makes later tests safe */

		if (!PageLRU(page))

		if (!folio_test_lru(folio))

			goto isolate_fail_put;

			goto isolate_fail_put;





		/* Compaction might skip unevictable pages but CMA takes them */

		/* Compaction might skip unevictable pages but CMA takes them */

		if (!(mode & ISOLATE_UNEVICTABLE) && PageUnevictable(page))

		if (!(mode & ISOLATE_UNEVICTABLE) && folio_test_unevictable(folio))

			goto isolate_fail_put;

			goto isolate_fail_put;





		/*

		/*
@@ -1063,10 +1068,10 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
		 * it will be able to migrate without blocking - clean pages

		 * it will be able to migrate without blocking - clean pages

		 * for the most part.  PageWriteback would require blocking.

		 * for the most part.  PageWriteback would require blocking.

		 */

		 */

		if ((mode & ISOLATE_ASYNC_MIGRATE) && PageWriteback(page))

		if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_writeback(folio))

			goto isolate_fail_put;

			goto isolate_fail_put;





		if ((mode & ISOLATE_ASYNC_MIGRATE) && PageDirty(page)) {

		if ((mode & ISOLATE_ASYNC_MIGRATE) && folio_test_dirty(folio)) {

			bool migrate_dirty;

			bool migrate_dirty;





			/*

			/*
@@ -1078,22 +1083,22 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
			 * the page lock until after the page is removed

			 * the page lock until after the page is removed

			 * from the page cache.

			 * from the page cache.

			 */

			 */

			if (!trylock_page(page))

			if (!folio_trylock(folio))

				goto isolate_fail_put;

				goto isolate_fail_put;





			mapping = page_mapping(page);

			mapping = folio_mapping(folio);

			migrate_dirty = !mapping ||

			migrate_dirty = !mapping ||

					mapping->a_ops->migrate_folio;

					mapping->a_ops->migrate_folio;

			unlock_page(page);

			folio_unlock(folio);

			if (!migrate_dirty)

			if (!migrate_dirty)

				goto isolate_fail_put;

				goto isolate_fail_put;

		}

		}





		/* Try isolate the page */

		/* Try isolate the folio */

		if (!TestClearPageLRU(page))

		if (!folio_test_clear_lru(folio))

			goto isolate_fail_put;

			goto isolate_fail_put;





		lruvec = folio_lruvec(page_folio(page));

		lruvec = folio_lruvec(folio);





		/* If we already hold the lock, we can skip some rechecking */

		/* If we already hold the lock, we can skip some rechecking */

		if (lruvec != locked) {

		if (lruvec != locked) {
@@ -1103,7 +1108,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
			compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);

			compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);

			locked = lruvec;

			locked = lruvec;





			lruvec_memcg_debug(lruvec, page_folio(page));

			lruvec_memcg_debug(lruvec, folio);





			/*

			/*

			 * Try get exclusive access under lock. If marked for

			 * Try get exclusive access under lock. If marked for
@@ -1119,34 +1124,33 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
			}

			}





			/*

			/*

			 * Page become compound since the non-locked check,

			 * folio become large since the non-locked check,

			 * and it's on LRU. It can only be a THP so the order

			 * and it's on LRU.

			 * is safe to read and it's 0 for tail pages.

			 */

			 */

			if (unlikely(PageCompound(page) && !cc->alloc_contig)) {

			if (unlikely(folio_test_large(folio) && !cc->alloc_contig)) {

				low_pfn += compound_nr(page) - 1;

				low_pfn += folio_nr_pages(folio) - 1;

				nr_scanned += compound_nr(page) - 1;

				nr_scanned += folio_nr_pages(folio) - 1;

				SetPageLRU(page);

				folio_set_lru(folio);

				goto isolate_fail_put;

				goto isolate_fail_put;

			}

			}

		}

		}





		/* The whole page is taken off the LRU; skip the tail pages. */

		/* The folio is taken off the LRU */

		if (PageCompound(page))

		if (folio_test_large(folio))

			low_pfn += compound_nr(page) - 1;

			low_pfn += folio_nr_pages(folio) - 1;





		/* Successfully isolated */

		/* Successfully isolated */

		del_page_from_lru_list(page, lruvec);

		lruvec_del_folio(lruvec, folio);

		mod_node_page_state(page_pgdat(page),

		node_stat_mod_folio(folio,

				NR_ISOLATED_ANON + page_is_file_lru(page),

				NR_ISOLATED_ANON + folio_is_file_lru(folio),

				thp_nr_pages(page));

				folio_nr_pages(folio));





isolate_success:

isolate_success:

		list_add(&page->lru, &cc->migratepages);

		list_add(&folio->lru, &cc->migratepages);

isolate_success_no_list:

isolate_success_no_list:

		cc->nr_migratepages += compound_nr(page);

		cc->nr_migratepages += folio_nr_pages(folio);

		nr_isolated += compound_nr(page);

		nr_isolated += folio_nr_pages(folio);

		nr_scanned += compound_nr(page) - 1;

		nr_scanned += folio_nr_pages(folio) - 1;





		/*

		/*

		 * Avoid isolating too much unless this block is being

		 * Avoid isolating too much unless this block is being
@@ -1168,7 +1172,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
			unlock_page_lruvec_irqrestore(locked, flags);

			unlock_page_lruvec_irqrestore(locked, flags);

			locked = NULL;

			locked = NULL;

		}

		}

		put_page(page);

		folio_put(folio);





isolate_fail:

isolate_fail:

		if (!skip_on_failure && ret != -ENOMEM)

		if (!skip_on_failure && ret != -ENOMEM)
@@ -1209,14 +1213,14 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, 
	if (unlikely(low_pfn > end_pfn))

	if (unlikely(low_pfn > end_pfn))

		low_pfn = end_pfn;

		low_pfn = end_pfn;





	page = NULL;

	folio = NULL;





isolate_abort:

isolate_abort:

	if (locked)

	if (locked)

		unlock_page_lruvec_irqrestore(locked, flags);

		unlock_page_lruvec_irqrestore(locked, flags);

	if (page) {

	if (folio) {

		SetPageLRU(page);

		folio_set_lru(folio);

		put_page(page);

		folio_put(folio);

	}

	}





	/*

	/*