highmem: High implementation details and document API (13f876ba) · Commits · jan.koester / Linux

include/linux/highmem-internal.h

0 → 100644

+174 −0

Original line number	Original line	Diff line number	Diff line
			/* SPDX-License-Identifier: GPL-2.0 */
			#ifndef _LINUX_HIGHMEM_INTERNAL_H
			#define _LINUX_HIGHMEM_INTERNAL_H

			/*
			* Outside of CONFIG_HIGHMEM to support X86 32bit iomap_atomic() cruft.
			*/
			#ifdef CONFIG_KMAP_LOCAL
			void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot);
			void __kmap_local_page_prot(struct page page, pgprot_t prot);
			void kunmap_local_indexed(void *vaddr);
			#endif

			#ifdef CONFIG_HIGHMEM
			#include <asm/highmem.h>

			#ifndef ARCH_HAS_KMAP_FLUSH_TLB
			static inline void kmap_flush_tlb(unsigned long addr) { }
			#endif

			#ifndef kmap_prot
			#define kmap_prot PAGE_KERNEL
			#endif

			void kmap_high(struct page page);
			void kunmap_high(struct page *page);
			void __kmap_flush_unused(void);
			struct page __kmap_to_page(void addr);

			static inline void kmap(struct page page)
			{
			void *addr;

			might_sleep();
			if (!PageHighMem(page))
			addr = page_address(page);
			else
			addr = kmap_high(page);
			kmap_flush_tlb((unsigned long)addr);
			return addr;
			}

			static inline void kunmap(struct page *page)
			{
			might_sleep();
			if (!PageHighMem(page))
			return;
			kunmap_high(page);
			}

			static inline struct page kmap_to_page(void addr)
			{
			return __kmap_to_page(addr);
			}

			static inline void kmap_flush_unused(void)
			{
			__kmap_flush_unused();
			}

			static inline void kmap_atomic_prot(struct page page, pgprot_t prot)
			{
			preempt_disable();
			pagefault_disable();
			return __kmap_local_page_prot(page, prot);
			}

			static inline void kmap_atomic(struct page page)
			{
			return kmap_atomic_prot(page, kmap_prot);
			}

			static inline void *kmap_atomic_pfn(unsigned long pfn)
			{
			preempt_disable();
			pagefault_disable();
			return __kmap_local_pfn_prot(pfn, kmap_prot);
			}

			static inline void __kunmap_atomic(void *addr)
			{
			kunmap_local_indexed(addr);
			pagefault_enable();
			preempt_enable();
			}

			unsigned int __nr_free_highpages(void);
			extern atomic_long_t _totalhigh_pages;

			static inline unsigned int nr_free_highpages(void)
			{
			return __nr_free_highpages();
			}

			static inline unsigned long totalhigh_pages(void)
			{
			return (unsigned long)atomic_long_read(&_totalhigh_pages);
			}

			static inline void totalhigh_pages_inc(void)
			{
			atomic_long_inc(&_totalhigh_pages);
			}

			static inline void totalhigh_pages_add(long count)
			{
			atomic_long_add(count, &_totalhigh_pages);
			}

			#else /* CONFIG_HIGHMEM */

			static inline struct page kmap_to_page(void addr)
			{
			return virt_to_page(addr);
			}

			static inline void kmap(struct page page)
			{
			might_sleep();
			return page_address(page);
			}

			static inline void kunmap_high(struct page *page) { }
			static inline void kmap_flush_unused(void) { }

			static inline void kunmap(struct page *page)
			{
			#ifdef ARCH_HAS_FLUSH_ON_KUNMAP
			kunmap_flush_on_unmap(page_address(page));
			#endif
			}

			static inline void kmap_atomic(struct page page)
			{
			preempt_disable();
			pagefault_disable();
			return page_address(page);
			}

			static inline void kmap_atomic_prot(struct page page, pgprot_t prot)
			{
			return kmap_atomic(page);
			}

			static inline void *kmap_atomic_pfn(unsigned long pfn)
			{
			return kmap_atomic(pfn_to_page(pfn));
			}

			static inline void __kunmap_atomic(void *addr)
			{
			#ifdef ARCH_HAS_FLUSH_ON_KUNMAP
			kunmap_flush_on_unmap(addr);
			#endif
			pagefault_enable();
			preempt_enable();
			}

			static inline unsigned int nr_free_highpages(void) { return 0; }
			static inline unsigned long totalhigh_pages(void) { return 0UL; }

			#endif /* CONFIG_HIGHMEM */

			/*
			* Prevent people trying to call kunmap_atomic() as if it were kunmap()
			* kunmap_atomic() should get the return value of kmap_atomic, not the page.
			*/
			#define kunmap_atomic(__addr) \
			do { \
			BUILD_BUG_ON(__same_type((__addr), struct page *)); \
			__kunmap_atomic(__addr); \
			} while (0)

			#endif

include/linux/highmem.h

+96 −170

Original line number	Original line	Diff line number	Diff line
	@@ -11,199 +11,125 @@

	#include <asm/cacheflush.h>		#include <asm/cacheflush.h>

	#ifndef ARCH_HAS_FLUSH_ANON_PAGE		#include "highmem-internal.h"
	static inline void flush_anon_page(struct vm_area_struct vma, struct page page, unsigned long vmaddr)
	{
	}
	#endif

	#ifndef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
	static inline void flush_kernel_dcache_page(struct page *page)
	{
	}
	static inline void flush_kernel_vmap_range(void *vaddr, int size)
	{
	}
	static inline void invalidate_kernel_vmap_range(void *vaddr, int size)
	{
	}
	#endif

	/*		/**
	* Outside of CONFIG_HIGHMEM to support X86 32bit iomap_atomic() cruft.		* kmap - Map a page for long term usage
			* @page: Pointer to the page to be mapped
			*
			* Returns: The virtual address of the mapping
			*
			* Can only be invoked from preemptible task context because on 32bit
			* systems with CONFIG_HIGHMEM enabled this function might sleep.
			*
			* For systems with CONFIG_HIGHMEM=n and for pages in the low memory area
			* this returns the virtual address of the direct kernel mapping.
			*
			* The returned virtual address is globally visible and valid up to the
			* point where it is unmapped via kunmap(). The pointer can be handed to
			* other contexts.
			*
			* For highmem pages on 32bit systems this can be slow as the mapping space
			* is limited and protected by a global lock. In case that there is no
			* mapping slot available the function blocks until a slot is released via
			* kunmap().
	*/		*/
	#ifdef CONFIG_KMAP_LOCAL		static inline void kmap(struct page page);
	void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot);
	void __kmap_local_page_prot(struct page page, pgprot_t prot);
	void kunmap_local_indexed(void *vaddr);
	#endif

	#ifdef CONFIG_HIGHMEM
	#include <asm/highmem.h>

	#ifndef ARCH_HAS_KMAP_FLUSH_TLB
	static inline void kmap_flush_tlb(unsigned long addr) { }
	#endif

	#ifndef kmap_prot
	#define kmap_prot PAGE_KERNEL
	#endif

	void kmap_high(struct page page);		/**
	static inline void kmap(struct page page)		* kunmap - Unmap the virtual address mapped by kmap()
	{		* @addr: Virtual address to be unmapped
	void *addr;		*
			* Counterpart to kmap(). A NOOP for CONFIG_HIGHMEM=n and for mappings of
	might_sleep();		* pages in the low memory area.
	if (!PageHighMem(page))		*/
	addr = page_address(page);		static inline void kunmap(struct page *page);
	else
	addr = kmap_high(page);
	kmap_flush_tlb((unsigned long)addr);
	return addr;
	}

	void kunmap_high(struct page *page);		/**
			* kmap_to_page - Get the page for a kmap'ed address
			* @addr: The address to look up
			*
			* Returns: The page which is mapped to @addr.
			*/
			static inline struct page kmap_to_page(void addr);

	static inline void kunmap(struct page *page)		/**
	{		* kmap_flush_unused - Flush all unused kmap mappings in order to
	might_sleep();		* remove stray mappings
	if (!PageHighMem(page))		*/
	return;		static inline void kmap_flush_unused(void);
	kunmap_high(page);
	}

	/*		/**
	* kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because		* kmap_atomic - Atomically map a page for temporary usage
	* no global lock is needed and because the kmap code must perform a global TLB		* @page: Pointer to the page to be mapped
	* invalidation when the kmap pool wraps.		*
			* Returns: The virtual address of the mapping
			*
			* Side effect: On return pagefaults and preemption are disabled.
			*
			* Can be invoked from any context.
			*
			* Requires careful handling when nesting multiple mappings because the map
			* management is stack based. The unmap has to be in the reverse order of
			* the map operation:
	*		*
	* However when holding an atomic kmap it is not legal to sleep, so atomic		* addr1 = kmap_atomic(page1);
	* kmaps are appropriate for short, tight code paths only.		* addr2 = kmap_atomic(page2);
			* ...
			* kunmap_atomic(addr2);
			* kunmap_atomic(addr1);
	*		*
	* The use of kmap_atomic/kunmap_atomic is discouraged - kmap/kunmap		* Unmapping addr1 before addr2 is invalid and causes malfunction.
	* gives a more generic (and caching) interface. But kmap_atomic can		*
	* be used in IRQ contexts, so in some (very limited) cases we need		* Contrary to kmap() mappings the mapping is only valid in the context of
	* it.		* the caller and cannot be handed to other contexts.
			*
			* On CONFIG_HIGHMEM=n kernels and for low memory pages this returns the
			* virtual address of the direct mapping. Only real highmem pages are
			* temporarily mapped.
			*
			* While it is significantly faster than kmap() it comes with restrictions
			* about the pointer validity and the side effects of disabling page faults
			* and preemption. Use it only when absolutely necessary, e.g. from non
			* preemptible contexts.
	*/		*/
	static inline void kmap_atomic_prot(struct page page, pgprot_t prot)		static inline void kmap_atomic(struct page page);
	{
	preempt_disable();
	pagefault_disable();
	return __kmap_local_page_prot(page, prot);
	}

	static inline void kmap_atomic(struct page page)		/**
	{		* kunmap_atomic - Unmap the virtual address mapped by kmap_atomic()
	return kmap_atomic_prot(page, kmap_prot);		* @addr: Virtual address to be unmapped
	}		*
			* Counterpart to kmap_atomic().
	static inline void *kmap_atomic_pfn(unsigned long pfn)		*
	{		* Undoes the side effects of kmap_atomic(), i.e. reenabling pagefaults and
	preempt_disable();		* preemption.
	pagefault_disable();		*
	return __kmap_local_pfn_prot(pfn, kmap_prot);		* Other than that a NOOP for CONFIG_HIGHMEM=n and for mappings of pages
	}		* in the low memory area. For real highmen pages the mapping which was
			* established with kmap_atomic() is destroyed.
	static inline void __kunmap_atomic(void *addr)		*/
	{
	kunmap_local_indexed(addr);
	}

	/* declarations for linux/mm/highmem.c */
	unsigned int nr_free_highpages(void);
	extern atomic_long_t _totalhigh_pages;
	static inline unsigned long totalhigh_pages(void)
	{
	return (unsigned long)atomic_long_read(&_totalhigh_pages);
	}

	static inline void totalhigh_pages_inc(void)
	{
	atomic_long_inc(&_totalhigh_pages);
	}

	static inline void totalhigh_pages_add(long count)
	{
	atomic_long_add(count, &_totalhigh_pages);
	}

	void kmap_flush_unused(void);

	struct page kmap_to_page(void addr);

	#else /* CONFIG_HIGHMEM */

	static inline unsigned int nr_free_highpages(void) { return 0; }

	static inline struct page kmap_to_page(void addr)
	{
	return virt_to_page(addr);
	}

	static inline unsigned long totalhigh_pages(void) { return 0UL; }

	static inline void kmap(struct page page)		/* Highmem related interfaces for management code */
	{		static inline unsigned int nr_free_highpages(void);
	might_sleep();		static inline unsigned long totalhigh_pages(void);
	return page_address(page);
	}

	static inline void kunmap_high(struct page *page)		#ifndef ARCH_HAS_FLUSH_ANON_PAGE
			static inline void flush_anon_page(struct vm_area_struct vma, struct page page, unsigned long vmaddr)
	{		{
	}		}

	static inline void kunmap(struct page *page)
	{
	#ifdef ARCH_HAS_FLUSH_ON_KUNMAP
	kunmap_flush_on_unmap(page_address(page));
	#endif		#endif
	}

	static inline void kmap_atomic(struct page page)		#ifndef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
			static inline void flush_kernel_dcache_page(struct page *page)
	{		{
	preempt_disable();
	pagefault_disable();
	return page_address(page);
	}		}
			static inline void flush_kernel_vmap_range(void *vaddr, int size)
	static inline void kmap_atomic_prot(struct page page, pgprot_t prot)
	{		{
	return kmap_atomic(page);
	}		}
			static inline void invalidate_kernel_vmap_range(void *vaddr, int size)
	static inline void *kmap_atomic_pfn(unsigned long pfn)
	{		{
	return kmap_atomic(pfn_to_page(pfn));
	}		}

	static inline void __kunmap_atomic(void *addr)
	{
	/*
	* Mostly nothing to do in the CONFIG_HIGHMEM=n case as kunmap_atomic()
	* handles re-enabling faults and preemption
	*/
	#ifdef ARCH_HAS_FLUSH_ON_KUNMAP
	kunmap_flush_on_unmap(addr);
	#endif		#endif
	}

	#define kmap_flush_unused() do {} while(0)


	#endif /* CONFIG_HIGHMEM */

	/*
	* Prevent people trying to call kunmap_atomic() as if it were kunmap()
	* kunmap_atomic() should get the return value of kmap_atomic, not the page.
	*/
	#define kunmap_atomic(__addr) \
	do { \
	BUILD_BUG_ON(__same_type((__addr), struct page *)); \
	__kunmap_atomic(__addr); \
	pagefault_enable(); \
	preempt_enable(); \
	} while (0)

	/* when CONFIG_HIGHMEM is not set these will be plain clear/copy_page */		/* when CONFIG_HIGHMEM is not set these will be plain clear/copy_page */
	#ifndef clear_user_highpage		#ifndef clear_user_highpage

mm/highmem.c

+4 −7

Original line number	Original line	Diff line number	Diff line
	@@ -104,7 +104,7 @@ static inline wait_queue_head_t *get_pkmap_wait_queue_head(unsigned int color)
	atomic_long_t _totalhigh_pages __read_mostly;		atomic_long_t _totalhigh_pages __read_mostly;
	EXPORT_SYMBOL(_totalhigh_pages);		EXPORT_SYMBOL(_totalhigh_pages);

	unsigned int nr_free_highpages (void)		unsigned int __nr_free_highpages (void)
	{		{
	struct zone *zone;		struct zone *zone;
	unsigned int pages = 0;		unsigned int pages = 0;
	@@ -141,7 +141,7 @@ pte_t * pkmap_page_table;
	do { spin_unlock(&kmap_lock); (void)(flags); } while (0)		do { spin_unlock(&kmap_lock); (void)(flags); } while (0)
	#endif		#endif

	struct page kmap_to_page(void vaddr)		struct page __kmap_to_page(void vaddr)
	{		{
	unsigned long addr = (unsigned long)vaddr;		unsigned long addr = (unsigned long)vaddr;

	@@ -152,7 +152,7 @@ struct page kmap_to_page(void vaddr)

	return virt_to_page(addr);		return virt_to_page(addr);
	}		}
	EXPORT_SYMBOL(kmap_to_page);		EXPORT_SYMBOL(__kmap_to_page);

	static void flush_all_zero_pkmaps(void)		static void flush_all_zero_pkmaps(void)
	{		{
	@@ -194,10 +194,7 @@ static void flush_all_zero_pkmaps(void)
	flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));		flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP));
	}		}

	/**		void __kmap_flush_unused(void)
	* kmap_flush_unused - flush all unused kmap mappings in order to remove stray mappings
	*/
	void kmap_flush_unused(void)
	{		{
	lock_kmap();		lock_kmap();
	flush_all_zero_pkmaps();		flush_all_zero_pkmaps();