diff options
| author | Andrea Arcangeli <aarcange@redhat.com> | 2023-06-08 13:26:25 -0400 |
|---|---|---|
| committer | Andrea Arcangeli <aarcange@redhat.com> | 2023-11-11 22:03:36 -0500 |
| commit | 051f877967290dbacff57b3f88cb0dbcc3651565 (patch) | |
| tree | 2931142ce91349098f3c127fdc3e56a50d7137f6 | |
| parent | 6de8dad8746037c34c3755707cb860c5b810534a (diff) | |
| download | aa-051f877967290dbacff57b3f88cb0dbcc3651565.tar.gz | |
mm: gup: introduce PageAnonGup helpers
This uses the same technique as David's PageAnonExclusive. The
implementation in turn is based on upstream commit
78fbe906cc900b33ce078102e13e0e99b5b8c406 .
(cherry picked from commit 78fbe906cc900b33ce078102e13e0e99b5b8c406)
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
| -rw-r--r-- | include/linux/page-flags.h | 15 | ||||
| -rw-r--r-- | mm/memory.c | 11 | ||||
| -rw-r--r-- | mm/swapfile.c | 3 |
3 files changed, 29 insertions, 0 deletions
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index a815a99ab23e8e..2146a9d8d0b3b4 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -185,6 +185,8 @@ enum pageflags { /* Only valid for buddy pages. Used to track pages that are reported */ PG_reported = PG_uptodate, + + PG_anon_gup = PG_mappedtodisk, }; #define PAGEFLAGS_MASK ((1UL << NR_PAGEFLAGS) - 1) @@ -628,6 +630,19 @@ TESTPAGEFLAG_FALSE(Huge) TESTPAGEFLAG_FALSE(HeadHuge) #endif +static __always_inline int PageAnonGup(struct page *page) +{ + VM_BUG_ON_PAGE(!PageAnon(page), page); + VM_BUG_ON_PAGE(PageHuge(page) && !PageHead(page), page); + return test_bit(PG_anon_gup, &PF_ANY(page, 1)->flags); +} + +static __always_inline void SetPageAnonGup(struct page *page) +{ + VM_BUG_ON_PAGE(!PageAnonNoKsm(page), page); + VM_BUG_ON_PAGE(PageHuge(page) && !PageHead(page), page); + set_bit(PG_anon_gup, &PF_ANY(page, 1)->flags); +} #ifdef CONFIG_TRANSPARENT_HUGEPAGE /* diff --git a/mm/memory.c b/mm/memory.c index 5d8075ca21ec8a..ab25ec3dcea1f9 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3728,6 +3728,17 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf) struct mmu_notifier_range range; /* + * PG_anon_gup reuses PG_mappedtodisk for anon pages. A swap pte + * must never point at an anonymous page in the swapcache that is + * PG_anon_gup. Sanity check that this holds and especially, that + * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity + * check after taking the PT lock and making sure that nobody + * concurrently faulted in this page and set PG_anon_gup. + */ + BUG_ON(!PageAnon(page) && PageMappedToDisk(page)); + BUG_ON(PageAnon(page) && PageAnonGup(page)); + + /* * We need a reference to lock the page because we don't hold * the PTL so a racing thread can remove the device-exclusive * entry and unmap it. If the page is free the entry must diff --git a/mm/swapfile.c b/mm/swapfile.c index 3616e87c399145..77fe42c1fc644c 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1985,6 +1985,9 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, goto out; } + /* See do_swap_page() */ + BUG_ON(!PageAnon(page) && PageMappedToDisk(page)); + dec_mm_counter(vma->vm_mm, MM_SWAPENTS); inc_mm_counter(vma->vm_mm, MM_ANONPAGES); get_page(page); |