diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 2ccbabfb2cc1..bac2eb5de63d 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -243,21 +243,31 @@ restart: */ pmde = pmdp_get_lockless(pvmw->pmd); - if (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde)) { + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && + (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde) || + pmd_is_device_private_entry(pmde))) { pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; - if (!pmd_present(pmde)) { + if (pmd_is_migration_entry(pmde)) { softleaf_t entry; - if (!thp_migration_supported() || - !(pvmw->flags & PVMW_MIGRATION)) + if (!(pvmw->flags & PVMW_MIGRATION)) return not_found(pvmw); entry = softleaf_from_pmd(pmde); - - if (!softleaf_is_migration(entry) || - !check_pmd(softleaf_to_pfn(entry), pvmw)) + if (!check_pmd(softleaf_to_pfn(entry), pvmw)) return not_found(pvmw); return true; + } else if (pmd_is_device_private_entry(pmde)) { + softleaf_t entry; + + if (pvmw->flags & PVMW_MIGRATION) + return not_found(pvmw); + entry = softleaf_from_pmd(pmde); + if (!check_pmd(softleaf_to_pfn(entry), pvmw)) + return not_found(pvmw); + return true; + } else if (!pmd_present(pmde)) { + return not_found(pvmw); } if (likely(pmd_trans_huge(pmde))) { if (pvmw->flags & PVMW_MIGRATION) @@ -266,17 +276,10 @@ restart: return not_found(pvmw); return true; } - /* THP pmd was split under us: handle on pte level */ + /* THP/device-private pmd was split under us: handle on pte level */ spin_unlock(pvmw->ptl); pvmw->ptl = NULL; } else if (!pmd_present(pmde)) { - const softleaf_t entry = softleaf_from_pmd(pmde); - - if (softleaf_is_device_private(entry)) { - pvmw->ptl = pmd_lock(mm, pvmw->pmd); - return true; - } - if ((pvmw->flags & PVMW_SYNC) && thp_vma_suitable_order(vma, pvmw->address, PMD_ORDER) &&