mTHP (multi-size THP)
๊ด๋ จ ์์ค:mm/huge_memory.c (4978์ค),mm/khugepaged.c (2873์ค),include/linux/huge_mm.h (807์ค)
๊ด๋ จ ๋ฌธ์: Huge Pages / THP ยท VMA / mmap ยท Folio / Page Cache ยท Compaction ยท ๋ฉ๋ชจ๋ฆฌ ๊ด๋ฆฌ ๊ฐ์
๊ฐ์ (Overview)
Linux 6.8+์์ ๋์
๋ mTHP (multi-size THP) ๋ ๊ธฐ์กด THP๊ฐ ์ง์ํ๋ PMD ํฌ๊ธฐ(2MB)๋ฟ ์๋๋ผ, order-2(16KB)๋ถํฐ order-9(PMD ํฌ๊ธฐ)๊น์ง ๋ค์ํ ํฌ๊ธฐ์ large folio๋ฅผ ์ต๋ช
๋ฉ๋ชจ๋ฆฌ์ ํ ๋นํ ์ ์๊ฒ ํ๋ ๋ฉ์ปค๋์ฆ์
๋๋ค. mTHP๋ huge_anon_orders_always, huge_anon_orders_madvise, huge_anon_orders_inherit ๋นํธ๋งต์ ํตํด order๋ณ๋ก ๊ฐ๋ณ์ ์ธ ํ์ฑํ ์ ์ฑ
์ ์ ๊ณตํ๋ฉฐ, sysfs ์ธํฐํ์ด์ค(/sys/kernel/mm/transparent_hugepage/hugepages-XkB/)๋ฅผ ํตํด order๋ณ ์ ์ด๊ฐ ๊ฐ๋ฅํฉ๋๋ค.
mTHP๋ ๊ธฐ์กด THP์ PMD_ORDER ์ ์ฉ ๊ฒฝ๋ก์ ๋ฌ๋ฆฌ, thp_vma_allowable_orders() ํจ์๊ฐ ๋นํธ๋ง์คํฌ ๊ธฐ๋ฐ์ผ๋ก ํ์ฉ๋๋ order๋ฅผ ๊ฒฐ์ ํ๊ณ , ํ์ด์ง ํดํธ ์ฒ๋ฆฌ ๊ฒฝ๋ก์์ order๋ณ large folio๋ฅผ ํ ๋นํฉ๋๋ค. khugepaged ๋ฐฑ๊ทธ๋ผ์ด๋ ์ค๋ ๋๋ ํ์ฌ๊น์ง PMD_ORDER(2MB) ์ ์ฉ์ผ๋ก ๋์ํ๋ฉฐ, sub-PMD order์ ๋ณํฉ์ ์์ง ๊ตฌํ๋์ง ์์์ต๋๋ค. mTHP๋ per-CPU ํต๊ณ(mthp_stat_item)๋ฅผ ํตํด order๋ณ ํ ๋น/ํด๋ฐฑ/์ค์/์คํ๋ฆฟ ์ด๋ฒคํธ๋ฅผ ์ถ์ ํฉ๋๋ค.
์ผ์ ๋น์ : ๊ธฐ์กด THP๋ ๋ชจ๋ ํฐ ๊ธ์จ๋ฅผ 2MB ํฌ๊ธฐ์ ๋ํ ์ฌ์ ์๋ง ๋ฃ์ ์ ์์๋ค๋ฉด, mTHP๋ 16KB, 32KB, 64KB ๋ฑ ๋ค์ํ ํฌ๊ธฐ์ ์ค๊ฐ ์ฌ์ ๋ ์ฌ์ฉํ ์ ์๊ฒ ๋ ๊ฒ๊ณผ ๊ฐ์ต๋๋ค. ์์ ๋ฌธ์๋ ์ค๊ฐ ์ฌ์ ์, ํฐ ๋ฌธ์๋ ๋ํ ์ฌ์ ์ ๋ฃ์ด ์ฑ ์ฅ ๋๊ธฐ๋ ํ์(TLB ๋ฏธ์ค)๋ฅผ ์ต์ ํํฉ๋๋ค.
/* ์ฃผ์ ์์ค ํ์ผ ๊ฒฝ๋ก */
mm/huge_memory.c /* mTHP sysfs, order๋ณ ์ ์ฑ
, folio ๋ถํ , ํต๊ณ */
mm/khugepaged.c /* khugepaged ์ค๋ ๋ (PMD_ORDER ์ ์ฉ) */
include/linux/huge_mm.h /* THP_ORDERS_ALL_* ๋งคํฌ๋ก, mthp_stat_item, thpsize */
include/linux/page-flags.h /* PG_partially_mapped, PG_large_rmappable */
include/linux/mmzone.h /* struct deferred_split, NR_PCP_THP */
mTHP Order ๊ด๋ฆฌ ๊ตฌ์กฐ๋
mTHP ํ์ด์ง ํดํธ/ํ ๋น ํ๋ฆ
๋น ๋ฅธ ์ ๊ฒ ๋ช ๋ น
# mTHP ์ง์ order ํ์ธ (x86_64: order 2~9 = 16KB~2MB)
ls /sys/kernel/mm/transparent_hugepage/
# hugepages-16kB hugepages-32kB hugepages-64kB hugepages-128kB
# hugepages-256kB hugepages-512kB hugepages-1024kB hugepages-2048kB
# ํน์ order์ ํ์ฑํ ์ ์ฑ
ํ์ธ (์: 64KB = order-4)
cat /sys/kernel/mm/transparent_hugepage/hugepages-64kB/enabled
# [always] inherit madvise never
# order๋ณ mTHP ํต๊ณ ํ์ธ
cat /sys/kernel/mm/transparent_hugepage/hugepages-64kB/stats/anon_fault_alloc
cat /sys/kernel/mm/transparent_hugepage/hugepages-64kB/stats/anon_fault_fallback
cat /sys/kernel/mm/transparent_hugepage/hugepages-64kB/stats/split
# ์ ์ฒด mTHP ํต๊ณ (๋ชจ๋ order)
cat /proc/vmstat | grep -E 'thp_|khugepaged'
# THP ๊ด๋ จ ์ฌ์ฉ๋๊ณผ ํฐ ํ์ด์ง ์งํ ํ์ธ
grep -E 'AnonHugePages|ShmemHugePages|FileHugePages|HugePages_' /proc/meminfo
# khugepaged ๊ด๋ จ (PMD_ORDER ์ ์ฉ)
cat /sys/kernel/mm/transparent_hugepage/khugepaged/pages_to_scan
cat /sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed
# THP ์ ์ญ ํ์ฑํ ์ํ
cat /sys/kernel/mm/transparent_hugepage/enabled
# ์ปค๋ ๋ถํธ ํ๋ผ๋ฏธํฐ๋ก mTHP ์ค์ ํ์ธ
cat /proc/cmdline | grep -o 'thp_anon=[^ ]*'
ํต์ฌ ์๋ฃ๊ตฌ์กฐ
1. `THP_ORDERS_ALL_ANON` (order ๋นํธ๋ง์คํฌ)
์ต๋ช ๋ฉ๋ชจ๋ฆฌ์ ํ์ฉ๋๋ ๋ชจ๋ mTHP order์ ๋นํธ๋ง์คํฌ์ ๋๋ค. order-0๊ณผ order-1์ ์ ์ธ๋ฉ๋๋ค.
/* include/linux/huge_mm.h:79 โ ์ต๋ช
mTHP ํ์ฉ order ๋นํธ๋งต */
#define THP_ORDERS_ALL_ANON ((BIT(PMD_ORDER + 1) - 1) & ~(BIT(0) | BIT(1)))
/* include/linux/huge_mm.h:86-89 โ ํน์/file mTHP ํ์ฉ order ๋นํธ๋งต */
#define THP_ORDERS_ALL_SPECIAL \
(BIT(PMD_ORDER) | BIT(PUD_ORDER))
#define THP_ORDERS_ALL_FILE_DEFAULT \
((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))
/* include/linux/huge_mm.h:94-95 โ ์ ์ฒด THP ํ์ฉ order ๋นํธ๋งต */
#define THP_ORDERS_ALL \
(THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT)
2. `huge_anon_orders_*` (order๋ณ ์ ์ฑ ๋นํธ๋งต)
order๋ณ THP ํ์ฑํ ์ ์ฑ ์ ๋นํธ๋งต์ผ๋ก ๊ด๋ฆฌํฉ๋๋ค. ๊ฐ ๋นํธ๋ ํ๋์ order๋ฅผ ๋ํ๋ ๋๋ค.
/* include/linux/huge_mm.h:181-183 โ order๋ณ ์ ์ฑ
๋นํธ๋งต ์ ์ธ */
extern unsigned long huge_anon_orders_always; /* ํญ์ ์ฌ์ฉ */
extern unsigned long huge_anon_orders_madvise; /* madvise ์ ์ฌ์ฉ */
extern unsigned long huge_anon_orders_inherit; /* ์์ (๊ธฐ๋ณธ๊ฐ: PMD_ORDER๋ง) */
/* mm/huge_memory.c:790-791 โ ๊ธฐ๋ณธ๊ฐ: PMD_ORDER๋ง inherit */
huge_anon_orders_inherit = BIT(PMD_ORDER);
3. `enum mthp_stat_item` (mTHP ํต๊ณ ํญ๋ชฉ)
order๋ณ๋ก ์ถ์ ๋๋ mTHP ์ด๋ฒคํธ ํต๊ณ ํญ๋ชฉ์ ๋๋ค.
/* include/linux/huge_mm.h:127-146 โ mTHP ํต๊ณ ํญ๋ชฉ ์ด๊ฑฐํ */
enum mthp_stat_item {
MTHP_STAT_ANON_FAULT_ALLOC, /* ์ต๋ช
ํดํธ ์ order๋ณ ํ ๋น ์ฑ๊ณต */
MTHP_STAT_ANON_FAULT_FALLBACK, /* ์ต๋ช
ํดํธ ์ order๋ณ ํด๋ฐฑ */
MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, /* ํด๋ฐฑ ํ memcg ์ฒญ๊ตฌ ์คํจ */
MTHP_STAT_ZSWPOUT, /* order๋ณ zswap out */
MTHP_STAT_SWPIN, /* order๋ณ swap in */
MTHP_STAT_SWPIN_FALLBACK, /* swap in ํด๋ฐฑ */
MTHP_STAT_SWPIN_FALLBACK_CHARGE, /* swap in ํด๋ฐฑ ํ ์ฒญ๊ตฌ ์คํจ */
MTHP_STAT_SWPOUT, /* order๋ณ swap out */
MTHP_STAT_SWPOUT_FALLBACK, /* swap out ํด๋ฐฑ */
MTHP_STAT_SHMEM_ALLOC, /* shmem order๋ณ ํ ๋น */
MTHP_STAT_SHMEM_FALLBACK, /* shmem ํด๋ฐฑ */
MTHP_STAT_SHMEM_FALLBACK_CHARGE, /* shmem ํด๋ฐฑ ํ ์ฒญ๊ตฌ ์คํจ */
MTHP_STAT_SPLIT, /* order๋ณ folio ๋ถํ ์ฑ๊ณต */
MTHP_STAT_SPLIT_FAILED, /* folio ๋ถํ ์คํจ */
MTHP_STAT_SPLIT_DEFERRED, /* ์ง์ฐ ๋ถํ */
MTHP_STAT_NR_ANON, /* ํ์ฌ order๋ณ ์ต๋ช
large folio ์ */
MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, /* ๋ถ๋ถ ๋งคํ๋ ์ต๋ช
folio ์ */
__MTHP_STAT_COUNT
};
4. `struct mthp_stat` (per-CPU ํต๊ณ ๊ตฌ์กฐ์ฒด)
per-CPU๋ก ๊ด๋ฆฌ๋๋ mTHP ํต๊ณ ๊ตฌ์กฐ์ฒด์ ๋๋ค. ๊ฐ order๋ณ๋ก ๋ชจ๋ ํต๊ณ ํญ๋ชฉ์ ๊ฐ์ง๋๋ค.
/* include/linux/huge_mm.h:149-151 โ per-CPU mTHP ํต๊ณ ๊ตฌ์กฐ์ฒด */
struct mthp_stat {
unsigned long stats[ilog2(MAX_PTRS_PER_PTE) + 1][__MTHP_STAT_COUNT];
};
DECLARE_PER_CPU(struct mthp_stat, mthp_stats);
5. `struct thpsize` (sysfs order๋ณ kobject)
sysfs์์ ๊ฐ order์ THP ํฌ๊ธฐ๋ฅผ ๋ํ๋ด๋ kobject ๊ตฌ์กฐ์ฒด์ ๋๋ค.
/* include/linux/huge_mm.h:315-321 โ THP ํฌ๊ธฐ๋ณ sysfs kobject */
struct thpsize {
struct kobject kobj;
struct list_head node;
int order;
};
#define to_thpsize(kobj) container_of(kobj, struct thpsize, kobj)
6. `struct deferred_split` (์ง์ฐ ๋ถํ ํ)
large folio์ ์ง์ฐ ๋ถํ ์ ๊ด๋ฆฌํ๋ ํ ๊ตฌ์กฐ์ฒด์ ๋๋ค.
/* include/linux/mmzone.h:1341-1345 โ ์ง์ฐ ๋ถํ ํ */
struct deferred_split {
spinlock_t split_queue_lock;
struct list_head split_queue;
unsigned long split_queue_len;
};
ํต์ฌ ํจ์
1. `__thp_vma_allowable_orders()` โ VMA๋ณ ํ์ฉ order ๊ฒฐ์
์ง์ ๋ VMA์์ ํ์ฉ๋๋ hugepage order ๋นํธ๋ง์คํฌ๋ฅผ ๊ฒฐ์ ํ๋ ํต์ฌ ํจ์์ ๋๋ค.
/* mm/huge_memory.c:103-211 โ VMA๋ณ ํ์ฉ order ๋นํธ๋ง์คํฌ ๊ฒฐ์ */
unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
vm_flags_t vm_flags,
enum tva_type type,
unsigned long orders)
๋ถ๊ธฐ ๋ก์ง:
1. VMA ์ ํ์ ๋ฐ๋ผ ์ง์ order ์ธํธ ์ ํ (์ต๋ช
: THP_ORDERS_ALL_ANON, ํน์: THP_ORDERS_ALL_SPECIAL, ํ์ผ: THP_ORDERS_ALL_FILE_DEFAULT)
2. ํ๋์จ์ด THP ๋นํ์ฑํ ํ์ธ
3. VMA ๋นํ์ฑํ ํ๋๊ทธ ๊ฒ์ฌ (VM_NOHUGEPAGE, VM_DONTEXPAND)
4. DAX/PFNMAP ์ฒ๋ฆฌ
5. khugepaged ์ ์ฉ VMA ์ ์ธ
6. ์ ๋ ฌ/ํฌ๊ธฐ ์ ํฉ์ฑ ๊ฒ์ฌ (thp_vma_suitable_order())
7. shmem ์ ์ฑ ์ ์ฉ
8. ํ์ผ THP ์๊ตฌ์ฌํญ ํ์ธ (๊ธ๋ก๋ฒ ํ์ฑํ, VM_HUGEPAGE ํ๋๊ทธ, huge_fault() ํธ๋ค๋ฌ)
2. `setup_thp_anon()` โ ๋ถํธ ํ๋ผ๋ฏธํฐ order ์ ์ฑ ์ค์
์ปค๋ ๋ถํธ ํ๋ผ๋ฏธํฐ thp_anon=๋ฅผ ํ์ฑํ์ฌ order๋ณ ์ ์ฑ
์ ์ค์ ํฉ๋๋ค.
/* mm/huge_memory.c:991-1074 โ thp_anon= ๋ถํธ ํ๋ผ๋ฏธํฐ ํ์ */
static int __init setup_thp_anon(char *str)
๋ถ๊ธฐ ๋ก์ง:
1. ํฌ๊ธฐ ๋ฌธ์์ด์ order๋ก ๋ณํ (get_order_from_str())
2. ์ ์ฑ
๋ฌธ์์ด ํ์ฑ (always, madvise, inherit, never)
3. huge_anon_orders_always/madvise/inherit ๋นํธ๋งต ์
๋ฐ์ดํธ
4. anon_orders_configured = true ์ค์
3. `do_huge_pmd_anonymous_page()` โ PMD ์์ค ์ต๋ช ํ์ด์ง ํดํธ ์ฒ๋ฆฌ
PMD ์์ค์ ์ต๋ช ํ์ด์ง ํดํธ๋ฅผ ์ฒ๋ฆฌํ๋ ํจ์์ ๋๋ค. zero page ์ต์ ํ์ ์ผ๋ฐ ํ ๋น ๊ฒฝ๋ก๋ฅผ ๊ฐ์ง๋๋ค.
/* mm/huge_memory.c:1461-1516 โ PMD ์ต๋ช
ํ์ด์ง ํดํธ ์ง์
์ */
vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
vm_fault_t ret;
if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER))
return VM_FAULT_FALLBACK;
ret = vmf_anon_prepare(vmf);
if (ret)
return ret;
khugepaged_enter_vma(vma, vma->vm_flags);
if (!(vmf->flags & FAULT_FLAG_WRITE) &&
!mm_forbids_zeropage(vma->vm_mm) &&
transparent_hugepage_use_zero_page()) {
pgtable_t pgtable;
struct folio *zero_folio;
vm_fault_t ret;
pgtable = pte_alloc_one(vma->vm_mm);
if (unlikely(!pgtable))
return VM_FAULT_OOM;
zero_folio = mm_get_huge_zero_folio(vma->vm_mm);
if (unlikely(!zero_folio)) {
pte_free(vma->vm_mm, pgtable);
count_vm_event(THP_FAULT_FALLBACK);
return VM_FAULT_FALLBACK;
}
vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
ret = 0;
if (pmd_none(*vmf->pmd)) {
ret = check_stable_address_space(vma->vm_mm);
if (ret) {
spin_unlock(vmf->ptl);
pte_free(vma->vm_mm, pgtable);
} else if (userfaultfd_missing(vma)) {
spin_unlock(vmf->ptl);
pte_free(vma->vm_mm, pgtable);
ret = handle_userfault(vmf, VM_UFFD_MISSING);
VM_BUG_ON(ret & VM_FAULT_FALLBACK);
} else {
set_huge_zero_folio(pgtable, vma->vm_mm, vma,
haddr, vmf->pmd, zero_folio);
update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
spin_unlock(vmf->ptl);
}
} else {
spin_unlock(vmf->ptl);
pte_free(vma->vm_mm, pgtable);
}
return ret;
}
return __do_huge_pmd_anonymous_page(vmf);
}
๋ถ๊ธฐ ๋ก์ง:
1. thp_vma_suitable_order()๋ก VMA ์ ๋ ฌ/ํฌ๊ธฐ ๊ฒ์ฌ
2. vmf_anon_prepare()๋ก anon_vma ์ด๊ธฐํ
3. khugepaged_enter_vma()๋ก khugepaged ์ค์บ ๋์ ๋ฑ๋ก
4. ์ฝ๊ธฐ ์ ์ฉ + zero page ๊ฐ๋ฅ ์ set_huge_zero_folio() ํธ์ถ
5. ์ผ๋ฐ ๊ฒฝ๋ก: __do_huge_pmd_anonymous_page() ํธ์ถ
4. `__do_huge_pmd_anonymous_page()` โ ์ค์ large folio ํ ๋น ๋ฐ ๋งคํ
PMD ์์ค large folio๋ฅผ ํ ๋นํ๊ณ ํ์ด์ง ํ ์ด๋ธ์ ๋งคํํ๋ ํจ์์ ๋๋ค.
/* mm/huge_memory.c:1323-1373 โ PMD large folio ํ ๋น ๋ฐ ๋งคํ */
static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf)
{
unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
struct vm_area_struct *vma = vmf->vma;
struct folio *folio;
pgtable_t pgtable;
vm_fault_t ret = 0;
folio = vma_alloc_anon_folio_pmd(vma, vmf->address);
if (unlikely(!folio))
return VM_FAULT_FALLBACK;
pgtable = pte_alloc_one(vma->vm_mm);
if (unlikely(!pgtable)) {
ret = VM_FAULT_OOM;
goto release;
}
vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
if (unlikely(!pmd_none(*vmf->pmd))) {
goto unlock_release;
} else {
ret = check_stable_address_space(vma->vm_mm);
if (ret)
goto unlock_release;
/* ํ์ด์ง ํดํธ๋ฅผ ์ฌ์ฉ์ ๊ณต๊ฐ์ ์ ๋ฌ */
if (userfaultfd_missing(vma)) {
spin_unlock(vmf->ptl);
folio_put(folio);
pte_free(vma->vm_mm, pgtable);
ret = handle_userfault(vmf, VM_UFFD_MISSING);
VM_BUG_ON(ret & VM_FAULT_FALLBACK);
return ret;
}
pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
map_anon_folio_pmd_pf(folio, vmf->pmd, vma, haddr);
mm_inc_nr_ptes(vma->vm_mm);
spin_unlock(vmf->ptl);
}
return 0;
unlock_release:
spin_unlock(vmf->ptl);
release:
if (pgtable)
pte_free(vma->vm_mm, pgtable);
folio_put(folio);
return ret;
}
๋ถ๊ธฐ ๋ก์ง:
1. vma_alloc_anon_folio_pmd()๋ก order๋ณ large folio ํ ๋น
2. pte_alloc_one()์ผ๋ก ํ์ PTE ํ
์ด๋ธ ํ ๋น
3. PMD ๋ฝ ํ๋ ํ pmd_none() ๊ฒ์ฌ
4. check_stable_address_space()์ผ๋ก ์์ ์ ์ฃผ์ ๊ณต๊ฐ ํ์ธ
5. userfaultfd_missing() ์ userfaultfd ํธ๋ค๋ฌ ํธ์ถ
6. pgtable_trans_huge_deposit() + map_anon_folio_pmd_pf()๋ก ๋งคํ
5. `do_huge_pmd_wp_page()` โ PMD ์์ค COW (Copy-on-Write) ์ฒ๋ฆฌ
PMD ์์ค large folio์ COW๋ฅผ ์ฒ๋ฆฌํ๋ ํจ์์ ๋๋ค.
/* mm/huge_memory.c:2060-2152 โ PMD COW ์ฒ๋ฆฌ */
vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
{
const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
struct vm_area_struct *vma = vmf->vma;
struct page *page;
struct folio *folio;
unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
pmd_t orig_pmd = vmf->orig_pmd;
vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
VM_BUG_ON_VMA(!vma->anon_vma, vma);
if (is_huge_zero_pmd(orig_pmd)) {
vm_fault_t ret = do_huge_zero_wp_pmd(vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
/* THP๋ฅผ ํ ๋นํ ์ ์์ผ๋ฉด PMD ๋ถํ ๋ก ํด๋ฐฑ */
goto fallback;
}
spin_lock(vmf->ptl);
if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
spin_unlock(vmf->ptl);
return 0;
}
page = pmd_page(orig_pmd);
folio = page_folio(page);
VM_BUG_ON_PAGE(!PageHead(page), page);
/* PT ๋ฝ๋ง ์ก์์ ๋์ ์กฐ๊ธฐ ๊ฒ์ฌ */
if (PageAnonExclusive(page))
goto reuse;
if (!folio_trylock(folio)) {
folio_get(folio);
spin_unlock(vmf->ptl);
folio_lock(folio);
spin_lock(vmf->ptl);
if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
spin_unlock(vmf->ptl);
folio_unlock(folio);
folio_put(folio);
return 0;
}
folio_put(folio);
}
/* ์ ๊น PT ๋ฝ์ ๋ด๋ ค๋์ ๋ค ๋ค์ ๊ฒ์ฌ */
if (PageAnonExclusive(page)) {
folio_unlock(folio);
goto reuse;
}
/* do_wp_page()์ฒ๋ผ ์ถ๊ฐ ์ฐธ์กฐ๊ฐ ์์ ๋๋ง ๋
์ ์ฌ์ฌ์ฉ ๊ฐ๋ฅ */
if (folio_ref_count(folio) >
1 + folio_test_swapcache(folio) * folio_nr_pages(folio))
goto unlock_fallback;
if (folio_test_swapcache(folio))
folio_free_swap(folio);
if (folio_ref_count(folio) == 1) {
pmd_t entry;
folio_move_anon_rmap(folio, vma);
SetPageAnonExclusive(page);
folio_unlock(folio);
reuse:
if (unlikely(unshare)) {
spin_unlock(vmf->ptl);
return 0;
}
entry = pmd_mkyoung(orig_pmd);
entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))
update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
spin_unlock(vmf->ptl);
return 0;
}
unlock_fallback:
folio_unlock(folio);
spin_unlock(vmf->ptl);
fallback:
/* PMD๋ฅผ 4KB PTE๋ก ๋ถํ ํ ํด๋ฐฑ */
__split_huge_pmd(vma, vmf->pmd, vmf->address, false);
return VM_FAULT_FALLBACK;
}
๋ถ๊ธฐ ๋ก์ง:
1. is_huge_zero_pmd()๋ก zero PMD ๊ฒ์ฌ
2. PageAnonExclusive()๋ก exclusive ์ฌ์ฌ์ฉ ๊ฐ๋ฅ ํ์ธ
3. folio_ref_count()๋ก ์ฐธ์กฐ ์นด์ดํธ ๊ฒ์ฌ (1์ด๋ฉด exclusive)
4. exclusive ๊ฐ๋ฅ ์ pmd_mkyoung() + maybe_pmd_mkwrite()๋ก ์ ๊ทผ/์ฐ๊ธฐ ํ๋๊ทธ ์
๋ฐ์ดํธ
5. ๋ถ๊ฐ๋ฅ ์ __split_huge_pmd()๋ก PMD ๋ถํ ํ VM_FAULT_FALLBACK ๋ฐํ
6. `anon_enabled_show()` / `anon_enabled_store()` โ sysfs order๋ณ ์ ์ฑ ์ ์ด
sysfs ์ธํฐํ์ด์ค๋ฅผ ํตํด order๋ณ THP ํ์ฑํ ์ ์ฑ ์ ์ฝ๊ณ ์๋๋ค.
/* mm/huge_memory.c:500-563 โ sysfs order๋ณ enabled ์ ์ด */
static ssize_t anon_enabled_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf);
static ssize_t anon_enabled_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count);
๋์:
show: [always], [inherit], [madvise], [never] ์ค ํ์ฌ ์ ์ฑ
๋ฐํstore: ์ ์ฑ
๋ฌธ์์ด ํ์ฑ ํ huge_anon_orders_always/madvise/inherit ๋นํธ๋งต ์
๋ฐ์ดํธ, start_stop_khugepaged() ํธ์ถ4. `deferred_split_folio()` โ large folio ์ง์ฐ ๋ถํ ํ ๋ฑ๋ก
large folio๋ฅผ ์ง์ฐ ๋ถํ ํ์ ๋ฑ๋กํ์ฌ ์ดํ shrinker์ ์ํด ๋ถํ ๋๋๋ก ํฉ๋๋ค.
/* mm/huge_memory.c:4315-4365 โ ์ง์ฐ ๋ถํ ํ ๋ฑ๋ก */
void deferred_split_folio(struct folio *folio, bool partially_mapped)
๋ถ๊ธฐ ๋ก์ง:
1. order โค 1 folio๋ ํ์ ๋ฃ์ง ์์ (๊ณต๊ฐ ๋ถ์กฑ)
2. folio_test_large() ๊ฒ์ฌ
3. partially_mapped ํ๋๊ทธ ์ค์ ์ MTHP_STAT_NR_ANON_PARTIALLY_MAPPED ์ฆ๊ฐ
4. MTHP_STAT_SPLIT_DEFERRED ํต๊ณ ์
๋ฐ์ดํธ
5. `folio_check_splittable()` โ folio ๋ถํ ๊ฐ๋ฅ ๊ฒ์ฌ
folio๊ฐ ์ง์ ๋ order๋ก ๋ถํ ๊ฐ๋ฅํ์ง ๊ฒ์ฌํฉ๋๋ค.
/* mm/huge_memory.c:3708-3769 โ folio ๋ถํ ๊ฐ๋ฅ์ฑ ๊ฒ์ฌ */
int folio_check_splittable(struct folio *folio, unsigned int new_order,
enum split_type split_type)
๋ถ๊ธฐ ๋ก์ง:
1. order-1 ์ต๋ช
folio ๋ถํ ๊ฑฐ๋ถ (-EINVAL)
2. ํ์ผ ๊ธฐ๋ฐ folio: mapping_large_folio_support() ๊ฒ์ฌ
3. swapcache folio: non-zero order ๋ถํ ๊ฑฐ๋ถ
4. non-uniform ๋ถํ ์ ์ถ๊ฐ ๊ฒ์ฌ
ํธ์ถ ํ๋ฆ
mTHP ํ์ด์ง ํดํธ ๊ฒฝ๋ก
์ฌ์ฉ์ ํ์ด์ง ํดํธ (PMD ์์ค)
โ handle_mm_fault()
โ __handle_mm_fault()
โ handle_pte_fault()
โ do_anonymous_page()
โ vma_alloc_anon_folio() โ order ์ ํ
โ thp_vma_allowable_orders() โ ํ์ฉ order ๋นํธ๋ง์คํฌ ๊ฒฐ์
โ __thp_vma_allowable_orders() โ VMA๋ณ ํํฐ๋ง
โ alloc_folio(gfp, order) โ order๋ณ large folio ํ ๋น
โ finish_fault()
โ do_set_pte() โ PTE ์ค์
์ฌ์ฉ์ ํ์ด์ง ํดํธ (PMD ์์ค large folio)
โ handle_mm_fault()
โ __handle_mm_fault()
โ handle_pte_fault()
โ do_huge_pmd_anonymous_page() โ PMD ์ง์
์ (huge_memory.c:1461)
โ thp_vma_suitable_order() โ VMA ์ ๋ ฌ/ํฌ๊ธฐ ๊ฒ์ฌ
โ vmf_anon_prepare() โ anon_vma ์ด๊ธฐํ
โ khugepaged_enter_vma() โ khugepaged ์ค์บ ๋์ ๋ฑ๋ก
โ __do_huge_pmd_anonymous_page() โ ์ค์ ํ ๋น (huge_memory.c:1323)
โ vma_alloc_anon_folio_pmd() โ order๋ณ large folio ํ ๋น
โ pte_alloc_one() โ ํ์ PTE ํ
์ด๋ธ ํ ๋น
โ pmd_lock() โ PMD ๋ฝ
โ check_stable_address_space() โ ์์ ์ ์ฃผ์ ๊ณต๊ฐ ํ์ธ
โ pgtable_trans_huge_deposit() โ PTE ํ
์ด๋ธ ๋ฐฐํฌ
โ map_anon_folio_pmd_pf() โ PMD์ large folio ๋งคํ
์ฌ์ฉ์ COW ํดํธ (PMD ์์ค)
โ handle_mm_fault()
โ __handle_mm_fault()
โ handle_pte_fault()
โ do_wp_page()
โ do_huge_pmd_wp_page() โ PMD COW ์ฒ๋ฆฌ (huge_memory.c:2060)
โ is_huge_zero_pmd() โ zero PMD ๊ฒ์ฌ
โ PageAnonExclusive() โ exclusive ์ฌ์ฌ์ฉ ๊ฐ๋ฅ ํ์ธ
โ folio_ref_count() โ ์ฐธ์กฐ ์นด์ดํธ ๊ฒ์ฌ
โ pmd_mkyoung() + maybe_pmd_mkwrite() โ ์ ๊ทผ/์ฐ๊ธฐ ํ๋๊ทธ
โ __split_huge_pmd() โ PMD ๋ถํ (๋ถ๊ฐ๋ฅ ์)
mTHP sysfs ์ ์ด ๊ฒฝ๋ก
์ฌ์ฉ์: echo always > /sys/kernel/mm/transparent_hugepage/hugepages-64kB/enabled
โ anon_enabled_store()
โ parsing: "always"
โ set_bit(order, &huge_anon_orders_always)
โ clear_bit(order, &huge_anon_orders_madvise)
โ clear_bit(order, &huge_anon_orders_inherit)
โ start_stop_khugepaged() โ khugepaged ์์/์ค์ง
mTHP folio ๋ถํ ๊ฒฝ๋ก
large folio ๋ถํ ํ์
โ deferred_split_folio()
โ list_add(&folio->deferred_list, &nm->deferred_split_queue)
โ count_mthp_stat(order, MTHP_STAT_SPLIT_DEFERRED)
shrinker ์ค์บ ์
โ deferred_split_scan()
โ folio_split() / split_folio_to_order()
โ __folio_split()
โ __folio_freeze_and_split_unmapped()
โ __split_unmapped_folio()
โ __split_folio_to_order() โ ์ค์ ๋ถํ
์กฐ๊ฑด๋ณ ๋น๊ต
mTHP vs ๊ธฐ์กด THP ๋น๊ต
| ํญ๋ชฉ | ๊ธฐ์กด THP | mTHP |
|---|---|---|
| ์ง์ order | PMD_ORDER (9)๋ง | order 2~9 (16KB~2MB) |
| ๊ธฐ๋ณธ ํ์ฑํ | global always/madvise | order๋ณ ๊ฐ๋ณ ์ ์ฑ |
| sysfs ์ธํฐํ์ด์ค | `/sys/kernel/mm/transparent_hugepage/enabled` | `/sys/kernel/mm/transparent_hugepage/hugepages-XkB/enabled` |
| khugepaged ๋ณํฉ | PMD_ORDER ์ ์ฉ | PMD_ORDER๋ง ์ง์ (sub-PMD ๋ฏธ๊ตฌํ) |
| per-CPU ํต๊ณ | ์์ | order๋ณ 17์ข ํต๊ณ |
| ์ง์ฐ ๋ถํ | PMD folio๋ง | order 2+ ๋ชจ๋ large folio |
| TLB ํจ์จ | 2MB ํ๋๋ก ๋งคํ | ํฌ๊ธฐ๋ณ ์ต์ ๋งคํ |
order๋ณ ํฌ๊ธฐ ๋ฐ ํน์ฑ ๋น๊ต
| order | ํฌ๊ธฐ | x86_64 PTE ์ | sysfs ์ด๋ฆ | ์ฃผ์ ์ฌ์ฉ ์๋๋ฆฌ์ค |
|---|---|---|---|---|
| 2 | 16KB | 4 | hugepages-16kB | ์๊ท๋ชจ large folio, ์ปจํ ์ด๋ |
| 3 | 32KB | 8 | hugepages-32kB | ์ค๊ฐ ํฌ๊ธฐ I/O ์ต์ ํ |
| 4 | 64KB | 16 | hugepages-64kB | ์ผ๋ฐ์ ์ธ mTHP ์ฌ์ฉ |
| 5 | 128KB | 32 | hugepages-128kB | ๋์ฉ๋ ๋ฐ์ดํฐ ์ฒ๋ฆฌ |
| 6 | 256KB | 64 | hugepages-256kB | ๋๊ท๋ชจ ๋ฉ๋ชจ๋ฆฌ ์์ญ |
| 7 | 512KB | 128 | hugepages-512kB | ๋งค์ฐ ํฐ ๋ฉ๋ชจ๋ฆฌ ์์ญ |
| 8 | 1MB | 256 | hugepages-1024kB | THP ๊ทผ์ฌ |
| 9 | 2MB | 512 | hugepages-2048kB | classic THP (PMD ํฌ๊ธฐ) |
mTHP ์ ์ฑ ๋ชจ๋ ๋น๊ต
| ์ ์ฑ | ๋์ | ๋นํธ๋งต | ์ฌ์ฉ ์๋๋ฆฌ์ค |
|---|---|---|---|
| `always` | ๋ชจ๋ ํดํธ์์ ํด๋น order ์ฌ์ฉ | `huge_anon_orders_always` | ์ฑ๋ฅ ์ฐ์ , ๋ฉ๋ชจ๋ฆฌ ์ฌ์ ์ |
| `madvise` | `MADV_HUGEPAGE` ์ ์ฉ ์์๋ง ์ฌ์ฉ | `huge_anon_orders_madvise` | ํน์ ํ๋ก์ธ์ค๋ง ์ต์ ํ |
| `inherit` | VMA ํ๋๊ทธ ์์ | `huge_anon_orders_inherit` | ๊ธฐ๋ณธ๊ฐ (PMD๋ง ์์) |
| `never` | ํด๋น order ์ฌ์ฉ ์ ํจ | ๋นํธ๋งต์์ ์ ์ธ | ๋ฉ๋ชจ๋ฆฌ ์ ์ฝ, ์ถฉ๋ ํํผ |
mTHP vs HugeTLB vs shmem THP ๋น๊ต
| ํญ๋ชฉ | mTHP | HugeTLB | shmem THP |
|---|---|---|---|
| ํ ๋น ์์ | ํดํธ ์ ๋์ | ๋ถํ ์ ์ฌ์ ์์ฝ | ํดํธ ์ ๋์ |
| ๋ฉ๋ชจ๋ฆฌ ์๋น | ํ์ ์๋ง | ์ฌ์ ์์ฝ | ํ์ ์๋ง |
| ์ค์ ๊ฐ๋ฅ | ๊ฐ๋ฅ | ๋ถ๊ฐ | ๊ฐ๋ฅ |
| ํ์ผ ๋งคํ | ๋ถ๊ฐ | ๊ฐ๋ฅ | tmpfs ์ ์ฉ |
| ์ปจํ ์ด๋ ์ง์ | ์ข์ | ์ ํ์ | ์ข์ |
| TLB ํจ์จ | ์ค๊ฐ~๋์ | ๋์ | ์ค๊ฐ~๋์ |
mTHP ํ์ฑํ ๋ฐฉ๋ฒ
1. ๋ถํธ ํ๋ผ๋ฏธํฐ
# ๋ชจ๋ order๋ฅผ always๋ก ํ์ฑํ
thp_anon=always:all
# ํน์ order๋ง madvise๋ก ํ์ฑํ (64KB, 128KB)
thp_anon=madvise:64k,128k
# PMD๋ง always, ๋๋จธ์ง never
thp_anon=always:2m
2. sysfs ๋์ ์ค์
# 64KB mTHP๋ฅผ always๋ก ํ์ฑํ
echo always > /sys/kernel/mm/transparent_hugepage/hugepages-64kB/enabled
# 128KB mTHP๋ฅผ madvise๋ก ์ค์
echo madvise > /sys/kernel/mm/transparent_hugepage/hugepages-128kB/enabled
# ๋ชจ๋ sub-PMD order๋ฅผ never๋ก ๋นํ์ฑํ
for sz in 16kB 32kB 64kB 128kB 256kB 512kB 1024kB; do
echo never > /sys/kernel/mm/transparent_hugepage/hugepages-${sz}/enabled
done
3. ํ๋ก์ธ์ค๋ณ madvise
/* ํน์ VMA์ mTHP ์ ์ฉ */
madvise(addr, length, MADV_HUGEPAGE); /* allow๋ order ๋ชจ๋ ์ฌ์ฉ */
/* mTHP ๋นํ์ฑํ */
madvise(addr, length, MADV_NOHUGEPAGE);
mTHP ํต๊ณ ํด์
| ํต๊ณ ํญ๋ชฉ | ์ค๋ช | ๋์ ๊ฐ์ ์๋ฏธ |
|---|---|---|
| `anon_fault_alloc` | order๋ณ ํ ๋น ์ฑ๊ณต ์ | ํด๋น order๊ฐ ํ๋ฐํ๊ฒ ์ฌ์ฉ๋จ |
| `anon_fault_fallback` | smaller order๋ก ํด๋ฐฑ ์ | ํฐ order ํ ๋น ์คํจ ๋น๋ฒ |
| `anon_fault_fallback_charge` | ํด๋ฐฑ ํ memcg ์ฒญ๊ตฌ ์คํจ | ๋ฉ๋ชจ๋ฆฌ ์๋ฐ |
| `split` | ๋ถํ ์ฑ๊ณต ์ | large folio๊ฐ ์ชผ๊ฐ์ง๋ ๋น๋ |
| `split_deferred` | ์ง์ฐ ๋ถํ ํ ๋ฑ๋ก ์ | partially-mapped folio ์ฆ๊ฐ |
| `nr_anon` | ํ์ฌ order๋ณ large folio ์ | ๋ฉ๋ชจ๋ฆฌ ์ฌ์ฉ ํจํด |
| `nr_anon_partially_mapped` | ๋ถ๋ถ ๋งคํ folio ์ | ๋ถํ ํ์์ฑ ์งํ |