--- linux/fs/proc/proc_misc.c.orig Sun Feb 24 15:36:55 2002
+++ linux/fs/proc/proc_misc.c Wed Feb 27 09:09:02 2002
@@ -179,7 +179,8 @@
"LowTotal: %8lu kB\n"
"LowFree: %8lu kB\n"
"SwapTotal: %8lu kB\n"
- "SwapFree: %8lu kB\n",
+ "SwapFree: %8lu kB\n"
+ "BigPagesFree: %8lu kB\n",
K(i.totalram),
K(i.freeram),
K(i.sharedram),
@@ -195,7 +196,8 @@
K(i.totalram-i.totalhigh),
K(i.freeram-i.freehigh),
K(i.totalswap),
- K(i.freeswap));
+ K(i.freeswap),
+ nr_bigpages << (PMD_SHIFT-10));
return proc_calc_metrics(page, start, off, count, eof, len);
#undef B
--- linux/fs/proc/array.c.orig Sun Feb 24 19:40:00 2002
+++ linux/fs/proc/array.c Tue Mar 5 23:02:48 2002
@@ -397,6 +397,13 @@
if (pmd_none(*pmd))
return;
+ if (pmd_bigpage(*pmd)) {
+ *total += BIGPAGE_PAGES;
+ *pages += BIGPAGE_PAGES;
+ *shared += BIGPAGE_PAGES;
+ *dirty += BIGPAGE_PAGES;
+ return;
+ }
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
@@ -557,6 +564,12 @@
str[1] = flags & VM_WRITE ? 'w' : '-';
str[2] = flags & VM_EXEC ? 'x' : '-';
str[3] = flags & VM_MAYSHARE ? 's' : 'p';
+ if (flags & VM_BIGPAGE)
+ str[3] = 'B';
+ else
+ if (map->vm_ops && (map->vm_ops->nopage == shmem_nopage) &&
+ I_BIGPAGE(map->vm_file->f_dentry->d_inode))
+ str[3] = 'b';
str[4] = 0;
dev = 0;
--- linux/fs/binfmt_elf.c.orig Sun Feb 24 19:39:14 2002
+++ linux/fs/binfmt_elf.c Wed Feb 27 09:03:34 2002
@@ -1212,12 +1212,12 @@
if (pgd_none(*pgd))
goto nextpage_coredump;
pmd = pmd_offset(pgd, addr);
- if (pmd_none(*pmd))
+ if (pmd_none(*pmd) || pmd_bigpage(*pmd))
goto nextpage_coredump;
pte = pte_offset_map(pmd, addr);
none = pte_none(*pte);
pte_unmap(pte);
- if (pte_none(*pte)) {
+ if (none) {
nextpage_coredump:
DUMP_SEEK (file->f_pos + PAGE_SIZE);
} else {
--- linux/kernel/ptrace.c.orig Sun Feb 24 19:41:59 2002
+++ linux/kernel/ptrace.c Sun Feb 24 20:26:07 2002
@@ -105,15 +105,24 @@
pgmiddle = pmd_offset(pgdir, addr);
if (pmd_none(*pgmiddle))
goto fault_in_page;
- if (pmd_bad(*pgmiddle))
- goto bad_pmd;
- pgtable = pte_offset_map(pgmiddle, addr);
- if (!pte_present(*pgtable))
- goto fault_in_page_unmap;
- if (write && (!pte_write(*pgtable) || !pte_dirty(*pgtable)))
- goto fault_in_page_unmap;
- page = pte_page(*pgtable);
- pte_unmap(pgtable);
+ if (pmd_bigpage(*pgmiddle)) {
+ unsigned long idx = (addr & BIGPAGE_MASK) / PAGE_SIZE;
+
+ page = pmd_page(*pgmiddle);
+ if (!BigPage(page))
+ BUG();
+ page += idx;
+ } else {
+ if (pmd_bad(*pgmiddle))
+ goto bad_pmd;
+ pgtable = pte_offset_map(pgmiddle, addr);
+ if (!pte_present(*pgtable))
+ goto fault_in_page_unmap;
+ if (write && (!pte_write(*pgtable) || !pte_dirty(*pgtable)))
+ goto fault_in_page_unmap;
+ page = pte_page(*pgtable);
+ pte_unmap(pgtable);
+ }
/* ZERO_PAGE is special: reads from it are ok even though it's marked reserved */
if (page != ZERO_PAGE(addr) || write) {
--- linux/kernel/fork.c.orig Sun Feb 24 21:16:40 2002
+++ linux/kernel/fork.c Wed Mar 6 20:49:40 2002
@@ -176,7 +176,8 @@
if (!tmp)
goto fail_nomem;
*tmp = *mpnt;
- tmp->vm_flags &= ~VM_LOCKED;
+ if (!(tmp->vm_flags & VM_BIGPAGE))
+ tmp->vm_flags &= ~VM_LOCKED;
tmp->vm_mm = mm;
tmp->vm_next = NULL;
file = tmp->vm_file;
--- linux/kernel/sysctl.c.orig Tue Feb 26 09:33:03 2002
+++ linux/kernel/sysctl.c Tue Feb 26 10:08:18 2002
@@ -222,6 +222,8 @@
0444, NULL, &proc_dointvec},
{KERN_RTSIGMAX, "rtsig-max", &max_queued_signals, sizeof(int),
0644, NULL, &proc_dointvec},
+ {KERN_SHMUSEBIGPAGES, "shm-use-bigpages", &shm_use_bigpages, sizeof(int),
+ 0644, NULL, &proc_dointvec},
#ifdef CONFIG_SYSVIPC
{KERN_SHMMAX, "shmmax", &shm_ctlmax, sizeof (size_t),
0644, NULL, &proc_doulongvec_minmax},
--- linux/mm/bigpages.c.orig Tue Mar 5 17:50:34 2002
+++ linux/mm/bigpages.c Sat Mar 2 11:54:40 2002
@@ -0,0 +1,96 @@
+/*
+ * linux/mm/bigpages.c
+ *
+ * Copyright (C) 2002 Ingo Molnar
+ */
+
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/spinlock.h>
+#include <linux/highmem.h>
+#include <linux/smp_lock.h>
+#include <linux/interrupt.h>
+#include <linux/init.h>
+#include <asm/uaccess.h>
+#include <asm/pgalloc.h>
+
+static spinlock_t bigpages_lock = SPIN_LOCK_UNLOCKED;
+long nr_bigpages;
+static LIST_HEAD(bigpages_list);
+
+#define ORDER_BIGPAGE (PMD_SHIFT - PAGE_SHIFT)
+
+struct page *alloc_bigpage(void)
+{
+ list_t *head = &bigpages_list;
+ struct page *page = NULL;
+
+ spin_lock(&bigpages_lock);
+ if (nr_bigpages) {
+ page = list_entry(head->next, struct page, list);
+ list_del_init(head->next);
+ nr_bigpages--;
+ }
+ spin_unlock(&bigpages_lock);
+
+ return page;
+}
+
+void free_bigpage(struct page *page)
+{
+ struct page *p;
+ int i;
+
+ if ((page - mem_map) % BIGPAGE_PAGES)
+ BUG();
+ for (i = 0 ; i < (1 << ORDER_BIGPAGE); i++) {
+ p = page + i;
+ set_page_count(p, 2);
+ set_bit(PG_bigpage, &p->flags);
+ clear_highpage(p);
+ }
+ spin_lock(&bigpages_lock);
+ nr_bigpages++;
+ list_add(&page->list, &bigpages_list);
+ spin_unlock(&bigpages_lock);
+}
+
+static int grow_bigpages_pool(int pages)
+{
+ struct page *page;
+ int allocated = 0;
+
+ while (pages) {
+ page = alloc_pages(__GFP_HIGHMEM, ORDER_BIGPAGE);
+ if (!page)
+ break;
+ free_bigpage(page);
+ pages--;
+ allocated++;
+ }
+ printk("bigpage subsystem: allocated %ld bigpages (=%ldMB).\n",
+ nr_bigpages, nr_bigpages << (BIGPAGE_SHIFT - 20));
+ return allocated;
+}
+
+static __initdata int boot_bigpages;
+
+static __init int reserve_bigpages(char *str)
+{
+ unsigned long pages = memparse(str, &str) >> PAGE_SHIFT;
+
+ pages >>= ORDER_BIGPAGE;
+ boot_bigpages = pages;
+
+ return 0;
+}
+
+static __init int init_bigpage_pool(void)
+{
+ grow_bigpages_pool(boot_bigpages);
+ return 0;
+}
+
+__setup("bigpages=", reserve_bigpages);
+__initcall(init_bigpage_pool);
+
--- linux/mm/page_alloc.c.orig Sun Feb 24 11:49:51 2002
+++ linux/mm/page_alloc.c Wed Mar 6 20:52:15 2002
@@ -72,13 +72,8 @@
per_cpu_t *per_cpu;
zone_t *zone;
- /*
- * This late check is safe because reserved pages do not
- * have a valid page->count. This trick avoids overhead
- * in __free_pages().
- */
- if (PageReserved(page))
- return;
+ if (BigPage(page))
+ BUG();
if (page->buffers)
BUG();
if (page->mapping)
@@ -209,6 +204,8 @@
__restore_flags(flags);
set_page_count(page, 1);
+ if (BigPage(page))
+ BUG();
return page;
}
@@ -236,6 +233,8 @@
if (BAD_RANGE(zone,page))
BUG();
DEBUG_ADD_PAGE
+ if (BigPage(page))
+ BUG();
return page;
}
curr_order++;
@@ -594,6 +595,8 @@
void __free_pages(struct page *page, unsigned long order)
{
+ if (PageReserved(page))
+ return;
if (put_page_testzero(page))
__free_pages_ok(page, order);
}
--- linux/mm/shmem.c.orig Sun Feb 24 13:08:00 2002
+++ linux/mm/shmem.c Wed Mar 6 20:55:39 2002
@@ -5,6 +5,7 @@
* 2000 Transmeta Corp.
* 2000-2001 Christoph Rohland
* 2000-2001 SAP AG
+ * 2002 Ingo Molnar, Red Hat Inc.
*
* This file is released under the GPL.
*/
@@ -21,6 +22,7 @@
#include <linux/devfs_fs_kernel.h>
#include <linux/fs.h>
#include <linux/mm.h>
+#include <linux/slab.h>
#include <linux/file.h>
#include <linux/swap.h>
#include <linux/pagemap.h>
@@ -49,6 +51,8 @@
static spinlock_t shmem_ilock = SPIN_LOCK_UNLOCKED;
atomic_t shmem_nrpages = ATOMIC_INIT(0);
+int shm_use_bigpages;
+
#define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512)
static void shmem_removepage(struct page *page)
@@ -76,19 +80,40 @@
* It has to be called with the spinlock held.
*/
+static int shm_alloc_space(struct inode * inode, unsigned long space)
+{
+ struct shmem_sb_info * sbinfo = SHMEM_SB(inode->i_sb);
+
+ spin_lock(&sbinfo->stat_lock);
+ if (sbinfo->free_blocks < space) {
+ spin_unlock(&sbinfo->stat_lock);
+ return -ENOSPC;
+ }
+ sbinfo->free_blocks -= space;
+ inode->i_blocks += space*BLOCKS_PER_PAGE;
+ spin_unlock(&sbinfo->stat_lock);
+
+ return 0;
+}
+
+static void shm_free_space(struct inode * inode, unsigned long freed)
+{
+ struct shmem_sb_info * sbinfo = SHMEM_SB(inode->i_sb);
+
+ spin_lock(&sbinfo->stat_lock);
+ sbinfo->free_blocks += freed;
+ inode->i_blocks -= freed*BLOCKS_PER_PAGE;
+ spin_unlock(&sbinfo->stat_lock);
+}
+
static void shmem_recalc_inode(struct inode * inode)
{
unsigned long freed;
freed = (inode->i_blocks/BLOCKS_PER_PAGE) -
(inode->i_mapping->nrpages + SHMEM_I(inode)->swapped);
- if (freed){
- struct shmem_sb_info * sbinfo = SHMEM_SB(inode->i_sb);
- inode->i_blocks -= freed*BLOCKS_PER_PAGE;
- spin_lock (&sbinfo->stat_lock);
- sbinfo->free_blocks += freed;
- spin_unlock (&sbinfo->stat_lock);
- }
+ if (freed)
+ shm_free_space(inode, freed);
}
/*
@@ -134,7 +159,7 @@
#define SHMEM_MAX_BLOCKS (SHMEM_NR_DIRECT + ENTRIES_PER_PAGE * ENTRIES_PER_PAGE/2*(ENTRIES_PER_PAGE+1))
-static swp_entry_t * shmem_swp_entry (struct shmem_inode_info *info, unsigned long index, unsigned long page)
+static swp_entry_t * shmem_swp_entry (shmem_info_t *info, unsigned long index, unsigned long page)
{
unsigned long offset;
void **dir;
@@ -181,7 +206,7 @@
* @info: info structure for the inode
* @index: index of the page to find
*/
-static inline swp_entry_t * shmem_alloc_entry (struct shmem_inode_info *info, unsigned long index)
+static inline swp_entry_t * shmem_alloc_entry (shmem_info_t *info, unsigned long index)
{
unsigned long page = 0;
swp_entry_t * res;
@@ -290,7 +315,7 @@
* then shmem_truncate_direct to do the real work
*/
static inline unsigned long
-shmem_truncate_indirect(struct shmem_inode_info *info, unsigned long index)
+shmem_truncate_indirect(shmem_info_t *info, unsigned long index)
{
swp_entry_t ***base;
unsigned long baseidx, len, start;
@@ -328,40 +353,61 @@
return shmem_truncate_direct(base, start, len);
}
-static void shmem_truncate (struct inode * inode)
+static void shmem_truncate(struct inode *inode)
{
- unsigned long index;
- unsigned long freed = 0;
- struct shmem_inode_info * info = SHMEM_I(inode);
+ shmem_info_t * info = SHMEM_I(inode);
+ unsigned long freed = 0, index, i;
+ struct page *page;
down(&info->sem);
inode->i_ctime = inode->i_mtime = CURRENT_TIME;
- spin_lock (&info->lock);
- index = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
+ spin_lock(&info->lock);
+ if (!info->bigpages) {
+ index = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
- while (index < info->next_index)
- freed += shmem_truncate_indirect(info, index);
+ while (index < info->next_index)
+ freed += shmem_truncate_indirect(info, index);
- info->swapped -= freed;
- shmem_recalc_inode(inode);
- spin_unlock (&info->lock);
+ info->swapped -= freed;
+ shmem_recalc_inode(inode);
+ } else {
+ index = (inode->i_size + BIGPAGE_SIZE - 1) >> BIGPAGE_SHIFT;
+
+ for (i = index; i < info->max_bigpages; i++) {
+ page = info->bigpages[i];
+ if (page) {
+ info->bigpages[i] = NULL;
+ free_bigpage(page);
+ shm_free_space(inode, BIGPAGE_PAGES);
+ }
+ }
+ }
+ spin_unlock(&info->lock);
up(&info->sem);
}
static void shmem_delete_inode(struct inode * inode)
{
struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+ shmem_info_t *info = SHMEM_I(inode);
inode->i_size = 0;
- if (inode->i_op->truncate == shmem_truncate){
- spin_lock (&shmem_ilock);
- list_del (&SHMEM_I(inode)->list);
- spin_unlock (&shmem_ilock);
- shmem_truncate (inode);
+ if (inode->i_op->truncate == shmem_truncate) {
+ spin_lock(&shmem_ilock);
+ list_del(&info->list);
+ spin_unlock(&shmem_ilock);
+ shmem_truncate(inode);
+ if (info->bigpages) {
+ kfree(info->bigpages);
+ info->bigpages = NULL;
+ info->max_bigpages = 0;
+ }
}
- spin_lock (&sbinfo->stat_lock);
+ if (info->bigpages)
+ BUG();
+ spin_lock(&sbinfo->stat_lock);
sbinfo->free_inodes++;
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
clear_inode(inode);
}
@@ -378,14 +424,14 @@
return -1;
}
-static int shmem_unuse_inode (struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
+static int shmem_unuse_inode (shmem_info_t *info, swp_entry_t entry, struct page *page)
{
swp_entry_t *ptr;
unsigned long idx;
int offset;
idx = 0;
- spin_lock (&info->lock);
+ spin_lock(&info->lock);
offset = shmem_clear_swp (entry, info->i_direct, SHMEM_NR_DIRECT);
if (offset >= 0)
goto found;
@@ -399,7 +445,7 @@
if (offset >= 0)
goto found;
}
- spin_unlock (&info->lock);
+ spin_unlock(&info->lock);
return 0;
found:
add_to_page_cache(page, info->inode->i_mapping, offset + idx);
@@ -418,16 +464,18 @@
void shmem_unuse(swp_entry_t entry, struct page *page)
{
struct list_head *p;
- struct shmem_inode_info * info;
+ shmem_info_t * info;
- spin_lock (&shmem_ilock);
+ if (BigPage(page))
+ BUG();
+ spin_lock(&shmem_ilock);
list_for_each(p, &shmem_inodes) {
- info = list_entry(p, struct shmem_inode_info, list);
+ info = list_entry(p, shmem_info_t, list);
if (shmem_unuse_inode(info, entry, page))
break;
}
- spin_unlock (&shmem_ilock);
+ spin_unlock(&shmem_ilock);
}
/*
@@ -440,12 +488,14 @@
static int shmem_writepage(struct page * page)
{
int error;
- struct shmem_inode_info *info;
+ shmem_info_t *info;
swp_entry_t *entry, swap;
struct inode *inode;
if (!PageLocked(page))
BUG();
+ if (BigPage(page))
+ BUG();
inode = page->mapping->host;
info = SHMEM_I(inode);
@@ -494,13 +544,14 @@
* still need to guard against racing with shm_writepage(), which might
* be trying to move the page to the swap cache as we run.
*/
-static struct page * shmem_getpage_locked(struct shmem_inode_info *info, struct inode * inode, unsigned long idx)
+static struct page * shmem_getpage_locked(shmem_info_t *info, struct inode * inode, unsigned long idx)
{
struct address_space * mapping = inode->i_mapping;
- struct shmem_sb_info *sbinfo;
struct page * page;
swp_entry_t *entry;
+ if (info->bigpages)
+ BUG();
repeat:
page = find_lock_page(mapping, idx);
if (page)
@@ -510,7 +561,7 @@
if (IS_ERR(entry))
return (void *)entry;
- spin_lock (&info->lock);
+ spin_lock(&info->lock);
/* The shmem_alloc_entry() call may have blocked, and
* shmem_writepage may have been moving a page between the page
@@ -521,7 +572,7 @@
if (page) {
if (TryLockPage(page))
goto wait_retry;
- spin_unlock (&info->lock);
+ spin_unlock(&info->lock);
return page;
}
@@ -533,7 +584,7 @@
page = __find_get_page(&swapper_space, entry->val,
page_hash(&swapper_space, entry->val));
if (!page) {
- spin_unlock (&info->lock);
+ spin_unlock(&info->lock);
lock_kernel();
swapin_readahead(*entry);
page = read_swap_cache_async(*entry);
@@ -563,15 +614,12 @@
page->flags = flags | (1 << PG_dirty);
add_to_page_cache_locked(page, mapping, idx);
info->swapped--;
- spin_unlock (&info->lock);
+ spin_unlock(&info->lock);
} else {
- sbinfo = SHMEM_SB(inode->i_sb);
- spin_unlock (&info->lock);
- spin_lock (&sbinfo->stat_lock);
- if (sbinfo->free_blocks == 0)
- goto no_space;
- sbinfo->free_blocks--;
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&info->lock);
+
+ if (shm_alloc_space(inode, 1))
+ return ERR_PTR(-ENOSPC);
/* Ok, get a new page. We don't have to worry about the
* info->lock spinlock here: we cannot race against
@@ -581,10 +629,11 @@
* new shm entry. The inode semaphore we already hold
* is enough to make this atomic. */
page = page_cache_alloc(mapping);
- if (!page)
+ if (!page) {
+ shm_free_space(inode, 1);
return ERR_PTR(-ENOMEM);
+ }
clear_highpage(page);
- inode->i_blocks += BLOCKS_PER_PAGE;
add_to_page_cache (page, mapping, idx);
}
@@ -594,23 +643,63 @@
if (info->locked)
page_cache_get(page);
return page;
-no_space:
- spin_unlock (&sbinfo->stat_lock);
- return ERR_PTR(-ENOSPC);
wait_retry:
- spin_unlock (&info->lock);
+ spin_unlock(&info->lock);
wait_on_page(page);
page_cache_release(page);
goto repeat;
}
+static struct page * shmem_getbigpage_locked(shmem_info_t *info, struct inode * inode, unsigned long idx)
+{
+ unsigned long bigidx, offset;
+ struct page *page;
+
+ bigidx = idx / BIGPAGE_PAGES;
+ offset = idx % BIGPAGE_PAGES;
+
+ if (bigidx >= info->max_bigpages)
+ return ERR_PTR(-ENOSPC);
+got_bigpage:
+ page = info->bigpages[bigidx];
+ if (page) {
+ page += offset;
+ get_page(page);
+ if (!BigPage(page))
+ BUG();
+ lock_page(page);
+ return page;
+ }
+
+ if (shm_alloc_space(inode, BIGPAGE_PAGES))
+ return ERR_PTR(-ENOSPC);
+
+ page = alloc_bigpage();
+ if (!page) {
+ shm_free_space(inode, BIGPAGE_PAGES);
+ return ERR_PTR(-ENOSPC);
+ }
+
+ spin_lock(&info->lock);
+ if (info->bigpages[bigidx]) {
+ spin_unlock(&info->lock);
+ free_bigpage(page);
+ shm_free_space(inode, BIGPAGE_PAGES);
+ goto got_bigpage;
+ }
+ info->bigpages[bigidx] = page;
+ spin_unlock(&info->lock);
+
+ goto got_bigpage;
+}
+
static int shmem_getpage(struct inode * inode, unsigned long idx, struct page **ptr)
{
- struct shmem_inode_info *info = SHMEM_I(inode);
+ shmem_info_t *info = SHMEM_I(inode);
int error;
- down (&info->sem);
+ down(&info->sem);
*ptr = ERR_PTR(-EFAULT);
if (inode->i_size <= (loff_t) idx * PAGE_CACHE_SIZE)
goto failed;
@@ -620,10 +709,36 @@
goto failed;
UnlockPage(*ptr);
- up (&info->sem);
+ up(&info->sem);
+ return 0;
+failed:
+ up(&info->sem);
+ error = PTR_ERR(*ptr);
+ *ptr = NOPAGE_SIGBUS;
+ if (error == -ENOMEM)
+ *ptr = NOPAGE_OOM;
+ return error;
+}
+
+static int shmem_getbigpage(struct inode * inode, unsigned long idx, struct page **ptr)
+{
+ shmem_info_t *info = SHMEM_I(inode);
+ int error;
+
+ down(&info->sem);
+ *ptr = ERR_PTR(-EFAULT);
+ if (inode->i_size <= (loff_t) idx * PAGE_SIZE)
+ goto failed;
+
+ *ptr = shmem_getbigpage_locked(info, inode, idx);
+ if (IS_ERR (*ptr))
+ goto failed;
+
+ UnlockPage(*ptr);
+ up(&info->sem);
return 0;
failed:
- up (&info->sem);
+ up(&info->sem);
error = PTR_ERR(*ptr);
*ptr = NOPAGE_SIGBUS;
if (error == -ENOMEM)
@@ -633,10 +748,31 @@
struct page * shmem_nopage(struct vm_area_struct * vma, unsigned long address, int no_share)
{
- struct page * page;
- unsigned int idx;
struct inode * inode = vma->vm_file->f_dentry->d_inode;
+ int bigpage = vma->vm_flags & VM_BIGPAGE;
+ unsigned long idx, bigidx;
+ struct page * page = NULL;
+
+ if (I_BIGPAGE(inode)) {
+ if (no_share)
+ BUG();
+ idx = (address - vma->vm_start) >> PAGE_SHIFT;
+ idx += vma->vm_pgoff;
+
+ if (shmem_getbigpage(inode, idx, &page))
+ return page;
+
+ if (bigpage) {
+ put_page(page);
+ bigidx = idx / BIGPAGE_PAGES;
+ if (bigidx >= SHMEM_I(inode)->max_bigpages)
+ BUG();
+ page = SHMEM_I(inode)->bigpages[bigidx];
+ get_page(page);
+ }
+ return page;
+ }
idx = (address - vma->vm_start) >> PAGE_CACHE_SHIFT;
idx += vma->vm_pgoff;
@@ -662,7 +798,7 @@
void shmem_lock(struct file * file, int lock)
{
struct inode * inode = file->f_dentry->d_inode;
- struct shmem_inode_info * info = SHMEM_I(inode);
+ shmem_info_t * info = SHMEM_I(inode);
struct page * page;
unsigned long idx, size;
@@ -670,6 +806,8 @@
if (info->locked == lock)
goto out;
info->locked = lock;
+ if (SHMEM_I(inode)->bigpages)
+ goto out;
size = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
for (idx = 0; idx < size; idx++) {
page = find_lock_page(inode->i_mapping, idx);
@@ -686,32 +824,97 @@
up(&info->sem);
}
+
+int shmem_make_bigpage_mmap(struct file * file, struct vm_area_struct * vma)
+{
+ struct inode *inode = file->f_dentry->d_inode;
+ unsigned long pages;
+ shmem_info_t *info;
+ int bigpage;
+
+ /*
+ * COW of 4MB/2MB pages is ... an interesting concept. Disallow it.
+ */
+ if (!(vma->vm_flags & VM_SHARED))
+ return -EINVAL;
+
+ info = SHMEM_I(inode);
+ /*
+ * Make sure the bigpage area is properly aligned and
+ * properly sized, both on the virtual and on the
+ * physical side.
+ */
+ bigpage = 0;
+ if ((vma->vm_flags & VM_BIGMAP) || shm_use_bigpages)
+ bigpage = 1;
+ if (vma->vm_start & BIGPAGE_MASK)
+ bigpage = 0;
+ if (vma->vm_end & BIGPAGE_MASK)
+ bigpage = 0;
+ if (vma->vm_pgoff % BIGPAGE_PAGES)
+ bigpage = 0;
+ if (!bigpage && (vma->vm_flags & VM_BIGMAP))
+ return -EINVAL;
+
+ pages = (vma->vm_end - vma->vm_start) / PAGE_SIZE + vma->vm_pgoff;
+ pages >>= (BIGPAGE_SHIFT - PAGE_SHIFT);
+ if (pages > info->max_bigpages)
+ return -ENOSPC;
+
+ vma->vm_flags |= VM_LOCKED;
+ if (bigpage)
+ vma->vm_flags |= VM_BIGPAGE;
+ return 0;
+}
+
static int shmem_mmap(struct file * file, struct vm_area_struct * vma)
{
- struct vm_operations_struct * ops;
+ struct vm_operations_struct * ops = &shmem_vm_ops;
struct inode *inode = file->f_dentry->d_inode;
- ops = &shmem_vm_ops;
if (!inode->i_sb || !S_ISREG(inode->i_mode))
return -EACCES;
+ if (I_BIGPAGE(inode)) {
+ int error = shmem_make_bigpage_mmap(file, vma);
+ if (error)
+ return error;
+ }
UPDATE_ATIME(inode);
vma->vm_ops = ops;
+ vma->vm_flags &= ~VM_BIGMAP;
+ return 0;
+}
+
+int shmem_munmap(struct vm_area_struct * vma, unsigned long addr, size_t size)
+{
+ int bigpage = vma->vm_flags & VM_BIGPAGE;
+
+ /*
+ * Make sure the unmapped bigpage area is properly aligned and
+ * properly sized:
+ */
+ if (bigpage) {
+ if (addr & BIGPAGE_MASK)
+ return -EINVAL;
+ if (size & BIGPAGE_MASK)
+ return -EINVAL;
+ }
return 0;
}
struct inode *shmem_get_inode(struct super_block *sb, int mode, int dev)
{
struct inode * inode;
- struct shmem_inode_info *info;
+ shmem_info_t *info;
struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
- spin_lock (&sbinfo->stat_lock);
+ spin_lock(&sbinfo->stat_lock);
if (!sbinfo->free_inodes) {
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
return NULL;
}
sbinfo->free_inodes--;
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
inode = new_inode(sb);
if (inode) {
@@ -727,6 +930,8 @@
info->inode = inode;
spin_lock_init (&info->lock);
sema_init (&info->sem, 1);
+ if (info->bigpages)
+ BUG();
switch (mode & S_IFMT) {
default:
init_special_inode(inode, mode, dev);
@@ -734,9 +939,9 @@
case S_IFREG:
inode->i_op = &shmem_inode_operations;
inode->i_fop = &shmem_file_operations;
- spin_lock (&shmem_ilock);
- list_add (&SHMEM_I(inode)->list, &shmem_inodes);
- spin_unlock (&shmem_ilock);
+ spin_lock(&shmem_ilock);
+ list_add(&SHMEM_I(inode)->list, &shmem_inodes);
+ spin_unlock(&shmem_ilock);
break;
case S_IFDIR:
inode->i_nlink++;
@@ -783,14 +988,23 @@
shmem_file_write(struct file *file,const char *buf,size_t count,loff_t *ppos)
{
struct inode *inode = file->f_dentry->d_inode;
- struct shmem_inode_info *info;
+ shmem_info_t *info = SHMEM_I(inode);
unsigned long limit = current->rlim[RLIMIT_FSIZE].rlim_cur;
+ unsigned int page_size, page_shift;
loff_t pos;
struct page *page;
unsigned long written;
long status;
int err;
+ if (I_BIGPAGE(inode)) {
+ page_size = PAGE_SIZE;
+ page_shift = PAGE_SHIFT;
+ } else {
+ page_size = PAGE_CACHE_SIZE;
+ page_shift = PAGE_CACHE_SHIFT;
+ }
+
if ((ssize_t) count < 0)
return -EINVAL;
@@ -845,9 +1059,9 @@
* Try to find the page in the cache. If it isn't there,
* allocate a free page.
*/
- offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
- index = pos >> PAGE_CACHE_SHIFT;
- bytes = PAGE_CACHE_SIZE - offset;
+ offset = (pos & (page_size -1)); /* Within page */
+ index = pos >> page_shift;
+ bytes = page_size - offset;
if (bytes > count) {
bytes = count;
deactivate = 0;
@@ -859,24 +1073,26 @@
* same page as we're writing to, without it being marked
* up-to-date.
*/
- { volatile unsigned char dummy;
+ {
+ volatile unsigned char dummy;
__get_user(dummy, buf);
__get_user(dummy, buf+bytes-1);
}
- info = SHMEM_I(inode);
- down (&info->sem);
- page = shmem_getpage_locked(info, inode, index);
- up (&info->sem);
+ down(&info->sem);
+ if (I_BIGPAGE(inode))
+ page = shmem_getbigpage_locked(info, inode, index);
+ else
+ page = shmem_getpage_locked(info, inode, index);
+ up(&info->sem);
status = PTR_ERR(page);
if (IS_ERR(page))
break;
/* We have exclusive IO access to the page.. */
- if (!PageLocked(page)) {
+ if (!PageLocked(page))
PAGE_BUG(page);
- }
kaddr = kmap(page);
status = copy_from_user(kaddr+offset, buf, bytes);
@@ -899,7 +1115,10 @@
UnlockPage(page);
if (deactivate)
deactivate_page(page);
- page_cache_release(page);
+ if (I_BIGPAGE(inode))
+ __free_page(page);
+ else
+ page_cache_release(page);
if (status < 0)
break;
@@ -921,29 +1140,43 @@
{
struct inode *inode = filp->f_dentry->d_inode;
struct address_space *mapping = inode->i_mapping;
+ unsigned int page_size, page_shift, page_mask;
unsigned long index, offset;
int nr = 1;
- index = *ppos >> PAGE_CACHE_SHIFT;
- offset = *ppos & ~PAGE_CACHE_MASK;
+ if (I_BIGPAGE(inode)) {
+ page_size = PAGE_SIZE;
+ page_shift = PAGE_SHIFT;
+ page_mask = PAGE_MASK;
+ } else {
+ page_size = PAGE_CACHE_SIZE;
+ page_shift = PAGE_CACHE_SHIFT;
+ page_mask = PAGE_CACHE_MASK;
+ }
+ index = *ppos >> page_shift;
+ offset = *ppos & ~page_mask;
while (nr && desc->count) {
struct page *page;
unsigned long end_index, nr;
- end_index = inode->i_size >> PAGE_CACHE_SHIFT;
+ end_index = inode->i_size >> page_shift;
if (index > end_index)
break;
- nr = PAGE_CACHE_SIZE;
+ nr = page_size;
if (index == end_index) {
- nr = inode->i_size & ~PAGE_CACHE_MASK;
+ nr = inode->i_size & ~page_mask;
if (nr <= offset)
break;
}
nr = nr - offset;
- if ((desc->error = shmem_getpage(inode, index, &page)))
+ if (I_BIGPAGE(inode))
+ desc->error = shmem_getbigpage(inode, index, &page);
+ else
+ desc->error = shmem_getpage(inode, index, &page);
+ if (desc->error)
break;
if (mapping->i_mmap_shared != NULL)
@@ -961,13 +1194,16 @@
*/
nr = file_read_actor(desc, page, offset, nr);
offset += nr;
- index += offset >> PAGE_CACHE_SHIFT;
- offset &= ~PAGE_CACHE_MASK;
-
- page_cache_release(page);
+ index += offset >> page_shift;
+ offset &= ~page_mask;
+
+ if (I_BIGPAGE(inode))
+ __free_page(page);
+ else
+ page_cache_release(page);
}
- *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
+ *ppos = ((loff_t) index << page_shift) + offset;
UPDATE_ATIME(inode);
}
@@ -1002,12 +1238,12 @@
buf->f_type = TMPFS_MAGIC;
buf->f_bsize = PAGE_CACHE_SIZE;
- spin_lock (&sbinfo->stat_lock);
+ spin_lock(&sbinfo->stat_lock);
buf->f_blocks = sbinfo->max_blocks;
buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
buf->f_files = sbinfo->max_inodes;
buf->f_ffree = sbinfo->free_inodes;
- spin_unlock (&sbinfo->stat_lock);
+ spin_unlock(&sbinfo->stat_lock);
buf->f_namelen = 255;
return 0;
}
@@ -1035,6 +1271,8 @@
d_instantiate(dentry, inode);
dget(dentry); /* Extra count - pin the dentry in core */
error = 0;
+ if (shm_use_bigpages > 1)
+ shm_enable_bigpages(dentry->d_inode, NULL, BIGPAGE_SIZE, 0);
}
return error;
}
@@ -1153,7 +1391,7 @@
struct inode *inode;
struct page *page;
char *kaddr;
- struct shmem_inode_info * info;
+ shmem_info_t * info;
error = shmem_mknod(dir, dentry, S_IFLNK | S_IRWXUGO, 0);
if (error)
@@ -1166,14 +1404,14 @@
inode = dentry->d_inode;
info = SHMEM_I(inode);
inode->i_size = len;
- if (len <= sizeof(struct shmem_inode_info)) {
+ if (len <= sizeof(shmem_info_t)) {
/* do it inline */
memcpy(info, symname, len);
inode->i_op = &shmem_symlink_inline_operations;
} else {
- spin_lock (&shmem_ilock);
- list_add (&info->list, &shmem_inodes);
- spin_unlock (&shmem_ilock);
+ spin_lock(&shmem_ilock);
+ list_add(&info->list, &shmem_inodes);
+ spin_unlock(&shmem_ilock);
down(&info->sem);
page = shmem_getpage_locked(info, inode, 0);
if (IS_ERR(page)) {
@@ -1299,6 +1537,7 @@
{
return 0;
}
+
#endif
static struct super_block *shmem_read_super(struct super_block * sb, void * data, int silent)
@@ -1347,7 +1586,70 @@
return sb;
}
+/*
+ * Limit kmalloc() size.
+ */
+#define MAX_BIGPAGES (16000/sizeof(void *))
+
+void shm_enable_bigpages(struct inode *inode, struct file *filp, size_t size, int prealloc)
+{
+ struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+ shmem_info_t * info = SHMEM_I(inode);
+ unsigned long pages, i, j;
+
+ if (!S_ISREG(inode->i_mode))
+ return;
+ if ((size < 1 /*BIGPAGE_SIZE*/) || info->bigpages || !nr_bigpages)
+ return;
+ info->max_bigpages = sbinfo->max_blocks >> (BIGPAGE_SHIFT - PAGE_CACHE_SHIFT);
+ if (info->max_bigpages > MAX_BIGPAGES)
+ info->max_bigpages = MAX_BIGPAGES;
+ if (!info->max_bigpages)
+ return;
+ info->bigpages = (struct page **) kmalloc(info->max_bigpages * sizeof(struct page *), GFP_KERNEL);
+ if (!info->bigpages)
+ return;
+ memset(info->bigpages, 0, info->max_bigpages * sizeof(struct page *));
+ if (prealloc) {
+ pages = size / BIGPAGE_SIZE;
+ for (i = 0; i < pages; i++) {
+ if (!shm_alloc_space(inode, BIGPAGE_PAGES)) {
+ info->bigpages[i] = alloc_bigpage();
+ if (!info->bigpages[i])
+ shm_free_space(inode, BIGPAGE_PAGES);
+ }
+ if (!info->bigpages[i]) {
+ for (j = 0; j < i; j++) {
+ shm_free_space(inode, BIGPAGE_PAGES);
+ free_bigpage(info->bigpages[j]);
+ info->bigpages[j] = NULL;
+ }
+ kfree(info->bigpages);
+ info->max_bigpages = 0;
+ info->bigpages = NULL;
+ return;
+ }
+ }
+ }
+}
+static int shmem_ioctl(struct inode * inode, struct file * filp,
+ unsigned int cmd, unsigned long arg)
+{
+ switch (cmd) {
+ case SHMEM_IOC_SETBIGPAGES:
+ if (!HAVE_ARCH_BIGPAGES || !nr_bigpages)
+ return -EINVAL;
+ if (inode->i_size || SHMEM_I(inode)->swapped)
+ return -EINVAL;
+ down(&inode->i_sem);
+ shm_enable_bigpages(inode, filp, BIGPAGE_SIZE, 0);
+ up(&inode->i_sem);
+ break;
+ default:
+ }
+ return 0;
+}
static struct address_space_operations shmem_aops = {
removepage: shmem_removepage,
@@ -1356,10 +1658,12 @@
static struct file_operations shmem_file_operations = {
mmap: shmem_mmap,
+ munmap: shmem_munmap,
#ifdef CONFIG_TMPFS
read: shmem_file_read,
write: shmem_file_write,
fsync: shmem_sync_file,
+ ioctl: shmem_ioctl,
#endif
};
@@ -1523,6 +1827,8 @@
file = shmem_file_setup("dev/zero", size);
if (IS_ERR(file))
return PTR_ERR(file);
+ if (shm_use_bigpages)
+ shm_enable_bigpages(file->f_dentry->d_inode, file, BIGPAGE_SIZE, 0);
if (vma->vm_file)
fput (vma->vm_file);
--- linux/mm/mlock.c.orig Sun Feb 24 21:18:24 2002
+++ linux/mm/mlock.c Sun Feb 24 21:18:51 2002
@@ -162,6 +162,8 @@
vma = find_vma(current->mm, start);
if (!vma || vma->vm_start > start)
return -ENOMEM;
+ if (vma->vm_flags & VM_BIGPAGE)
+ return -EINVAL;
for (nstart = start ; ; ) {
unsigned int newflags;
--- linux/mm/mmap.c.orig Sun Feb 24 17:10:07 2002
+++ linux/mm/mmap.c Wed Mar 6 20:47:33 2002
@@ -199,6 +199,7 @@
flag_bits =
_trans(flags, MAP_GROWSDOWN, VM_GROWSDOWN) |
_trans(flags, MAP_DENYWRITE, VM_DENYWRITE) |
+ _trans(flags, MAP_BIGPAGE, VM_BIGMAP) |
_trans(flags, MAP_EXECUTABLE, VM_EXECUTABLE);
return prot_bits | flag_bits;
#undef _trans
@@ -329,7 +330,7 @@
vma->vm_start = addr;
vma->vm_end = addr + len;
vma->vm_flags = vm_flags;
- vma->vm_page_prot = protection_map[vm_flags & 0x0f];
+ vma->vm_page_prot = protection_map[vm_flags & MAP_TYPE];
vma->vm_ops = NULL;
vma->vm_pgoff = pgoff;
vma->vm_file = NULL;
@@ -348,9 +349,18 @@
error = file->f_op->mmap(file, vma);
if (error)
goto unmap_and_free_vma;
- } else if (flags & MAP_SHARED) {
- error = shmem_zero_setup(vma);
- if (error)
+ if (vma->vm_flags & VM_BIGMAP) {
+ error = -EINVAL;
+ goto unmap_and_free_vma;
+ }
+ } else {
+ if (flags & MAP_SHARED) {
+ error = shmem_zero_setup(vma);
+ if (error)
+ goto free_vma;
+ }
+ error = -EINVAL;
+ if (vma->vm_flags & VM_BIGMAP)
goto free_vma;
}
@@ -751,6 +761,10 @@
if ((mpnt->vm_start < addr && mpnt->vm_end > addr+len)
&& mm->map_count >= max_map_count)
return -ENOMEM;
+ if (mpnt->vm_file && mpnt->vm_file->f_op &&
+ mpnt->vm_file->f_op->munmap)
+ if (mpnt->vm_file->f_op->munmap(mpnt, addr, len))
+ return 0; // FIXME: -EINVAL;
/*
* We may need one additional vma to fix up the mappings ...
@@ -1056,8 +1070,13 @@
mm->map_count--;
__remove_shared_vm_struct(next);
spin_unlock(&mm->page_table_lock);
+ if (next->vm_ops && next->vm_ops->close)
+ next->vm_ops->close(vma);
unlock_vma_mappings(vma);
-
+ if (next->vm_file) {
+ fput(next->vm_file);
+ next->vm_file = NULL;
+ }
kmem_cache_free(vm_area_cachep, next);
}
--- linux/mm/memory.c.orig Sun Feb 24 18:05:08 2002
+++ linux/mm/memory.c Thu Mar 7 03:23:03 2002
@@ -82,6 +82,8 @@
if (pmd_none(*dir))
return;
+ if (pmd_bigpage(*dir))
+ return;
if (pmd_bad(*dir)) {
pmd_ERROR(*dir);
pmd_clear(dir);
@@ -179,6 +181,7 @@
unsigned long address = vma->vm_start;
unsigned long end = vma->vm_end;
unsigned long cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
+ int bigpage = (vma->vm_flags & VM_BIGPAGE);
src_pgd = pgd_offset(src, address)-1;
dst_pgd = pgd_offset(dst, address)-1;
@@ -213,6 +216,18 @@
if (pmd_none(*src_pmd))
goto skip_copy_pte_range;
+ if (bigpage) {
+ if (!pmd_bigpage(*src_pmd))
+ pmd_clear(dst_pmd);
+ else
+ *dst_pmd = *src_pmd;
+ address += PMD_SIZE;
+ if (address >= end)
+ goto out;
+ goto cont_copy_pmd_range;
+ }
+ if (pmd_bigpage(*src_pmd))
+ BUG();
if (pmd_bad(*src_pmd)) {
pmd_ERROR(*src_pmd);
pmd_clear(src_pmd);
@@ -297,11 +312,13 @@
static inline int zap_pte_range(mmu_gather_t *tlb, pmd_t * pmd, unsigned long address, unsigned long size)
{
unsigned long offset;
- pte_t *mapped, *ptep;
int freed = 0;
+ pte_t *ptep;
if (pmd_none(*pmd))
return 0;
+ if (pmd_bigpage(*pmd))
+ BUG();
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
@@ -351,7 +368,10 @@
end = ((address + PGDIR_SIZE) & PGDIR_MASK);
freed = 0;
do {
- freed += zap_pte_range(tlb, pmd, address, end - address);
+ if (pmd_bigpage(*pmd))
+ pmd_clear(pmd);
+ else
+ freed += zap_pte_range(tlb, pmd, address, end - address);
address = (address + PMD_SIZE) & PMD_MASK;
pmd++;
} while (address < end);
@@ -403,11 +423,10 @@
spin_unlock(&mm->page_table_lock);
}
-
/*
* Do a quick page-table lookup for a single page.
*/
-static struct page * follow_page(unsigned long address, int write)
+struct page * follow_page(unsigned long address, int write)
{
pgd_t *pgd;
pmd_t *pmd;
@@ -418,6 +437,8 @@
goto out;
pmd = pmd_offset(pgd, address);
+ if (pmd_bigpage(*pmd))
+ return pmd_page(*pmd) + (address & BIGPAGE_MASK) / PAGE_SIZE;
if (pmd_none(*pmd) || pmd_bad(*pmd))
goto out;
@@ -1381,6 +1402,40 @@
return 1;
}
+static int handle_bigpage_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long long address, int write_access, pmd_t *pmd)
+{
+ struct inode *inode = vma->vm_file->f_dentry->d_inode;
+ struct page *new_page;
+
+ spin_unlock(&mm->page_table_lock);
+ down_read(&inode->i_truncate_sem);
+ new_page = vma->vm_ops->nopage(vma, address & PAGE_MASK, (vma->vm_flags & VM_SHARED)?0:write_access);
+ up_read(&inode->i_truncate_sem);
+ spin_lock(&mm->page_table_lock);
+
+ if (!new_page)
+ return 0;
+ if (new_page == NOPAGE_OOM)
+ return -1;
+ if (!BigPage(new_page))
+ BUG();
+ /*
+ * Another context was faster.
+ */
+ if (pmd_present(*pmd)) {
+ if (pmd_bigpage(*pmd))
+ return 1;
+ free_one_pmd(pmd);
+ }
+ /*
+ * Major fault.
+ */
+ pmd_populate_bigpage(mm, pmd, new_page);
+ flush_tlb_page(vma, address & PMD_MASK);
+ update_mmu_cache(vma, address, entry);
+ return 2;
+}
+
/*
* By the time we get here, we already hold the mm semaphore
*/
@@ -1390,6 +1445,7 @@
int ret = -1;
pgd_t *pgd;
pmd_t *pmd;
+ int bigpage = (vma->vm_flags & VM_BIGPAGE);
current->state = TASK_RUNNING;
pgd = pgd_offset(mm, address);
@@ -1401,13 +1457,15 @@
spin_lock(&mm->page_table_lock);
pmd = pmd_alloc(mm, pgd, address);
- if (pmd) {
+ if (pmd && !bigpage) {
pte_t * pte = pte_alloc_map(mm, pmd, address);
if (pte) {
ret = handle_pte_fault(mm, vma, address, write_access, &pte, pmd);
pte_unmap(pte);
}
- }
+ } else
+ if (pmd)
+ ret = handle_bigpage_fault(mm, vma, address, write_access, pmd);
spin_unlock(&mm->page_table_lock);
return ret;
}
@@ -1712,6 +1770,8 @@
if (!pgd_none(*pgd)) {
pmd = pmd_offset(pgd, addr);
if (!pmd_none(*pmd)) {
+ if (pmd_bigpage(*pmd))
+ BUG();
ptep = pte_offset_map(pmd, addr);
pte = *ptep;
if (pte_present(pte))
--- linux/mm/mremap.c.orig Sun Feb 24 19:59:04 2002
+++ linux/mm/mremap.c Mon Feb 25 18:17:48 2002
@@ -245,6 +245,11 @@
vma = find_vma(current->mm, addr);
if (!vma || vma->vm_start > addr)
goto out;
+ /*
+ * Do not remap bigpages, yet.
+ */
+ if (vma->vm_flags & VM_BIGPAGE)
+ goto out;
/* We can't remap across vm area boundaries */
if (old_len > vma->vm_end - addr)
goto out;
--- linux/mm/mprotect.c.orig Sun Feb 24 20:04:35 2002
+++ linux/mm/mprotect.c Wed Mar 6 00:55:20 2002
@@ -21,6 +21,9 @@
if (pmd_none(*pmd))
return;
+ // FIXME: we now silently 'succeed', we should split up the bigpage.
+ if (pmd_bigpage(*pmd))
+ return;
if (pmd_bad(*pmd)) {
pmd_ERROR(*pmd);
pmd_clear(pmd);
@@ -250,6 +253,10 @@
vma = find_vma(current->mm, start);
error = -EFAULT;
if (!vma || vma->vm_start > start)
+ goto out;
+
+ error = 0;
+ if (vma->vm_flags & VM_BIGPAGE)
goto out;
for (nstart = start ; ; ) {
--- linux/mm/Makefile.orig Sun Feb 24 22:05:59 2002
+++ linux/mm/Makefile Sun Feb 24 13:04:26 2002
@@ -14,7 +14,7 @@
obj-y := memory.o mmap.o filemap.o mprotect.o mlock.o mremap.o \
vmalloc.o slab.o bootmem.o swap.o vmscan.o page_io.o \
page_alloc.o swap_state.o swapfile.o numa.o oom_kill.o \
- shmem.o
+ shmem.o bigpages.o
obj-$(CONFIG_HIGHMEM) += highmem.o
obj-y += wtd.o
--- linux/include/linux/mm.h.orig Sun Feb 24 13:57:11 2002
+++ linux/include/linux/mm.h Thu Mar 7 04:48:18 2002
@@ -102,6 +102,8 @@
#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
#define VM_RESERVED 0x00080000 /* Don't unmap it from swap_out */
+#define VM_BIGPAGE 0x00100000 /* bigpage mappings, no pte's */
+#define VM_BIGMAP 0x00200000 /* user wants bigpage mapping */
#define VM_STACK_FLAGS 0x00000177
@@ -286,6 +288,7 @@
#define PG_inactive_clean 11
#define PG_highmem 12
#define PG_checked 13 /* kill me in 2.5.<early>. */
+#define PG_bigpage 14
/* bits 21-29 unused */
#define PG_arch_1 30
#define PG_reserved 31
@@ -302,6 +305,7 @@
#define TryLockPage(page) test_and_set_bit(PG_locked, &(page)->flags)
#define PageChecked(page) test_bit(PG_checked, &(page)->flags)
#define SetPageChecked(page) set_bit(PG_checked, &(page)->flags)
+#define BigPage(page) test_bit(PG_bigpage, &(page)->flags)
extern void __set_page_dirty(struct page *);
@@ -424,6 +428,10 @@
* The old interface name will be removed in 2.5:
*/
#define get_free_page get_zeroed_page
+
+extern long nr_bigpages;
+extern struct page * FASTCALL(alloc_bigpage(void));
+extern void FASTCALL(free_bigpage(struct page *page));
/*
* There is only one 'core' page-freeing function.
--- linux/include/linux/shmem_fs.h.orig Sun Feb 24 13:06:31 2002
+++ linux/include/linux/shmem_fs.h Tue Mar 5 22:34:13 2002
@@ -19,7 +19,7 @@
extern atomic_t shmem_nrpages;
-struct shmem_inode_info {
+typedef struct shmem_inode_info {
spinlock_t lock;
struct semaphore sem;
unsigned long next_index;
@@ -29,7 +29,9 @@
int locked; /* into memory */
struct list_head list;
struct inode *inode;
-};
+ unsigned long max_bigpages;
+ struct page **bigpages;
+} shmem_info_t;
struct shmem_sb_info {
unsigned long max_blocks; /* How many blocks are allowed */
@@ -39,6 +41,19 @@
spinlock_t stat_lock;
};
-#define SHMEM_I(inode) (&inode->u.shmem_i)
+#define SHMEM_I(inode) (&inode->u.shmem_i)
+#define I_BIGPAGE(inode) (SHMEM_I(inode)->bigpages)
+
+#define SHMEM_IOC_SETBIGPAGES _IO('f', 1)
+
+extern size_t shm_ctlmax;
+extern void shm_enable_bigpages(struct inode *inode, struct file *filp, size_t size, int prealloc);
+extern int shm_use_bigpages;
+extern struct page * shmem_nopage(struct vm_area_struct * vma, unsigned long address, int no_share);
+extern int shmem_make_bigpage_mmap(struct file * file, struct vm_area_struct * vma);
+
+extern void check_fault_page(struct vm_area_struct *vma, unsigned long addr);
+extern struct page * follow_page(unsigned long address, int write);
+extern int shmem_munmap(struct vm_area_struct * vma, unsigned long addr, size_t size);
#endif
--- linux/include/linux/fs.h.orig Mon Feb 25 13:22:41 2002
+++ linux/include/linux/fs.h Thu Mar 7 04:48:15 2002
@@ -855,6 +855,7 @@
ssize_t (*aio_readx)(struct file *, struct kiocb *, struct iocb);
ssize_t (*aio_write)(struct file *, struct kiocb *, struct iocb);
ssize_t (*aio_fsync)(struct file *, struct kiocb *, struct iocb);
+ int (*munmap) (struct vm_area_struct *, unsigned long, size_t);
};
struct inode_operations {
--- linux/include/linux/sysctl.h.orig Tue Feb 26 09:36:17 2002
+++ linux/include/linux/sysctl.h Tue Mar 5 19:41:47 2002
@@ -125,6 +125,7 @@
KERN_CADPID=54, /* int: PID of the process to notify on CAD */
KERN_TAINTED=55, /* int: various kernel tainted flags */
KERN_CHILD_RUNS_FIRST=56, /* int: child-runs-first forking */
+ KERN_SHMUSEBIGPAGES=57, /* int: use bigpages wherever possible */
};
--- linux/include/asm-i386/pgtable.h.orig Sun Feb 24 19:36:09 2002
+++ linux/include/asm-i386/pgtable.h Thu Mar 7 04:48:15 2002
@@ -262,6 +262,14 @@
#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT)
#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0)
#define pmd_bad(x) ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) != _KERNPG_TABLE)
+#define pmd_bigpage(x) (pmd_val(x) & _PAGE_PSE)
+
+#define BIGPAGE_SHIFT (PMD_SHIFT)
+#define BIGPAGE_SIZE (1UL << BIGPAGE_SHIFT)
+#define BIGPAGE_MASK (BIGPAGE_SIZE - 1)
+#define BIGPAGE_PAGES (BIGPAGE_SIZE / PAGE_SIZE)
+
+#define HAVE_ARCH_BIGPAGES cpu_has_pse
/*
* Permanent address of a page. Obviously must never be
--- linux/include/asm-i386/pgalloc.h.orig Sun Feb 24 18:12:48 2002
+++ linux/include/asm-i386/pgalloc.h Thu Mar 7 04:48:15 2002
@@ -16,6 +16,18 @@
((unsigned long long)(pte - mem_map) <<
(unsigned long long) PAGE_SHIFT)));
}
+
+static inline void pmd_populate_bigpage(struct mm_struct *mm, pmd_t *pmd, struct page *page)
+{
+ unsigned int idx = page - mem_map;
+
+ if (idx & ((1 << (BIGPAGE_SHIFT - PAGE_SHIFT)) -1)) {
+ printk("ugh, page idx %d (%p) cannot be PSE page!\n", idx, page);
+ BUG();
+ }
+ set_pmd(pmd, __pmd(_PAGE_TABLE + _PAGE_PSE +
+ ((unsigned long long)idx << (unsigned long long) PAGE_SHIFT)));
+}
/*
* Allocate and free page tables.
*/
--- linux/include/asm-i386/mman.h.orig Mon Feb 25 12:16:26 2002
+++ linux/include/asm-i386/mman.h Mon Feb 25 12:32:19 2002
@@ -11,6 +11,7 @@
#define MAP_TYPE 0x0f /* Mask for type of mapping */
#define MAP_FIXED 0x10 /* Interpret addr exactly */
#define MAP_ANONYMOUS 0x20 /* don't use a file */
+#define MAP_BIGPAGE 0x40 /* bigpage mapping */
#define MAP_GROWSDOWN 0x0100 /* stack-like segment */
#define MAP_DENYWRITE 0x0800 /* ETXTBSY */
--- linux/include/asm-i386/atomic.h.orig Thu Feb 28 11:43:34 2002
+++ linux/include/asm-i386/atomic.h Thu Mar 7 04:48:15 2002
@@ -2,6 +2,7 @@
#define __ARCH_I386_ATOMIC__
#include <linux/config.h>
+#include <asm/page.h>
/*
* Atomic operations that C can't guarantee us. Useful for
--- linux/ipc/shm.c.orig Tue Feb 26 08:39:46 2002
+++ linux/ipc/shm.c Wed Mar 6 11:15:40 2002
@@ -124,6 +124,10 @@
shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
shm_rmid (shp->id);
shmem_lock(shp->shm_file, 0);
+ /*
+ * Unlock the spinlock first, fput() might sleep.
+ */
+ shm_unlock(shp->id);
fput (shp->shm_file);
kfree (shp);
}
@@ -149,22 +153,30 @@
shp->shm_nattch--;
if(shp->shm_nattch == 0 &&
shp->shm_flags & SHM_DEST)
- shm_destroy (shp);
-
- shm_unlock(id);
+ shm_destroy(shp); /* unlocks */
+ else
+ shm_unlock(id);
up (&shm_ids.sem);
}
static int shm_mmap(struct file * file, struct vm_area_struct * vma)
{
+ int error;
+
UPDATE_ATIME(file->f_dentry->d_inode);
+ if (SHMEM_I(file->f_dentry->d_inode)->bigpages) {
+ error = shmem_make_bigpage_mmap(file, vma);
+ if (error)
+ return -EINVAL;
+ }
vma->vm_ops = &shm_vm_ops;
shm_inc(file->f_dentry->d_inode->i_ino);
return 0;
}
static struct file_operations shm_file_operations = {
- mmap: shm_mmap
+ mmap: shm_mmap,
+ munmap: shmem_munmap
};
static struct vm_operations_struct shm_vm_ops = {
@@ -197,10 +209,16 @@
if (IS_ERR(file))
goto no_file;
+ file->f_op = &shm_file_operations;
+ error = -ENOMEM;
+ if (shm_use_bigpages)
+ shm_enable_bigpages(file->f_dentry->d_inode, file, size, 1);
+
error = -ENOSPC;
id = shm_addid(shp);
if(id == -1)
goto no_id;
+
shp->shm_perm.key = key;
shp->shm_flags = (shmflg & S_IRWXUGO);
shp->shm_cprid = current->pid;
@@ -212,7 +230,6 @@
shp->id = shm_buildid(id,shp->shm_perm.seq);
shp->shm_file = file;
file->f_dentry->d_inode->i_ino = shp->id;
- file->f_op = &shm_file_operations;
shm_tot += numpages;
shm_unlock (id);
return shp->id;
@@ -511,11 +528,11 @@
shp->shm_flags |= SHM_DEST;
/* Do not find it any more */
shp->shm_perm.key = IPC_PRIVATE;
+ /* Unlock */
+ shm_unlock(shmid);
} else
- shm_destroy (shp);
+ shm_destroy (shp); /* unlocks */
- /* Unlock */
- shm_unlock(shmid);
up(&shm_ids.sem);
return err;
}
@@ -631,8 +648,9 @@
shp->shm_nattch--;
if(shp->shm_nattch == 0 &&
shp->shm_flags & SHM_DEST)
- shm_destroy (shp);
- shm_unlock(shmid);
+ shm_destroy (shp); /* unlocks */
+ else
+ shm_unlock(shmid);
up (&shm_ids.sem);
*raddr = (unsigned long) user_addr;
@@ -649,18 +667,19 @@
*/
asmlinkage long sys_shmdt (char *shmaddr)
{
+ int error = -EINVAL;
struct mm_struct *mm = current->mm;
struct vm_area_struct *shmd, *shmdnext;
down_write(&mm->mmap_sem);
for (shmd = mm->mmap; shmd; shmd = shmdnext) {
shmdnext = shmd->vm_next;
- if (shmd->vm_ops == &shm_vm_ops
- && shmd->vm_start - (shmd->vm_pgoff << PAGE_SHIFT) == (ulong) shmaddr)
- do_munmap(mm, shmd->vm_start, shmd->vm_end - shmd->vm_start);
+ if ((shmd->vm_ops == &shm_vm_ops) &&
+ (shmd->vm_start == (ulong) shmaddr))
+ error = do_munmap(mm, shmd->vm_start, shmd->vm_end - shmd->vm_start);
}
up_write(&mm->mmap_sem);
- return 0;
+ return error;
}
#ifdef CONFIG_PROC_FS
--- linux/mm/mmap.c~ Tue Mar 26 10:59:01 2002
+++ linux/mm/mmap.c Tue Mar 26 17:00:18 2002
@@ -1070,9 +1070,9 @@
mm->map_count--;
__remove_shared_vm_struct(next);
spin_unlock(&mm->page_table_lock);
- if (next->vm_ops && next->vm_ops->close)
- next->vm_ops->close(vma);
unlock_vma_mappings(vma);
+ if (next->vm_ops && next->vm_ops->close)
+ next->vm_ops->close(next);
if (next->vm_file) {
fput(next->vm_file);
next->vm_file = NULL;
