--- linux/fs/proc/array.c.orig Wed Sep 1 08:29:08 1999 +++ linux/fs/proc/array.c Sun Sep 26 16:31:24 1999 @@ -244,6 +244,8 @@ nr_running, nr_threads, last_pid); } +extern int get_buddy_stat(char * buffer); + static int get_kstat(char * buffer) { int i, len; @@ -319,6 +321,7 @@ kstat.context_swtch, xtime.tv_sec - jif / HZ, total_forks); + len += get_buddy_stat(buffer+len); return len; } --- linux/fs/inode.c.orig Wed Sep 1 08:29:08 1999 +++ linux/fs/inode.c Sun Sep 26 16:31:24 1999 @@ -129,6 +129,7 @@ init_waitqueue_head(&inode->i_wait); INIT_LIST_HEAD(&inode->i_hash); INIT_LIST_HEAD(&inode->i_dentry); + INIT_LIST_HEAD(&inode->i_pages); sema_init(&inode->i_sem, 1); spin_lock_init(&inode->i_shared_lock); } --- linux/mm/memory.c.orig Wed Sep 1 08:29:09 1999 +++ linux/mm/memory.c Sun Sep 26 16:31:24 1999 @@ -52,6 +52,19 @@ unsigned long num_physpages = 0; void * high_memory = NULL; +void clear_page(unsigned long page) +{ + memset((void *)(page), 0, PAGE_SIZE); +} + +void copy_page(unsigned long to, unsigned long from) +{ + memcpy((void *)(to), (void *)(from), PAGE_SIZE); +} + +//#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) +//#define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE) + /* * We special-case the C-O-W ZERO_PAGE, because it's such * a common occurrence (no need to read the page to know @@ -59,8 +72,12 @@ */ static inline void copy_cow_page(unsigned long from, unsigned long to) { + struct page *page; if (from == ZERO_PAGE(to)) { - clear_bigpage(to); + page = mem_map + MAP_NR(to); + if (!PageZeroed(page)) + BUG(); + ClearPageZeroed(page); return; } copy_bigpage(to, from); @@ -810,29 +827,42 @@ } /* - * Ok, we need to copy. Oh, well.. + * First lets see wether we can get a zeroed page atomically. + * (we might as well get a nonzero one) */ - spin_unlock(&tsk->mm->page_table_lock); - new_page = __get_free_page(GFP_BIGUSER); - if (!new_page) - return -1; - spin_lock(&tsk->mm->page_table_lock); - + new_page = 0; + if (old_page == ZERO_PAGE(0)) + new_page = get_atomic_zero_page(); + if (!new_page) { + /* + * Ok, might have to reschedule so lets drop the lock. + */ + spin_unlock(&tsk->mm->page_table_lock); + new_page = __get_free_page(GFP_BIGUSER); + if (!new_page) + return -1; + clear_bigpage(new_page); + spin_lock(&tsk->mm->page_table_lock); + /* + * Re-check the pte - we dropped the lock + */ + if (pte_val(*page_table) != pte_val(pte)) + goto skip_page; + } /* * Re-check the pte - we dropped the lock */ - if (pte_val(*page_table) == pte_val(pte)) { - if (PageReserved(page)) - ++vma->vm_mm->rss; - copy_cow_page(old_page,new_page); - flush_page_to_ram(new_page); - flush_cache_page(vma, address); - set_pte(page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)))); - flush_tlb_page(vma, address); + if (PageReserved(page)) + ++vma->vm_mm->rss; + copy_cow_page(old_page,new_page); + flush_page_to_ram(new_page); + flush_cache_page(vma, address); + set_pte(page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)))); + flush_tlb_page(vma, address); - /* Free the old page.. */ - new_page = old_page; - } + /* Free the old page.. */ + new_page = old_page; +skip_page: spin_unlock(&tsk->mm->page_table_lock); free_page(new_page); return 1; @@ -1017,10 +1047,19 @@ { pte_t entry = pte_wrprotect(mk_pte(ZERO_PAGE(addr), vma->vm_page_prot)); if (write_access) { - unsigned long page = __get_free_page(GFP_BIGUSER); - if (!page) - return -1; - clear_bigpage(page); + struct page *map; + unsigned long page; + page = get_atomic_zero_page(); + if (!page) { + page = __get_free_page(GFP_BIGUSER); + if (!page) + return -1; + } + map = mem_map + MAP_NR(page); + if (!PageZeroed(map)) + clear_bigpage(page); + else + ClearPageZeroed(map); entry = pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot))); vma->vm_mm->rss++; tsk->min_flt++; --- linux/mm/filemap.c.orig Sun Sep 26 12:57:16 1999 +++ linux/mm/filemap.c Sun Sep 26 16:31:24 1999 @@ -75,24 +75,6 @@ atomic_dec(&page_cache_size); } -static void remove_page_from_inode_queue(struct page * page) -{ - struct inode * inode = page->inode; - struct page *prev, *next; - - inode->i_nrpages--; - next = page->next; - prev = page->prev; - if (inode->i_pages == page) - inode->i_pages = next; - if (next) - next->prev = prev; - if (prev) - prev->next = next; - page->next = NULL; - page->prev = NULL; -} - /* * Remove a page from the page cache and free it. Caller has to make * sure the page is locked and that nobody else uses it - or that usage @@ -112,13 +94,17 @@ void invalidate_inode_pages(struct inode * inode) { - struct page ** p; + struct list_head *head, *curr; struct page * page; + head = &inode->i_pages; repeat: spin_lock(&pagecache_lock); - p = &inode->i_pages; - while ((page = *p) != NULL) { + curr = head->next; + + while (curr != head) { + page = list_entry(curr, struct page, list); + curr = curr->next; get_page(page); if (TryLockPage(page)) { spin_unlock(&pagecache_lock); @@ -136,7 +122,6 @@ UnlockPage(page); page_cache_release(page); page_cache_release(page); - } spin_unlock(&pagecache_lock); } @@ -146,15 +131,21 @@ */ void truncate_inode_pages(struct inode * inode, unsigned long start) { - struct page ** p; + struct list_head *head, *curr; + unsigned long offset; struct page * page; int partial = 0; repeat: + head = &inode->i_pages; spin_lock(&pagecache_lock); - p = &inode->i_pages; - while ((page = *p) != NULL) { - unsigned long offset = page->offset; + curr = head->next; + while (curr != head) { + + page = list_entry(curr, struct page, list); + curr = curr->next; + + offset = page->offset; /* page wholly truncated - free it */ if (offset >= start) { @@ -190,7 +181,6 @@ */ goto repeat; } - p = &page->next; /* * there is only one partial page possible. */ @@ -431,16 +421,18 @@ static int do_buffer_fdatasync(struct inode *inode, unsigned long start, unsigned long end, int (*fn)(struct page *)) { - struct page *next; + struct list_head *head, *curr; + struct page *page; int retval = 0; + head = &inode->i_pages; start &= PAGE_MASK; spin_lock(&pagecache_lock); - next = inode->i_pages; - while (next) { - struct page *page = next; - next = page->next; + curr = head->next; + while (curr != head) { + page = list_entry(curr, struct page, list); + curr = curr->next; if (!page->buffers) continue; if (page->offset >= end) @@ -458,7 +450,7 @@ UnlockPage(page); spin_lock(&pagecache_lock); - next = page->next; + curr = page->list.next; page_cache_release(page); } spin_unlock(&pagecache_lock); @@ -487,6 +479,7 @@ struct inode * inode, unsigned long offset, struct page **hash) { + struct page *alias; unsigned long flags; flags = page->flags & ~((1 << PG_uptodate) | (1 << PG_error) | (1 << PG_referenced)); @@ -497,6 +490,9 @@ add_page_to_inode_queue(inode, page); __add_page_to_hash_queue(page, hash); lru_cache_add(page); + alias = __find_page_nolock(inode, offset, *hash); + if (alias != page) + BUG(); } void add_to_page_cache(struct page * page, struct inode * inode, unsigned long offset) @@ -553,7 +549,6 @@ page_cache_release(page); return; } - /* * We arrive here in the unlikely event that someone * raced with us and added our page to the cache first. --- linux/mm/slab.c.orig Wed Sep 1 08:29:09 1999 +++ linux/mm/slab.c Sun Sep 26 16:31:24 1999 @@ -317,10 +317,10 @@ * slab an obj belongs to. With kmalloc(), and kfree(), these are used * to find the cache which an obj belongs to. */ -#define SLAB_SET_PAGE_CACHE(pg, x) ((pg)->next = (struct page *)(x)) -#define SLAB_GET_PAGE_CACHE(pg) ((kmem_cache_t *)(pg)->next) -#define SLAB_SET_PAGE_SLAB(pg, x) ((pg)->prev = (struct page *)(x)) -#define SLAB_GET_PAGE_SLAB(pg) ((kmem_slab_t *)(pg)->prev) +#define SLAB_SET_PAGE_CACHE(pg,x) ((pg)->list.next = (struct list_head *)(x)) +#define SLAB_GET_PAGE_CACHE(pg) ((kmem_cache_t *)(pg)->list.next) +#define SLAB_SET_PAGE_SLAB(pg,x) ((pg)->list.prev = (struct list_head *)(x)) +#define SLAB_GET_PAGE_SLAB(pg) ((kmem_slab_t *)(pg)->list.prev) /* Size description struct for general caches. */ typedef struct cache_sizes { --- linux/mm/page_alloc.c.orig Wed Sep 1 08:29:09 1999 +++ linux/mm/page_alloc.c Sun Sep 26 22:43:04 1999 @@ -4,6 +4,7 @@ * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds * Swap reorganised 29.12.95, Stephen Tweedie * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 + * Background page clearing, Ingo Molnar , Sep 1999 */ #include @@ -40,15 +41,15 @@ #define NR_MEM_LISTS 10 #endif -/* The start of this MUST match the start of "struct page" */ struct free_area_struct { - struct page *next; - struct page *prev; + struct list_head free_list; + struct list_head zero_list; // these are free pages as well, but zeroed + long nr_free; + long nr_zero; unsigned int * map; + unsigned int __dummy; // fill out a cacheline }; -#define memory_head(x) ((struct page *)(x)) - #ifdef CONFIG_BIGMEM #define BIGMEM_LISTS_OFFSET NR_MEM_LISTS static struct free_area_struct free_area[NR_MEM_LISTS*2]; @@ -56,30 +57,6 @@ static struct free_area_struct free_area[NR_MEM_LISTS]; #endif -static inline void init_mem_queue(struct free_area_struct * head) -{ - head->next = memory_head(head); - head->prev = memory_head(head); -} - -static inline void add_mem_queue(struct free_area_struct * head, struct page * entry) -{ - struct page * next = head->next; - - entry->prev = memory_head(head); - entry->next = next; - next->prev = entry; - head->next = entry; -} - -static inline void remove_mem_queue(struct page * entry) -{ - struct page * next = entry->next; - struct page * prev = entry->prev; - next->prev = prev; - prev->next = next; -} - /* * Free_page() adds the page to the free lists. This is optimized for * fast normal cases (no error jumps taken normally). @@ -99,12 +76,69 @@ */ spinlock_t page_alloc_lock = SPIN_LOCK_UNLOCKED; +unsigned int + nr_freed, + nr_allocated, + nr_zero_pages, + nr_zero_freed, + nr_zero_created, + nr_zero_allocated, + nr_zero_buddy_up, + nr_zero_buddy_up_hit, + nr_zero_buddy_up_1_loss, + nr_zero_buddy_up_2_loss, + nr_zero_buddy_down_loss, + nr_zero_alloc_sum, + nr_zero_alloc_hit, + nr_zero_alloc_miss, + nr_zero_alloc_direct_miss, + nr_zero_highorder_freed, +dummy; + +#define memlist_init(x) INIT_LIST_HEAD(x) +#define memlist_add_head list_add +#define memlist_add_tail list_add_tail +#define memlist_del list_del +#define memlist_entry list_entry +#define memlist_next(x) ((x)->next) +#define memlist_prev(x) ((x)->prev) + +//#define inline + +int get_buddy_stat (char * buffer) +{ + int len = 0; + +#define P(x) \ + len += sprintf(buffer+len, #x": %u\n", x); + + P(nr_freed); + P(nr_allocated); + P(nr_zero_pages); + P(nr_zero_freed); + P(nr_zero_created); + P(nr_zero_allocated); + P(nr_zero_buddy_up); + P(nr_zero_buddy_up_hit); + P(nr_zero_buddy_up_1_loss); + P(nr_zero_buddy_up_2_loss); + P(nr_zero_buddy_down_loss); + P(nr_zero_alloc_sum); + P(nr_zero_alloc_hit); + P(nr_zero_alloc_miss); + P(nr_zero_alloc_direct_miss); + P(nr_zero_highorder_freed); + + return len; +} + static inline void free_pages_ok(unsigned long map_nr, unsigned long order) { struct free_area_struct *area = free_area + order; unsigned long index = map_nr >> (1 + order); unsigned long mask = (~0UL) << order; unsigned long flags; + struct page *page, *buddy; spin_lock_irqsave(&page_alloc_lock, flags); @@ -118,31 +152,154 @@ #endif map_nr &= mask; nr_free_pages -= mask; + nr_freed += 1<map)) + /* + * the buddy page is still allocated. + */ break; - remove_mem_queue(list(map_nr ^ -mask)); + /* + * Move the buddy up one level. + */ + buddy = list(map_nr ^ -mask); + page = list(map_nr); + + memlist_del(&buddy->list); mask <<= 1; area++; index >>= 1; map_nr &= mask; + + if (PageZeroed(page)) + BUG(); + if (PageZeroed(buddy)) { + (area-1)->nr_zero--; + ClearPageZeroed(buddy); + nr_zero_buddy_up_1_loss++; + nr_zero_pages--; + } else + (area-1)->nr_free--; + if (PageZeroed(list(map_nr))) + BUG(); } - add_mem_queue(area, list(map_nr)); + memlist_add_head(&(list(map_nr))->list, &area->free_list); + area->nr_free++; #undef list spin_unlock_irqrestore(&page_alloc_lock, flags); } +static inline void free_zero_pages_ok(unsigned long map_nr, unsigned long order) +{ + struct free_area_struct *area = free_area + order; + unsigned long index = map_nr >> (1 + order); + unsigned long mask = (~0UL) << order; + unsigned long flags; + struct page *page, *buddy; + int all_zero; + + spin_lock_irqsave(&page_alloc_lock, flags); + +#define list(x) (mem_map+(x)) + +#ifdef CONFIG_BIGMEM + if (map_nr >= bigmem_mapnr) { + area += BIGMEM_LISTS_OFFSET; + nr_free_bigpages -= mask; + } +#endif + map_nr &= mask; + nr_free_pages -= mask; + nr_zero_pages += 1<map)) + /* + * the buddy page is still allocated. + */ + break; + /* + * Move the two buddy pages up one level. + */ + buddy = list(map_nr ^ -mask); + page = list(map_nr); + + memlist_del(&buddy->list); + mask <<= 1; + area++; + index >>= 1; + map_nr &= mask; + + if (PageZeroed(buddy)) + (area-1)->nr_zero--; + else + (area-1)->nr_free--; + if (PageZeroed(page) && PageZeroed(buddy)) { + ClearPageZeroed(page); + ClearPageZeroed(buddy); + SetPageZeroed(list(map_nr)); + nr_zero_buddy_up_hit++; + } else { + all_zero = 0; + if (PageZeroed(page) || PageZeroed(buddy)) { + nr_zero_buddy_up_2_loss++; + nr_zero_pages--; + } + ClearPageZeroed(page); + ClearPageZeroed(buddy); + ClearPageZeroed(list(map_nr)); + } + nr_zero_buddy_up++; + } + if (all_zero) { + memlist_add_head(&(list(map_nr))->list, &area->zero_list); + area->nr_zero++; + } else { + memlist_add_head(&(list(map_nr))->list, &area->free_list); + area->nr_free++; + } + if (area != free_area) { + if (PageZeroed(list(map_nr))) + nr_zero_highorder_freed++; + } + +#undef list + + spin_unlock_irqrestore(&page_alloc_lock, flags); +} + +/* + * Some ugly macros to speed up __get_free_pages().. + */ +#define MARK_USED(index, order, area) \ + change_bit((index) >> (1+(order)), (area)->map) +#define CAN_DMA(x) (PageDMA(x)) +#define ADDRESS(x) (PAGE_OFFSET + ((x) << PAGE_SHIFT)) + int __free_page(struct page *page) { + unsigned long addr; + if (!PageReserved(page) && put_page_testzero(page)) { if (PageSwapCache(page)) PAGE_BUG(page); if (PageLocked(page)) PAGE_BUG(page); - free_pages_ok(page - mem_map, 0); + addr = ADDRESS(page-mem_map); + free_pages_ok(page-mem_map, 0); return 1; } return 0; @@ -166,148 +323,374 @@ return 0; } -/* - * Some ugly macros to speed up __get_free_pages().. - */ -#define MARK_USED(index, order, area) \ - change_bit((index) >> (1+(order)), (area)->map) -#define CAN_DMA(x) (PageDMA(x)) -#define ADDRESS(x) (PAGE_OFFSET + ((x) << PAGE_SHIFT)) +static inline unsigned long EXPAND (struct page *map, unsigned long index, + int low, int high, struct free_area_struct * area) +{ + unsigned long size = 1 << high; -#ifdef CONFIG_BIGMEM -#define RMQUEUEBIG(order, gfp_mask) \ -if (gfp_mask & __GFP_BIGMEM) { \ - struct free_area_struct * area = free_area+order+BIGMEM_LISTS_OFFSET; \ - unsigned long new_order = order; \ - do { struct page *prev = memory_head(area), *ret = prev->next; \ - if (memory_head(area) != ret) { \ - unsigned long map_nr; \ - (prev->next = ret->next)->prev = prev; \ - map_nr = ret - mem_map; \ - MARK_USED(map_nr, new_order, area); \ - nr_free_pages -= 1 << order; \ - nr_free_bigpages -= 1 << order; \ - EXPAND(ret, map_nr, order, new_order, area); \ - spin_unlock_irqrestore(&page_alloc_lock, flags); \ - return ADDRESS(map_nr); \ - } \ - new_order++; area++; \ - } while (new_order < NR_MEM_LISTS); \ + while (high > low) { + area--; + high--; + size >>= 1; + if (PageZeroed(map)) { + SetPageZeroed(map+size); + memlist_add_head(&(map)->list, &(area)->zero_list); + area->nr_zero++; + } else { + if (PageZeroed(map+size)) + BUG(); + memlist_add_head(&(map)->list, &(area)->free_list); + area->nr_free++; + } + MARK_USED(index, high, area); + index += size; + map += size; + } + set_page_count(map, 1); + return index; } -#endif -#define RMQUEUE(order, gfp_mask) \ -do { struct free_area_struct * area = free_area+order; \ - unsigned long new_order = order; \ - do { struct page *prev = memory_head(area), *ret = prev->next; \ - while (memory_head(area) != ret) { \ - if (!(gfp_mask & __GFP_DMA) || CAN_DMA(ret)) { \ - unsigned long map_nr; \ - (prev->next = ret->next)->prev = prev; \ - map_nr = ret - mem_map; \ - MARK_USED(map_nr, new_order, area); \ - nr_free_pages -= 1 << order; \ - EXPAND(ret, map_nr, order, new_order, area); \ - spin_unlock_irqrestore(&page_alloc_lock,flags);\ - return ADDRESS(map_nr); \ - } \ - prev = ret; \ - ret = ret->next; \ - } \ - new_order++; area++; \ - } while (new_order < NR_MEM_LISTS); \ -} while (0) - -#define EXPAND(map,index,low,high,area) \ -do { unsigned long size = 1 << high; \ - while (high > low) { \ - area--; high--; size >>= 1; \ - add_mem_queue(area, map); \ - MARK_USED(index, high, area); \ - index += size; \ - map += size; \ - } \ - set_page_count(map, 1); \ -} while (0) +static inline unsigned long EXPAND_zero (struct page *map, unsigned long index, + int high, struct free_area_struct * area) +{ + unsigned long size = 1 << high; -unsigned long __get_free_pages(int gfp_mask, unsigned long order) + if (!PageZeroed(map)) + BUG(); + while (high > 0) { + area--; + high--; + size >>= 1; + SetPageZeroed(map+size); + memlist_add_head(&(map)->list, &(area)->zero_list); + area->nr_zero++; + MARK_USED(index, high, area); + index += size; + map += size; + } + set_page_count(map, 1); + return index; +} + +static inline unsigned long rmqueue (int order, int gfp_mask, int offset) { - unsigned long flags; + struct free_area_struct * area = free_area+order+offset; + unsigned long curr_order = order, map_nr; + struct page *page; + struct list_head *head, *curr; - if (order >= NR_MEM_LISTS) - goto nopage; + do { + head = &area->free_list; + curr = memlist_next(head); -#ifdef ATOMIC_MEMORY_DEBUGGING - if ((gfp_mask & __GFP_WAIT) && in_interrupt()) { - static int count = 0; - if (++count < 5) { - printk("gfp called nonatomically from interrupt %p\n", - __builtin_return_address(0)); + while (curr != head) { + page = memlist_entry(curr, struct page, list); + if (PageZeroed(page)) + BUG(); + if (!(gfp_mask & __GFP_DMA) || CAN_DMA(page)) { + memlist_del(curr); + area->nr_free--; + map_nr = page - mem_map; + MARK_USED(map_nr, curr_order, area); + nr_free_pages -= 1 << order; + map_nr = EXPAND(page, map_nr, order, curr_order, area); + page = mem_map + map_nr; + if (PageZeroed(page)) + BUG(); + nr_allocated += 1<zero_list; + curr = memlist_next(head); + + while (curr != head) { + page = memlist_entry(curr, struct page, list); + if (!PageZeroed(page)) + BUG(); + if (!(gfp_mask & __GFP_DMA) || CAN_DMA(page)) { + memlist_del(curr); + area->nr_zero--; + map_nr = page - mem_map; + MARK_USED(map_nr, curr_order, area); + nr_free_pages -= 1 << order; + map_nr = EXPAND(page, map_nr, order, curr_order, area); + page = mem_map + map_nr; + if (!PageZeroed(page)) + BUG(); + nr_zero_buddy_down_loss += 1<zero_list; + curr = memlist_next(head); + + page = memlist_entry(curr, struct page, list); + if ((curr != head) && PageZeroed(page)) { + memlist_del(curr); + area->nr_zero--; + map_nr = page - mem_map; + MARK_USED(map_nr, curr_order, area); + nr_free_pages--; + map_nr = EXPAND_zero(page, map_nr, curr_order, area); + page = mem_map + map_nr; + if (!PageZeroed(page)) + BUG(); + nr_zero_pages--; + return ADDRESS(map_nr); + } + curr_order++; + area++; + } while (curr_order < NR_MEM_LISTS); + + return 0; +} + +struct zeroing_goal { + unsigned long addr; + int order; +}; + +static inline int out_of_balance (struct free_area_struct * area) +{ /* - * If this is a recursive call, we'd better - * do our best to just allocate things without - * further thought. + * Keep at most half of all free pages zeroed, plus some + * small constant in favor of free pages. (this keeps high + * cache-locality free pages in the free list in low memory + * situations) */ - if (!(current->flags & PF_MEMALLOC)) { - int freed; - static int low_on_memory = 0; + if (area->nr_zero < area->nr_free-5) + return 1; + return 0; +} -#ifndef CONFIG_BIGMEM - if (nr_free_pages > freepages.min) { - if (!low_on_memory) - goto ok_to_allocate; - if (nr_free_pages >= freepages.high) { - low_on_memory = 0; - goto ok_to_allocate; +static inline int get_zeroing_goal (struct zeroing_goal * goal) +{ + struct free_area_struct * area = free_area; + unsigned long curr_order = 0, map_nr, flags; + struct page *page; + struct list_head *head, *curr; + + spin_lock_irqsave(&page_alloc_lock, flags); + do { + head = &area->free_list; + curr = memlist_prev(head); + + page = memlist_entry(curr, struct page, list); + if ((curr != head) && out_of_balance(area)) { + if (PageZeroed(page)) + BUG(); + memlist_del(curr); + area->nr_free--; + map_nr = page - mem_map; + MARK_USED(map_nr, curr_order, area); + nr_free_pages -= 1 << curr_order;; + nr_allocated += 1<addr = ADDRESS(map_nr); + goal->order = curr_order; + return 1; + } + curr_order++; + area++; + } while (curr_order < NR_MEM_LISTS); + spin_unlock_irqrestore(&page_alloc_lock, flags); + + return 0; +} + +int create_one_zero_page (void) +{ + struct zeroing_goal goal = { 0, 0}; + struct page *page; + unsigned long map_nr, addr; + int i; + + if (!get_zeroing_goal(&goal)) + return 0; + if (goal.order == -1) + BUG(); + + addr = goal.addr; + for (i = 0; i < 1 << goal.order; i++) { + clear_page_cachefriendly(addr); + addr += PAGE_SIZE; + } + nr_zero_created++; + map_nr = MAP_NR(goal.addr); + page = mem_map + map_nr; + SetPageZeroed(page); + + free_zero_pages_ok(map_nr, goal.order); + + return 1; +} + +/* + * This is not atomic in the GFP_ATOMIC sense, we just try to + * find a zeroed page if possible ... + */ +unsigned long get_atomic_zero_page (void) +{ + unsigned long ret; + unsigned long flags; + struct page *page; + + spin_lock_irqsave(&page_alloc_lock, flags); + nr_zero_alloc_sum++; + ret = rmqueue_zero(); + if (!ret) { + nr_zero_alloc_direct_miss++; + ret = rmqueue(0, 0, 0); + } + spin_unlock_irqrestore(&page_alloc_lock, flags); + if (ret) { + page = mem_map + MAP_NR(ret); + /* + * now the page is ours. + */ + if (!PageZeroed(page)) { + nr_zero_alloc_miss++; + clear_page(ret); + SetPageZeroed(page); + } else { + nr_zero_alloc_hit++; +#if 0 + int * data; + int i; + + data = (int *)ret; + for (i = 0; i < PAGE_SIZE/4; i++) { + if (data[i]) { + printk("hm, page %08lx not zero at pos %04x\n", ret, i); + clear_page(ret); + break; + BUG(); + } } +#endif } + } + return ret; +} - low_on_memory = 1; +static inline int balance_lowmemory (int gfp_mask) +{ + int freed; + static int low_on_memory = 0; + +#ifndef CONFIG_BIGMEM + if (nr_free_pages > freepages.min) { + if (!low_on_memory) + return 1; + if (nr_free_pages >= freepages.high) { + low_on_memory = 0; + return 1; + } + } + + low_on_memory = 1; #else - static int low_on_bigmemory = 0; + static int low_on_bigmemory = 0; - if (gfp_mask & __GFP_BIGMEM) - { - if (nr_free_pages > freepages.min) { - if (!low_on_bigmemory) - goto ok_to_allocate; - if (nr_free_pages >= freepages.high) { - low_on_bigmemory = 0; - goto ok_to_allocate; - } + if (gfp_mask & __GFP_BIGMEM) + { + if (nr_free_pages > freepages.min) { + if (!low_on_bigmemory) + return 1; + if (nr_free_pages >= freepages.high) { + low_on_bigmemory = 0; + return 1; } - low_on_bigmemory = 1; - } else { - if (nr_free_pages-nr_free_bigpages > freepages.min) { - if (!low_on_memory) - goto ok_to_allocate; - if (nr_free_pages-nr_free_bigpages >= freepages.high) { - low_on_memory = 0; - goto ok_to_allocate; - } + } + low_on_bigmemory = 1; + } else { + if (nr_free_pages-nr_free_bigpages > freepages.min) { + if (!low_on_memory) + return 1; + if (nr_free_pages-nr_free_bigpages >= freepages.high) { + low_on_memory = 0; + return 1; } - low_on_memory = 1; } + low_on_memory = 1; + } #endif - current->flags |= PF_MEMALLOC; - freed = try_to_free_pages(gfp_mask); - current->flags &= ~PF_MEMALLOC; + current->flags |= PF_MEMALLOC; + freed = try_to_free_pages(gfp_mask); + current->flags &= ~PF_MEMALLOC; + + if (!freed && !(gfp_mask & (__GFP_MED | __GFP_HIGH))) + return 0; + return 1; +} + +unsigned long __get_free_pages(int gfp_mask, unsigned long order) +{ + unsigned long ret; + unsigned long flags; + struct page *page; + + if (order >= NR_MEM_LISTS) + goto nopage; + + /* + * If anyone calls gfp from interrupts nonatomically then it + * will sooner or later tripped up by a schedule(). + */ + + /* + * If this is a recursive call, we'd better + * do our best to just allocate things without + * further thought. + */ + if (!(current->flags & PF_MEMALLOC)) + goto lowmemory; - if (!freed && !(gfp_mask & (__GFP_MED | __GFP_HIGH))) - goto nopage; - } ok_to_allocate: spin_lock_irqsave(&page_alloc_lock, flags); + #ifdef CONFIG_BIGMEM - RMQUEUEBIG(order, gfp_mask); + if (gfp_mask & __GFP_BIGMEM) { + ret = rmqueue(order, gfp_mask, BIGMEM_LISTS_OFFSET); + page = mem_map + MAP_NR(ret); + if (ret) { + spin_unlock_irqrestore(&page_alloc_lock, flags); + return ret; + } + } #endif - RMQUEUE(order, gfp_mask); + ret = rmqueue(order, gfp_mask, 0); + page = mem_map + MAP_NR(ret); spin_unlock_irqrestore(&page_alloc_lock, flags); + if (ret) { + struct page *p; + p = mem_map + MAP_NR(ret); + return ret; + } /* * If we can schedule, do so, and make sure to yield. @@ -321,8 +704,37 @@ nopage: return 0; + +lowmemory: + if (balance_lowmemory(gfp_mask)) + goto ok_to_allocate; + goto nopage; } +#define NR_PAGES 1024 +static void benchmark_allocator (void) +{ + int i, err; + unsigned int t0, t1; + unsigned long pages[NR_PAGES]; + + rdtscl(t0); + for (i = 0; i < NR_PAGES; i++) + pages[i] = __get_free_page(GFP_KERNEL); + err = 0; + for (i = 0; i < NR_PAGES; i++) + if (pages[i]) + free_page(pages[i]); + else + err = 1; + rdtscl(t1); + if (err) + printk("could not allocate %d pages for benchmark.\n",NR_PAGES); + else + printk("allocated and freed %d pages in %u cycles, avg: %d cycles.\n", NR_PAGES, t1-t0, (t1-t0)/NR_PAGES); +} +#undef NR_PAGES + /* * Show free area list (used inside shift_scroll-lock stuff) * We also calculate the percentage fragmentation. We do this by counting the @@ -342,28 +754,45 @@ freepages.min, freepages.low, freepages.high); + spin_lock_irqsave(&page_alloc_lock, flags); - for (order=0 ; order < NR_MEM_LISTS; order++) { - struct page * tmp; - unsigned long nr = 0; - for (tmp = free_area[order].next ; tmp != memory_head(free_area+order) ; tmp = tmp->next) { - nr ++; + for (order = 0; order < NR_MEM_LISTS; order++) { + unsigned long nr = 0, nr_zero = 0; + struct list_head *head, *curr; + struct page *page; + + head = &free_area[order].free_list; + for (curr = memlist_next(head); curr != head; curr = memlist_next(curr)) { + page = memlist_entry(curr, struct page, list); + nr++; + if (PageZeroed(page)) + BUG(); } -#ifdef CONFIG_BIGMEM - for (tmp = free_area[BIGMEM_LISTS_OFFSET+order].next; - tmp != memory_head(free_area+BIGMEM_LISTS_OFFSET+order); - tmp = tmp->next) { - nr ++; + head = &free_area[order].zero_list; + for (curr = memlist_next(head); curr != head; curr = memlist_next(curr)) { + page = memlist_entry(curr, struct page, list); + nr_zero++; + if (!PageZeroed(page)) + BUG(); } +#ifdef CONFIG_BIGMEM + head = &free_area[order+BIGMEM_LISTS_OFFSET].free_list; + for (curr = memlist_next(head); curr != head; curr = memlist_next(curr)) + nr++; + head = &free_area[order+BIGMEM_LISTS_OFFSET].zero_list; + for (curr = memlist_next(head); curr != head; curr = memlist_next(curr)) + nr_zero++; #endif total += nr * ((PAGE_SIZE>>10) << order); - printk("%lu*%lukB ", nr, (unsigned long)((PAGE_SIZE>>10) << order)); + printk("%lu*%lukB{%lu zero}(nr_free:%lu, nr_zero:%lu) ", nr, (unsigned long)((PAGE_SIZE>>10) << order), nr_zero, free_area[order].nr_free, free_area[order].nr_zero); } spin_unlock_irqrestore(&page_alloc_lock, flags); + printk("= %lukB)\n", total); #ifdef SWAP_CACHE_INFO show_swap_cache_info(); #endif + benchmark_allocator(); } #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) @@ -404,13 +833,16 @@ set_page_count(p, 0); p->flags = (1 << PG_DMA) | (1 << PG_reserved); init_waitqueue_head(&p->wait); + memlist_init(&p->list); } while (p > mem_map); for (i = 0 ; i < NR_MEM_LISTS ; i++) { unsigned long bitmap_size; - init_mem_queue(free_area+i); + memlist_init(&(free_area+i)->free_list); + memlist_init(&(free_area+i)->zero_list); #ifdef CONFIG_BIGMEM - init_mem_queue(free_area+BIGMEM_LISTS_OFFSET+i); + memlist_init(&(free_area+BIGMEM_LISTS_OFFSET+i)->free_list); + memlist_init(&(free_area+BIGMEM_LISTS_OFFSET+i)->zero_list); #endif mask += mask; end_mem = (end_mem + ~mask) & mask; --- linux/include/linux/fs.h.orig Sun Sep 26 12:57:16 1999 +++ linux/include/linux/fs.h Sun Sep 26 18:43:33 1999 @@ -323,6 +323,11 @@ #include #include +/* + * oh the beauties of C type declarations. + */ +struct page; + struct inode { struct list_head i_hash; struct list_head i_list; @@ -350,7 +355,7 @@ wait_queue_head_t i_wait; struct file_lock *i_flock; struct vm_area_struct *i_mmap; - struct page *i_pages; + struct list_head i_pages; spinlock_t i_shared_lock; struct dquot *i_dquot[MAXQUOTAS]; struct pipe_inode_info *i_pipe; --- linux/include/linux/mm.h.orig Sun Sep 26 12:57:16 1999 +++ linux/include/linux/mm.h Sun Sep 26 23:01:18 1999 @@ -8,6 +8,7 @@ #include #include +#include extern unsigned long max_mapnr; extern unsigned long num_physpages; @@ -109,6 +110,9 @@ int (*swapout)(struct vm_area_struct *, struct page *); }; +#define DD +// printk("<%s:%d>\n", __FILE__, __LINE__) + /* * Try to keep the most commonly accessed fields in single cache lines * here (16 bytes or greater). This ordering should be particularly @@ -119,8 +123,7 @@ */ typedef struct page { /* these must be first (free area handling) */ - struct page *next; - struct page *prev; + struct list_head list; struct inode *inode; unsigned long offset; struct page *next_hash; @@ -149,11 +152,12 @@ #define PG_uptodate 3 #define PG_decr_after 5 #define PG_DMA 7 -#define PG_Slab 8 +#define PG_slab 8 #define PG_swap_cache 9 #define PG_skip 10 #define PG_swap_entry 11 -#define PG_BIGMEM 12 +#define PG_bigmem 12 +#define PG_zeroed 13 /* bits 21-30 unused */ #define PG_reserved 31 @@ -183,28 +187,48 @@ #define PageReferenced(page) (test_bit(PG_referenced, &(page)->flags)) #define PageDecrAfter(page) (test_bit(PG_decr_after, &(page)->flags)) #define PageDMA(page) (test_bit(PG_DMA, &(page)->flags)) -#define PageSlab(page) (test_bit(PG_Slab, &(page)->flags)) +#define PageSlab(page) (test_bit(PG_slab, &(page)->flags)) #define PageSwapCache(page) (test_bit(PG_swap_cache, &(page)->flags)) #define PageReserved(page) (test_bit(PG_reserved, &(page)->flags)) -#define PageSetSlab(page) (set_bit(PG_Slab, &(page)->flags)) +#define PageSetSlab(page) (set_bit(PG_slab, &(page)->flags)) #define PageSetSwapCache(page) (set_bit(PG_swap_cache, &(page)->flags)) #define PageTestandSetSwapCache(page) \ (test_and_set_bit(PG_swap_cache, &(page)->flags)) -#define PageClearSlab(page) (clear_bit(PG_Slab, &(page)->flags)) +#define PageClearSlab(page) (clear_bit(PG_slab, &(page)->flags)) #define PageClearSwapCache(page)(clear_bit(PG_swap_cache, &(page)->flags)) #define PageTestandClearSwapCache(page) \ (test_and_clear_bit(PG_swap_cache, &(page)->flags)) #ifdef CONFIG_BIGMEM -#define PageBIGMEM(page) (test_bit(PG_BIGMEM, &(page)->flags)) +#define PageBIGMEM(page) (test_bit(PG_bigmem, &(page)->flags)) #else #define PageBIGMEM(page) 0 /* needed to optimize away at compile time */ #endif /* + * These are nonatomic bit-operations because all access to + * the PG_zeroed flag is within the page_alloc spinlock or has + * been done on the local CPU. + */ +#define PageZeroed(page) ((page)->flags & (1<flags |= (1<flags &= ~(1<flags bits: * * PG_reserved is set for a page which must never be accessed (which @@ -295,6 +319,8 @@ return page; } +unsigned long get_atomic_zero_page (void); + /* memory.c & swap.c*/ #define free_page(addr) free_pages((addr),0) @@ -369,8 +395,8 @@ #define GFP_BUFFER (__GFP_LOW | __GFP_WAIT) #define GFP_ATOMIC (__GFP_HIGH) -#define GFP_BIGUSER (__GFP_LOW | __GFP_WAIT | __GFP_IO | __GFP_BIGMEM) #define GFP_USER (__GFP_LOW | __GFP_WAIT | __GFP_IO) +#define GFP_BIGUSER (GFP_USER | __GFP_BIGMEM) #define GFP_KERNEL (__GFP_MED | __GFP_WAIT | __GFP_IO) #define GFP_NFS (__GFP_HIGH | __GFP_WAIT | __GFP_IO) #define GFP_KSWAPD (__GFP_IO | __GFP_SWAP) --- linux/include/linux/pagemap.h.orig Sun Sep 26 12:57:16 1999 +++ linux/include/linux/pagemap.h Sun Sep 26 23:01:18 1999 @@ -11,6 +11,7 @@ #include #include +#include static inline unsigned long page_address(struct page * page) { @@ -89,14 +90,25 @@ static inline void add_page_to_inode_queue(struct inode * inode, struct page * page) { - struct page **p = &inode->i_pages; + struct list_head *head = &inode->i_pages; - inode->i_nrpages++; + if (!inode->i_nrpages++) { + if (!list_empty(head)) + BUG(); + } else { + if (list_empty(head)) + BUG(); + } + list_add(&page->list, head); page->inode = inode; - page->prev = NULL; - if ((page->next = *p) != NULL) - page->next->prev = page; - *p = page; +} + +static inline void remove_page_from_inode_queue(struct page * page) +{ + struct inode * inode = page->inode; + + inode->i_nrpages--; + list_del(&page->list); } extern void ___wait_on_page(struct page *); --- linux/include/asm-i386/page.h.orig Wed Sep 1 08:29:08 1999 +++ linux/include/asm-i386/page.h Sun Sep 26 16:31:24 1999 @@ -11,8 +11,10 @@ #define STRICT_MM_TYPECHECKS -#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) -#define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE) +extern void clear_page(unsigned long page); +extern void copy_page(unsigned long to, unsigned long from); +//#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) +//#define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE) #ifdef STRICT_MM_TYPECHECKS /* --- linux/include/asm-i386/fixmap.h.orig Sun Sep 26 12:57:16 1999 +++ linux/include/asm-i386/fixmap.h Sun Sep 26 19:43:27 1999 @@ -65,10 +65,13 @@ FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */ FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, #endif + FIX_ZEROING_WINDOW_0, + FIX_ZEROING_WINDOW_END = FIX_ZEROING_WINDOW_0 + NR_CPUS-1, __end_of_fixed_addresses }; extern void set_fixmap (enum fixed_addresses idx, unsigned long phys); +extern void set_fixmap_wt (enum fixed_addresses idx, unsigned long phys); /* * used by vmalloc.c. --- linux/include/asm-i386/smp.h.orig Sun Sep 26 22:56:05 1999 +++ linux/include/asm-i386/smp.h Sun Sep 26 22:59:05 1999 @@ -171,6 +171,12 @@ extern volatile int cpu_number_map[NR_CPUS]; extern volatile unsigned long smp_invalidate_needed; extern void smp_flush_tlb(void); +extern volatile unsigned long cpu_idle_map; + +extern inline int all_cpus_idle (void) +{ + return (cpu_online_map == cpu_idle_map); +} extern volatile unsigned long cpu_callin_map[NR_CPUS]; extern void smp_message_irq(int cpl, void *dev_id, struct pt_regs *regs); --- linux/arch/i386/mm/Makefile.orig Wed Sep 1 08:29:05 1999 +++ linux/arch/i386/mm/Makefile Sun Sep 26 16:31:24 1999 @@ -10,8 +10,4 @@ O_TARGET := mm.o O_OBJS := init.o fault.o ioremap.o extable.o -ifeq ($(CONFIG_BIGMEM),y) -O_OBJS += bigmem.o -endif - include $(TOPDIR)/Rules.make --- linux/arch/i386/mm/bigmem.c.orig Wed Sep 1 08:29:05 1999 +++ linux/arch/i386/mm/bigmem.c Sun Sep 26 16:31:24 1999 @@ -1,33 +0,0 @@ -/* - * BIGMEM IA32 code and variables. - * - * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de - * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de - */ - -#include -#include - -unsigned long bigmem_start, bigmem_end; - -/* NOTE: fixmap_init alloc all the fixmap pagetables contigous on the - physical space so we can cache the place of the first one and move - around without checking the pgd every time. */ -pte_t *kmap_pte; -pgprot_t kmap_prot; - -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)) - -void __init kmap_init(void) -{ - unsigned long kmap_vstart; - - /* cache the first kmap pte */ - kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); - kmap_pte = kmap_get_fixmap_pte(kmap_vstart); - - kmap_prot = PAGE_KERNEL; - if (boot_cpu_data.x86_capability & X86_FEATURE_PGE) - pgprot_val(kmap_prot) |= _PAGE_GLOBAL; -} --- linux/arch/i386/mm/init.c.orig Sun Sep 26 18:52:34 1999 +++ linux/arch/i386/mm/init.c Sun Sep 26 22:57:41 1999 @@ -4,6 +4,7 @@ * Copyright (C) 1995 Linus Torvalds * * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 + * Support for background page-zeroing added by Ingo Molnar */ #include @@ -32,6 +33,7 @@ #include #include +unsigned long bigmem_start, bigmem_end; static unsigned long totalram = 0; static unsigned long totalbig = 0; @@ -192,6 +194,31 @@ extern char _text, _etext, _edata, __bss_start, _end; extern char __init_begin, __init_end; +/* NOTE: fixmap_init alloc all the fixmap pagetables contigous on the + physical space so we can cache the place of the first one and move + around without checking the pgd every time. */ + +#if CONFIG_BIGMEM +pte_t *kmap_pte; +pgprot_t kmap_prot; + +#define kmap_get_fixmap_pte(vaddr) \ + pte_offset(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)) + +void __init kmap_init(void) +{ + unsigned long kmap_vstart; + + /* cache the first kmap pte */ + kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); + kmap_pte = kmap_get_fixmap_pte(kmap_vstart); + + kmap_prot = PAGE_KERNEL; + if (boot_cpu_data.x86_capability & X86_FEATURE_PGE) + pgprot_val(kmap_prot) |= _PAGE_GLOBAL; +} +#endif + /* * allocate page table(s) for compile-time fixed mappings */ @@ -215,7 +242,8 @@ return start_mem; } -static void set_pte_phys (unsigned long vaddr, unsigned long phys) +static void __set_pte_phys (unsigned long vaddr, unsigned long phys, + unsigned long flags) { pgprot_t prot; pte_t * pte; @@ -223,10 +251,13 @@ pte = pte_offset(pmd_offset(pgd_offset_k(vaddr), vaddr), vaddr); prot = PAGE_KERNEL; if (boot_cpu_data.x86_capability & X86_FEATURE_PGE) - pgprot_val(prot) |= _PAGE_GLOBAL; + pgprot_val(prot) |= _PAGE_GLOBAL | flags; set_pte(pte, mk_pte_phys(phys, prot)); - local_flush_tlb(); + /* + * No need to flush the whole TLB ... + */ + __flush_tlb_one(vaddr); } void set_fixmap (enum fixed_addresses idx, unsigned long phys) @@ -237,7 +268,45 @@ printk("Invalid set_fixmap\n"); return; } - set_pte_phys (address,phys); + __set_pte_phys (address, phys, 0); +} + +void set_fixmap_wt (enum fixed_addresses idx, unsigned long phys) +{ + unsigned long address = __fix_to_virt(idx); + + if (idx >= __end_of_fixed_addresses) { + printk("Invalid set_fixmap\n"); + return; + } + __set_pte_phys (address, phys, _PAGE_PWT); +} + +/* + * Cache and scheduling-friendly page clearing function. + * Speed is _not_ the primary goal. This function behaves + * as if the system was executing nothing: it's clearing + * pages through a write-through virtual memory window, + * and it's checking for reschedule events frequently. + * + * Later versions of this code will use the KNI cache-control + * instructions to get more speed nevertheless ... + */ +void clear_page_cachefriendly (unsigned long addr) +{ + int fixmap_entry = FIX_ZEROING_WINDOW_0 + smp_processor_id(); + unsigned long phys = __pa(addr); + unsigned long vaddr; + int i; + + set_fixmap_wt(fixmap_entry, phys); + vaddr = __fix_to_virt(fixmap_entry); + for (i = 0; i < PAGE_SIZE/L1_CACHE_BYTES; i++) { + memset((void *)vaddr, 0, L1_CACHE_BYTES); + vaddr += L1_CACHE_BYTES; + if (!all_cpus_idle() || current->need_resched) + schedule(); + } } /* @@ -487,7 +556,7 @@ #ifdef CONFIG_BIGMEM for (tmp = bigmem_start; tmp < bigmem_end; tmp += PAGE_SIZE) { clear_bit(PG_reserved, &mem_map[PHYSMAP_NR(tmp)].flags); - set_bit(PG_BIGMEM, &mem_map[PHYSMAP_NR(tmp)].flags); + set_bit(PG_bigmem, &mem_map[PHYSMAP_NR(tmp)].flags); atomic_set(&mem_map[PHYSMAP_NR(tmp)].count, 1); free_page(tmp + PAGE_OFFSET); totalbig += PAGE_SIZE; --- linux/arch/i386/kernel/process.c.orig Mon Aug 23 17:10:52 1999 +++ linux/arch/i386/kernel/process.c Sun Sep 26 23:01:54 1999 @@ -74,6 +74,8 @@ * low exit latency (ie sit in a loop waiting for * somebody to say that they'd like to reschedule) */ +volatile unsigned long cpu_idle_map = -1; + void cpu_idle(void) { /* endless idle loop with no priority at all */ @@ -82,6 +84,9 @@ current->counter = -100; while (1) { + while (all_cpus_idle() && create_one_zero_page()) + if (current->need_resched) + break; while (!current->need_resched) { if (!current_cpu_data.hlt_works_ok) continue; @@ -89,7 +94,9 @@ continue; asm volatile("sti ; hlt" : : : "memory"); } + clear_bit(smp_processor_id(), &cpu_idle_map); schedule(); + set_bit(smp_processor_id(), &cpu_idle_map); check_pgt_cache(); if (acpi_idle) acpi_idle(); --- linux/arch/i386/kernel/irq.c.orig Wed Sep 1 08:29:05 1999 +++ linux/arch/i386/kernel/irq.c Sun Sep 26 16:31:24 1999 @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -30,14 +29,13 @@ #include #include #include -#include #include #include +#include #include #include #include -#include #include #include #include --- linux/arch/i386/kernel/i386_ksyms.c.orig Wed Sep 1 08:29:05 1999 +++ linux/arch/i386/kernel/i386_ksyms.c Sun Sep 26 16:31:24 1999 @@ -1,6 +1,5 @@ #include #include -#include #include #include #include --- linux/arch/i386/kernel/i8259.c.orig Sun Sep 26 18:49:07 1999 +++ linux/arch/i386/kernel/i8259.c Sun Sep 26 18:49:14 1999 @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include