v2.5.0.7 -> v2.5.0.8
[opensuse:kernel.git] / mm / highmem.c
1 /*
2  * High memory handling common code and variables.
3  *
4  * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de
5  *          Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de
6  *
7  *
8  * Redesigned the x86 32-bit VM architecture to deal with
9  * 64-bit physical space. With current x86 CPUs this
10  * means up to 64 Gigabytes physical RAM.
11  *
12  * Rewrote high memory support to move the page cache into
13  * high memory. Implemented permanent (schedulable) kmaps
14  * based on Linus' idea.
15  *
16  * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com>
17  */
18
19 #include <linux/mm.h>
20 #include <linux/pagemap.h>
21 #include <linux/highmem.h>
22 #include <linux/swap.h>
23 #include <linux/slab.h>
24 #include <linux/compiler.h>
25
26 #include <linux/kernel_stat.h>
27
28 /*
29  * Virtual_count is not a pure "count".
30  *  0 means that it is not mapped, and has not been mapped
31  *    since a TLB flush - it is usable.
32  *  1 means that there are no users, but it has been mapped
33  *    since the last TLB flush - so we can't use it.
34  *  n means that there are (n-1) current users of it.
35  */
36 static int pkmap_count[LAST_PKMAP];
37 static unsigned int last_pkmap_nr;
38 static spinlock_t kmap_lock = SPIN_LOCK_UNLOCKED;
39
40 pte_t * pkmap_page_table;
41
42 static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait);
43
44 static void flush_all_zero_pkmaps(void)
45 {
46         int i;
47
48         flush_cache_all();
49
50         for (i = 0; i < LAST_PKMAP; i++) {
51                 struct page *page;
52
53                 /*
54                  * zero means we don't have anything to do,
55                  * >1 means that it is still in use. Only
56                  * a count of 1 means that it is free but
57                  * needs to be unmapped
58                  */
59                 if (pkmap_count[i] != 1)
60                         continue;
61                 pkmap_count[i] = 0;
62
63                 /* sanity check */
64                 if (pte_none(pkmap_page_table[i]))
65                         BUG();
66
67                 /*
68                  * Don't need an atomic fetch-and-clear op here;
69                  * no-one has the page mapped, and cannot get at
70                  * its virtual address (and hence PTE) without first
71                  * getting the kmap_lock (which is held here).
72                  * So no dangers, even with speculative execution.
73                  */
74                 page = pte_page(pkmap_page_table[i]);
75                 pte_clear(&pkmap_page_table[i]);
76
77                 page->virtual = NULL;
78         }
79         flush_tlb_all();
80 }
81
82 static inline unsigned long map_new_virtual(struct page *page)
83 {
84         unsigned long vaddr;
85         int count;
86
87 start:
88         count = LAST_PKMAP;
89         /* Find an empty entry */
90         for (;;) {
91                 last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
92                 if (!last_pkmap_nr) {
93                         flush_all_zero_pkmaps();
94                         count = LAST_PKMAP;
95                 }
96                 if (!pkmap_count[last_pkmap_nr])
97                         break;  /* Found a usable entry */
98                 if (--count)
99                         continue;
100
101                 /*
102                  * Sleep for somebody else to unmap their entries
103                  */
104                 {
105                         DECLARE_WAITQUEUE(wait, current);
106
107                         current->state = TASK_UNINTERRUPTIBLE;
108                         add_wait_queue(&pkmap_map_wait, &wait);
109                         spin_unlock(&kmap_lock);
110                         schedule();
111                         remove_wait_queue(&pkmap_map_wait, &wait);
112                         spin_lock(&kmap_lock);
113
114                         /* Somebody else might have mapped it while we slept */
115                         if (page->virtual)
116                                 return (unsigned long) page->virtual;
117
118                         /* Re-start */
119                         goto start;
120                 }
121         }
122         vaddr = PKMAP_ADDR(last_pkmap_nr);
123         set_pte(&(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot));
124
125         pkmap_count[last_pkmap_nr] = 1;
126         page->virtual = (void *) vaddr;
127
128         return vaddr;
129 }
130
131 void *kmap_high(struct page *page)
132 {
133         unsigned long vaddr;
134
135         /*
136          * For highmem pages, we can't trust "virtual" until
137          * after we have the lock.
138          *
139          * We cannot call this from interrupts, as it may block
140          */
141         spin_lock(&kmap_lock);
142         vaddr = (unsigned long) page->virtual;
143         if (!vaddr)
144                 vaddr = map_new_virtual(page);
145         pkmap_count[PKMAP_NR(vaddr)]++;
146         if (pkmap_count[PKMAP_NR(vaddr)] < 2)
147                 BUG();
148         spin_unlock(&kmap_lock);
149         return (void*) vaddr;
150 }
151
152 void kunmap_high(struct page *page)
153 {
154         unsigned long vaddr;
155         unsigned long nr;
156         int need_wakeup;
157
158         spin_lock(&kmap_lock);
159         vaddr = (unsigned long) page->virtual;
160         if (!vaddr)
161                 BUG();
162         nr = PKMAP_NR(vaddr);
163
164         /*
165          * A count must never go down to zero
166          * without a TLB flush!
167          */
168         need_wakeup = 0;
169         switch (--pkmap_count[nr]) {
170         case 0:
171                 BUG();
172         case 1:
173                 /*
174                  * Avoid an unnecessary wake_up() function call.
175                  * The common case is pkmap_count[] == 1, but
176                  * no waiters.
177                  * The tasks queued in the wait-queue are guarded
178                  * by both the lock in the wait-queue-head and by
179                  * the kmap_lock.  As the kmap_lock is held here,
180                  * no need for the wait-queue-head's lock.  Simply
181                  * test if the queue is empty.
182                  */
183                 need_wakeup = waitqueue_active(&pkmap_map_wait);
184         }
185         spin_unlock(&kmap_lock);
186
187         /* do wake-up, if needed, race-free outside of the spin lock */
188         if (need_wakeup)
189                 wake_up(&pkmap_map_wait);
190 }
191
192 #define POOL_SIZE 64
193
194 /*
195  * This lock gets no contention at all, normally.
196  */
197 static spinlock_t emergency_lock = SPIN_LOCK_UNLOCKED;
198
199 int nr_emergency_pages;
200 static LIST_HEAD(emergency_pages);
201
202 int nr_emergency_bhs;
203 static LIST_HEAD(emergency_bhs);
204
205 /*
206  * Simple bounce buffer support for highmem pages. Depending on the
207  * queue gfp mask set, *to may or may not be a highmem page. kmap it
208  * always, it will do the Right Thing
209  */
210 static inline void copy_to_high_bio_irq(struct bio *to, struct bio *from)
211 {
212         unsigned char *vto, *vfrom;
213         unsigned long flags;
214         struct bio_vec *tovec, *fromvec;
215         int i;
216
217         __bio_for_each_segment(tovec, to, i, 0) {
218                 fromvec = &from->bi_io_vec[i];
219
220                 /*
221                  * not bounced
222                  */
223                 if (tovec->bv_page == fromvec->bv_page)
224                         continue;
225
226                 vfrom = page_address(fromvec->bv_page) + fromvec->bv_offset;
227
228                 local_irq_save(flags);
229                 vto = kmap_atomic(tovec->bv_page, KM_BOUNCE_READ);
230                 memcpy(vto + tovec->bv_offset, vfrom, tovec->bv_len);
231                 kunmap_atomic(vto, KM_BOUNCE_READ);
232                 local_irq_restore(flags);
233         }
234 }
235
236 static __init int init_emergency_pool(void)
237 {
238         struct sysinfo i;
239         si_meminfo(&i);
240         si_swapinfo(&i);
241         
242         if (!i.totalhigh)
243                 return 0;
244
245         spin_lock_irq(&emergency_lock);
246         while (nr_emergency_pages < POOL_SIZE) {
247                 struct page * page = alloc_page(GFP_ATOMIC);
248                 if (!page) {
249                         printk("couldn't refill highmem emergency pages");
250                         break;
251                 }
252                 list_add(&page->list, &emergency_pages);
253                 nr_emergency_pages++;
254         }
255         spin_unlock_irq(&emergency_lock);
256         printk("allocated %d pages reserved for the highmem bounces\n", nr_emergency_pages);
257         return 0;
258 }
259
260 __initcall(init_emergency_pool);
261
262 static inline int bounce_end_io (struct bio *bio, int nr_sectors)
263 {
264         struct bio *bio_orig = bio->bi_private;
265         struct bio_vec *bvec, *org_vec;
266         unsigned long flags;
267         int ret, i;
268
269         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
270                 goto out_eio;
271
272         set_bit(BIO_UPTODATE, &bio_orig->bi_flags);
273
274         /*
275          * free up bounce indirect pages used
276          */
277         spin_lock_irqsave(&emergency_lock, flags);
278         __bio_for_each_segment(bvec, bio, i, 0) {
279                 org_vec = &bio_orig->bi_io_vec[i];
280                 if (bvec->bv_page == org_vec->bv_page)
281                         continue;
282         
283                 if (nr_emergency_pages >= POOL_SIZE)
284                         __free_page(bvec->bv_page);
285                 else {
286                         /*
287                          * We are abusing page->list to manage
288                          * the highmem emergency pool:
289                          */
290                         list_add(&bvec->bv_page->list, &emergency_pages);
291                         nr_emergency_pages++;
292                 }
293         }
294         spin_unlock_irqrestore(&emergency_lock, flags);
295
296 out_eio:
297         ret = bio_orig->bi_end_io(bio_orig, nr_sectors);
298
299         bio_put(bio);
300         return ret;
301 }
302
303 static int bounce_end_io_write(struct bio *bio, int nr_sectors)
304 {
305         return bounce_end_io(bio, nr_sectors);
306 }
307
308 static int bounce_end_io_read (struct bio *bio, int nr_sectors)
309 {
310         struct bio *bio_orig = bio->bi_private;
311
312         if (test_bit(BIO_UPTODATE, &bio->bi_flags))
313                 copy_to_high_bio_irq(bio_orig, bio);
314
315         return bounce_end_io(bio, nr_sectors);
316 }
317
318 struct page *alloc_bounce_page(int gfp_mask)
319 {
320         struct list_head *tmp;
321         struct page *page;
322
323         page = alloc_page(gfp_mask);
324         if (page)
325                 return page;
326         /*
327          * No luck. First, kick the VM so it doesnt idle around while
328          * we are using up our emergency rations.
329          */
330         wakeup_bdflush();
331
332 repeat_alloc:
333         /*
334          * Try to allocate from the emergency pool.
335          */
336         tmp = &emergency_pages;
337         spin_lock_irq(&emergency_lock);
338         if (!list_empty(tmp)) {
339                 page = list_entry(tmp->next, struct page, list);
340                 list_del(tmp->next);
341                 nr_emergency_pages--;
342         }
343         spin_unlock_irq(&emergency_lock);
344         if (page)
345                 return page;
346
347         /* we need to wait I/O completion */
348         run_task_queue(&tq_disk);
349
350         current->policy |= SCHED_YIELD;
351         __set_current_state(TASK_RUNNING);
352         schedule();
353         goto repeat_alloc;
354 }
355
356 void create_bounce(unsigned long pfn, struct bio **bio_orig)
357 {
358         struct page *page;
359         struct bio *bio = NULL;
360         int i, rw = bio_data_dir(*bio_orig);
361         struct bio_vec *to, *from;
362
363         BUG_ON((*bio_orig)->bi_idx);
364
365         bio_for_each_segment(from, *bio_orig, i) {
366                 page = from->bv_page;
367
368                 /*
369                  * is destination page below bounce pfn?
370                  */
371                 if ((page - page->zone->zone_mem_map) + (page->zone->zone_start_paddr >> PAGE_SHIFT) < pfn)
372                         continue;
373
374                 /*
375                  * irk, bounce it
376                  */
377                 if (!bio)
378                         bio = bio_alloc(GFP_NOHIGHIO, (*bio_orig)->bi_vcnt);
379
380                 to = &bio->bi_io_vec[i];
381
382                 to->bv_page = alloc_bounce_page(GFP_NOHIGHIO);
383                 to->bv_len = from->bv_len;
384                 to->bv_offset = from->bv_offset;
385
386                 if (rw & WRITE) {
387                         char *vto, *vfrom;
388
389                         vto = page_address(to->bv_page) + to->bv_offset;
390                         vfrom = kmap(from->bv_page) + from->bv_offset;
391                         memcpy(vto, vfrom, to->bv_len);
392                         kunmap(from->bv_page);
393                 }
394         }
395
396         /*
397          * no pages bounced
398          */
399         if (!bio)
400                 return;
401
402         /*
403          * at least one page was bounced, fill in possible non-highmem
404          * pages
405          */
406         bio_for_each_segment(from, *bio_orig, i) {
407                 to = &bio->bi_io_vec[i];
408                 if (!to->bv_page) {
409                         to->bv_page = from->bv_page;
410                         to->bv_len = from->bv_len;
411                         to->bv_offset = to->bv_offset;
412                 }
413         }
414
415         bio->bi_dev = (*bio_orig)->bi_dev;
416         bio->bi_sector = (*bio_orig)->bi_sector;
417         bio->bi_rw = (*bio_orig)->bi_rw;
418
419         bio->bi_vcnt = (*bio_orig)->bi_vcnt;
420         bio->bi_idx = 0;
421         bio->bi_size = (*bio_orig)->bi_size;
422
423         if (rw & WRITE)
424                 bio->bi_end_io = bounce_end_io_write;
425         else
426                 bio->bi_end_io = bounce_end_io_read;
427
428         bio->bi_private = *bio_orig;
429         *bio_orig = bio;
430 }