Skip to content

Commit 2bee1b5

Browse files
rpptpaulburton
rppt
authored and
paulburton
committed
mips: add support for folded p4d page tables
Implement primitives necessary for the 4th level folding, add walks of p4d level where appropriate, replace 5leve-fixup.h with pgtable-nop4d.h and drop usage of __ARCH_USE_5LEVEL_HACK. Signed-off-by: Mike Rapoport <[email protected]> Signed-off-by: Paul Burton <[email protected]> Cc: Ralf Baechle <[email protected]> Cc: James Hogan <[email protected]> Cc: [email protected] Cc: [email protected] Cc: [email protected] Cc: Mike Rapoport <[email protected]>
1 parent 31168f0 commit 2bee1b5

14 files changed

+75
-39
lines changed

‎arch/mips/include/asm/fixmap.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ enum fixed_addresses {
7070
#include <asm-generic/fixmap.h>
7171

7272
#define kmap_get_fixmap_pte(vaddr) \
73-
pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr))
73+
pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr)), (vaddr))
7474

7575
/*
7676
* Called from pgtable_init()

‎arch/mips/include/asm/pgalloc.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,9 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)
9696
free_pages((unsigned long)pud, PUD_ORDER);
9797
}
9898

99-
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
99+
static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
100100
{
101-
set_pgd(pgd, __pgd((unsigned long)pud));
101+
set_p4d(p4d, __p4d((unsigned long)pud));
102102
}
103103

104104
#define __pud_free_tlb(tlb, x, addr) pud_free((tlb)->mm, x)

‎arch/mips/include/asm/pgtable-32.h

-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
#include <asm/cachectl.h>
1717
#include <asm/fixmap.h>
1818

19-
#define __ARCH_USE_5LEVEL_HACK
2019
#include <asm-generic/pgtable-nopmd.h>
2120

2221
#ifdef CONFIG_HIGHMEM

‎arch/mips/include/asm/pgtable-64.h

+19-18
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,12 @@
1717
#include <asm/cachectl.h>
1818
#include <asm/fixmap.h>
1919

20-
#define __ARCH_USE_5LEVEL_HACK
2120
#if CONFIG_PGTABLE_LEVELS == 2
2221
#include <asm-generic/pgtable-nopmd.h>
2322
#elif CONFIG_PGTABLE_LEVELS == 3
2423
#include <asm-generic/pgtable-nopud.h>
2524
#else
26-
#include <asm-generic/5level-fixup.h>
25+
#include <asm-generic/pgtable-nop4d.h>
2726
#endif
2827

2928
/*
@@ -188,47 +187,49 @@ extern pud_t invalid_pud_table[PTRS_PER_PUD];
188187
/*
189188
* Empty pgd entries point to the invalid_pud_table.
190189
*/
191-
static inline int pgd_none(pgd_t pgd)
190+
static inline int p4d_none(p4d_t p4d)
192191
{
193-
return pgd_val(pgd) == (unsigned long)invalid_pud_table;
192+
return p4d_val(p4d) == (unsigned long)invalid_pud_table;
194193
}
195194

196-
static inline int pgd_bad(pgd_t pgd)
195+
static inline int p4d_bad(p4d_t p4d)
197196
{
198-
if (unlikely(pgd_val(pgd) & ~PAGE_MASK))
197+
if (unlikely(p4d_val(p4d) & ~PAGE_MASK))
199198
return 1;
200199

201200
return 0;
202201
}
203202

204-
static inline int pgd_present(pgd_t pgd)
203+
static inline int p4d_present(p4d_t p4d)
205204
{
206-
return pgd_val(pgd) != (unsigned long)invalid_pud_table;
205+
return p4d_val(p4d) != (unsigned long)invalid_pud_table;
207206
}
208207

209-
static inline void pgd_clear(pgd_t *pgdp)
208+
static inline void p4d_clear(p4d_t *p4dp)
210209
{
211-
pgd_val(*pgdp) = (unsigned long)invalid_pud_table;
210+
p4d_val(*p4dp) = (unsigned long)invalid_pud_table;
212211
}
213212

214213
#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
215214

216-
static inline unsigned long pgd_page_vaddr(pgd_t pgd)
215+
static inline unsigned long p4d_page_vaddr(p4d_t p4d)
217216
{
218-
return pgd_val(pgd);
217+
return p4d_val(p4d);
219218
}
220219

221-
#define pgd_phys(pgd) virt_to_phys((void *)pgd_val(pgd))
222-
#define pgd_page(pgd) (pfn_to_page(pgd_phys(pgd) >> PAGE_SHIFT))
220+
#define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d))
221+
#define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT))
223222

224-
static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
223+
#define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
224+
225+
static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
225226
{
226-
return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address);
227+
return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address);
227228
}
228229

229-
static inline void set_pgd(pgd_t *pgd, pgd_t pgdval)
230+
static inline void set_p4d(p4d_t *p4d, p4d_t p4dval)
230231
{
231-
*pgd = pgdval;
232+
*p4d = p4dval;
232233
}
233234

234235
#endif

‎arch/mips/kvm/mmu.c

+12-4
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ pgd_t *kvm_pgd_alloc(void)
136136
static pte_t *kvm_mips_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache,
137137
unsigned long addr)
138138
{
139+
p4d_t *p4d;
139140
pud_t *pud;
140141
pmd_t *pmd;
141142

@@ -145,7 +146,8 @@ static pte_t *kvm_mips_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache,
145146
BUG();
146147
return NULL;
147148
}
148-
pud = pud_offset(pgd, addr);
149+
p4d = p4d_offset(pgd, addr);
150+
pud = pud_offset(p4d, addr);
149151
if (pud_none(*pud)) {
150152
pmd_t *new_pmd;
151153

@@ -258,6 +260,7 @@ static bool kvm_mips_flush_gpa_pud(pud_t *pud, unsigned long start_gpa,
258260
static bool kvm_mips_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa,
259261
unsigned long end_gpa)
260262
{
263+
p4d_t *p4d;
261264
pud_t *pud;
262265
unsigned long end = ~0ul;
263266
int i_min = pgd_index(start_gpa);
@@ -269,7 +272,8 @@ static bool kvm_mips_flush_gpa_pgd(pgd_t *pgd, unsigned long start_gpa,
269272
if (!pgd_present(pgd[i]))
270273
continue;
271274

272-
pud = pud_offset(pgd + i, 0);
275+
p4d = p4d_offset(pgd, 0);
276+
pud = pud_offset(p4d + i, 0);
273277
if (i == i_max)
274278
end = end_gpa;
275279

@@ -378,6 +382,7 @@ static int kvm_mips_##name##_pgd(pgd_t *pgd, unsigned long start, \
378382
unsigned long end) \
379383
{ \
380384
int ret = 0; \
385+
p4d_t *p4d; \
381386
pud_t *pud; \
382387
unsigned long cur_end = ~0ul; \
383388
int i_min = pgd_index(start); \
@@ -388,7 +393,8 @@ static int kvm_mips_##name##_pgd(pgd_t *pgd, unsigned long start, \
388393
if (!pgd_present(pgd[i])) \
389394
continue; \
390395
\
391-
pud = pud_offset(pgd + i, 0); \
396+
p4d = p4d_offset(pgd, 0); \
397+
pud = pud_offset(p4d + i, 0); \
392398
if (i == i_max) \
393399
cur_end = end; \
394400
\
@@ -916,6 +922,7 @@ static bool kvm_mips_flush_gva_pud(pud_t *pud, unsigned long start_gva,
916922
static bool kvm_mips_flush_gva_pgd(pgd_t *pgd, unsigned long start_gva,
917923
unsigned long end_gva)
918924
{
925+
p4d_t *p4d;
919926
pud_t *pud;
920927
unsigned long end = ~0ul;
921928
int i_min = pgd_index(start_gva);
@@ -927,7 +934,8 @@ static bool kvm_mips_flush_gva_pgd(pgd_t *pgd, unsigned long start_gva,
927934
if (!pgd_present(pgd[i]))
928935
continue;
929936

930-
pud = pud_offset(pgd + i, 0);
937+
p4d = p4d_offset(pgd, 0);
938+
pud = pud_offset(p4d + i, 0);
931939
if (i == i_max)
932940
end = end_gva;
933941

‎arch/mips/kvm/trap_emul.c

+3-1
Original file line numberDiff line numberDiff line change
@@ -564,6 +564,7 @@ static void kvm_mips_emul_free_gva_pt(pgd_t *pgd)
564564
/* Don't free host kernel page tables copied from init_mm.pgd */
565565
const unsigned long end = 0x80000000;
566566
unsigned long pgd_va, pud_va, pmd_va;
567+
p4d_t *p4d;
567568
pud_t *pud;
568569
pmd_t *pmd;
569570
pte_t *pte;
@@ -576,7 +577,8 @@ static void kvm_mips_emul_free_gva_pt(pgd_t *pgd)
576577
pgd_va = (unsigned long)i << PGDIR_SHIFT;
577578
if (pgd_va >= end)
578579
break;
579-
pud = pud_offset(pgd + i, 0);
580+
p4d = p4d_offset(pgd, 0);
581+
pud = pud_offset(p4d + i, 0);
580582
for (j = 0; j < PTRS_PER_PUD; j++) {
581583
if (pud_none(pud[j]))
582584
continue;

‎arch/mips/mm/c-r3k.c

+3-1
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,7 @@ static void r3k_flush_cache_page(struct vm_area_struct *vma,
241241
int exec = vma->vm_flags & VM_EXEC;
242242
struct mm_struct *mm = vma->vm_mm;
243243
pgd_t *pgdp;
244+
p4d_t *p4dp;
244245
pud_t *pudp;
245246
pmd_t *pmdp;
246247
pte_t *ptep;
@@ -253,7 +254,8 @@ static void r3k_flush_cache_page(struct vm_area_struct *vma,
253254
return;
254255

255256
pgdp = pgd_offset(mm, addr);
256-
pudp = pud_offset(pgdp, addr);
257+
p4dp = p4d_offset(pgdp, addr);
258+
pudp = pud_offset(p4dp, addr);
257259
pmdp = pmd_offset(pudp, addr);
258260
ptep = pte_offset(pmdp, addr);
259261

‎arch/mips/mm/c-r4k.c

+3-1
Original file line numberDiff line numberDiff line change
@@ -654,6 +654,7 @@ static inline void local_r4k_flush_cache_page(void *args)
654654
struct mm_struct *mm = vma->vm_mm;
655655
int map_coherent = 0;
656656
pgd_t *pgdp;
657+
p4d_t *p4dp;
657658
pud_t *pudp;
658659
pmd_t *pmdp;
659660
pte_t *ptep;
@@ -668,7 +669,8 @@ static inline void local_r4k_flush_cache_page(void *args)
668669

669670
addr &= PAGE_MASK;
670671
pgdp = pgd_offset(mm, addr);
671-
pudp = pud_offset(pgdp, addr);
672+
p4dp = p4d_offset(pgdp, addr);
673+
pudp = pud_offset(p4dp, addr);
672674
pmdp = pmd_offset(pudp, addr);
673675
ptep = pte_offset(pmdp, addr);
674676

‎arch/mips/mm/c-tx39.c

+3-1
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page
170170
int exec = vma->vm_flags & VM_EXEC;
171171
struct mm_struct *mm = vma->vm_mm;
172172
pgd_t *pgdp;
173+
p4d_t *p4dp;
173174
pud_t *pudp;
174175
pmd_t *pmdp;
175176
pte_t *ptep;
@@ -183,7 +184,8 @@ static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page
183184

184185
page &= PAGE_MASK;
185186
pgdp = pgd_offset(mm, page);
186-
pudp = pud_offset(pgdp, page);
187+
p4dp = p4d_offset(pgdp, page);
188+
pudp = pud_offset(p4dp, page);
187189
pmdp = pmd_offset(pudp, page);
188190
ptep = pte_offset(pmdp, page);
189191

‎arch/mips/mm/fault.c

+8-2
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
294294
*/
295295
int offset = pgd_index(address);
296296
pgd_t *pgd, *pgd_k;
297+
p4d_t *p4d, *p4d_k;
297298
pud_t *pud, *pud_k;
298299
pmd_t *pmd, *pmd_k;
299300
pte_t *pte_k;
@@ -305,8 +306,13 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
305306
goto no_context;
306307
set_pgd(pgd, *pgd_k);
307308

308-
pud = pud_offset(pgd, address);
309-
pud_k = pud_offset(pgd_k, address);
309+
p4d = p4d_offset(pgd, address);
310+
p4d_k = p4d_offset(pgd_k, address);
311+
if (!p4d_present(*p4d_k))
312+
goto no_context;
313+
314+
pud = pud_offset(p4d, address);
315+
pud_k = pud_offset(p4d_k, address);
310316
if (!pud_present(*pud_k))
311317
goto no_context;
312318

‎arch/mips/mm/hugetlbpage.c

+10-4
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr,
2525
unsigned long sz)
2626
{
2727
pgd_t *pgd;
28+
p4d_t *p4d;
2829
pud_t *pud;
2930
pte_t *pte = NULL;
3031

3132
pgd = pgd_offset(mm, addr);
32-
pud = pud_alloc(mm, pgd, addr);
33+
p4d = p4d_alloc(mm, pgd, addr);
34+
pud = pud_alloc(mm, p4d, addr);
3335
if (pud)
3436
pte = (pte_t *)pmd_alloc(mm, pud, addr);
3537

@@ -40,14 +42,18 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
4042
unsigned long sz)
4143
{
4244
pgd_t *pgd;
45+
p4d_t *p4d;
4346
pud_t *pud;
4447
pmd_t *pmd = NULL;
4548

4649
pgd = pgd_offset(mm, addr);
4750
if (pgd_present(*pgd)) {
48-
pud = pud_offset(pgd, addr);
49-
if (pud_present(*pud))
50-
pmd = pmd_offset(pud, addr);
51+
p4d = p4d_offset(pgd, addr);
52+
if (p4d_present(*p4d)) {
53+
pud = pud_offset(p4d, addr);
54+
if (pud_present(*pud))
55+
pmd = pmd_offset(pud, addr);
56+
}
5157
}
5258
return (pte_t *) pmd;
5359
}

‎arch/mips/mm/ioremap.c

+5-1
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,15 @@ static int remap_area_pages(unsigned long address, phys_addr_t phys_addr,
7878
flush_cache_all();
7979
BUG_ON(address >= end);
8080
do {
81+
p4d_t *p4d;
8182
pud_t *pud;
8283
pmd_t *pmd;
8384

8485
error = -ENOMEM;
85-
pud = pud_alloc(&init_mm, dir, address);
86+
p4d = p4d_alloc(&init_mm, dir, address);
87+
if (!p4d)
88+
break;
89+
pud = pud_alloc(&init_mm, p4d, address);
8690
if (!pud)
8791
break;
8892
pmd = pmd_alloc(&init_mm, pud, address);

‎arch/mips/mm/pgtable-32.c

+3-1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ void __init pagetable_init(void)
5656
pgd_t *pgd_base;
5757
#ifdef CONFIG_HIGHMEM
5858
pgd_t *pgd;
59+
p4d_t *p4d;
5960
pud_t *pud;
6061
pmd_t *pmd;
6162
pte_t *pte;
@@ -82,7 +83,8 @@ void __init pagetable_init(void)
8283
fixrange_init(vaddr & PMD_MASK, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
8384

8485
pgd = swapper_pg_dir + pgd_index(vaddr);
85-
pud = pud_offset(pgd, vaddr);
86+
p4d = p4d_offset(pgd, vaddr);
87+
pud = pud_offset(p4d, vaddr);
8688
pmd = pmd_offset(pud, vaddr);
8789
pte = pte_offset_kernel(pmd, vaddr);
8890
pkmap_page_table = pte;

‎arch/mips/mm/tlb-r4k.c

+3-1
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
295295
{
296296
unsigned long flags;
297297
pgd_t *pgdp;
298+
p4d_t *p4dp;
298299
pud_t *pudp;
299300
pmd_t *pmdp;
300301
pte_t *ptep;
@@ -320,7 +321,8 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
320321
mtc0_tlbw_hazard();
321322
tlb_probe();
322323
tlb_probe_hazard();
323-
pudp = pud_offset(pgdp, address);
324+
p4dp = p4d_offset(pgdp, address);
325+
pudp = pud_offset(p4dp, address);
324326
pmdp = pmd_offset(pudp, address);
325327
idx = read_c0_index();
326328
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT

0 commit comments

Comments
 (0)