pgtable_64.h 9.4 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3
#ifndef _ASM_X86_PGTABLE_64_H
#define _ASM_X86_PGTABLE_64_H
Linus Torvalds's avatar
Linus Torvalds committed
4

5
#include <linux/const.h>
6 7
#include <asm/pgtable_64_types.h>

8 9
#ifndef __ASSEMBLY__

Linus Torvalds's avatar
Linus Torvalds committed
10 11 12 13 14
/*
 * This file contains the functions and defines necessary to modify and use
 * the x86-64 page table tree.
 */
#include <asm/processor.h>
Jiri Slaby's avatar
Jiri Slaby committed
15
#include <linux/bitops.h>
Linus Torvalds's avatar
Linus Torvalds committed
16
#include <linux/threads.h>
17
#include <asm/fixmap.h>
Linus Torvalds's avatar
Linus Torvalds committed
18

19 20
extern p4d_t level4_kernel_pgt[512];
extern p4d_t level4_ident_pgt[512];
Linus Torvalds's avatar
Linus Torvalds committed
21 22 23
extern pud_t level3_kernel_pgt[512];
extern pud_t level3_ident_pgt[512];
extern pmd_t level2_kernel_pgt[512];
24 25
extern pmd_t level2_fixmap_pgt[512];
extern pmd_t level2_ident_pgt[512];
26
extern pte_t level1_fixmap_pgt[512 * FIXMAP_PMD_NUM];
27
extern pgd_t init_top_pgt[];
Linus Torvalds's avatar
Linus Torvalds committed
28

29
#define swapper_pg_dir init_top_pgt
Linus Torvalds's avatar
Linus Torvalds committed
30 31

extern void paging_init(void);
32
static inline void sync_initial_page_table(void) { }
Linus Torvalds's avatar
Linus Torvalds committed
33

34
#define pte_ERROR(e)					\
35
	pr_err("%s:%d: bad pte %p(%016lx)\n",		\
36 37
	       __FILE__, __LINE__, &(e), pte_val(e))
#define pmd_ERROR(e)					\
38
	pr_err("%s:%d: bad pmd %p(%016lx)\n",		\
39 40
	       __FILE__, __LINE__, &(e), pmd_val(e))
#define pud_ERROR(e)					\
41
	pr_err("%s:%d: bad pud %p(%016lx)\n",		\
42
	       __FILE__, __LINE__, &(e), pud_val(e))
43 44 45 46 47 48 49

#if CONFIG_PGTABLE_LEVELS >= 5
#define p4d_ERROR(e)					\
	pr_err("%s:%d: bad p4d %p(%016lx)\n",		\
	       __FILE__, __LINE__, &(e), p4d_val(e))
#endif

50
#define pgd_ERROR(e)					\
51
	pr_err("%s:%d: bad pgd %p(%016lx)\n",		\
52
	       __FILE__, __LINE__, &(e), pgd_val(e))
Linus Torvalds's avatar
Linus Torvalds committed
53

54 55
struct mm_struct;

56
void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte);
57 58
void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte);

59 60
static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
				    pte_t *ptep)
Linus Torvalds's avatar
Linus Torvalds committed
61
{
62 63
	*ptep = native_make_pte(0);
}
Linus Torvalds's avatar
Linus Torvalds committed
64

65
static inline void native_set_pte(pte_t *ptep, pte_t pte)
Linus Torvalds's avatar
Linus Torvalds committed
66
{
67 68
	*ptep = pte;
}
Linus Torvalds's avatar
Linus Torvalds committed
69

70 71 72 73 74
static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
{
	native_set_pte(ptep, pte);
}

75 76 77 78 79 80 81 82 83 84
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
	*pmdp = pmd;
}

static inline void native_pmd_clear(pmd_t *pmd)
{
	native_set_pmd(pmd, native_make_pmd(0));
}

85
static inline pte_t native_ptep_get_and_clear(pte_t *xp)
Linus Torvalds's avatar
Linus Torvalds committed
86
{
87 88 89
#ifdef CONFIG_SMP
	return native_make_pte(xchg(&xp->pte, 0));
#else
90 91
	/* native_local_ptep_get_and_clear,
	   but duplicated because of cyclic dependency */
92 93 94 95
	pte_t ret = *xp;
	native_pte_clear(NULL, 0, xp);
	return ret;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
96 97
}

98
static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
Linus Torvalds's avatar
Linus Torvalds committed
99
{
100 101 102 103 104 105 106 107 108
#ifdef CONFIG_SMP
	return native_make_pmd(xchg(&xp->pmd, 0));
#else
	/* native_local_pmdp_get_and_clear,
	   but duplicated because of cyclic dependency */
	pmd_t ret = *xp;
	native_pmd_clear(xp);
	return ret;
#endif
109
}
Linus Torvalds's avatar
Linus Torvalds committed
110

111
static inline void native_set_pud(pud_t *pudp, pud_t pud)
Linus Torvalds's avatar
Linus Torvalds committed
112
{
113
	*pudp = pud;
Linus Torvalds's avatar
Linus Torvalds committed
114 115
}

116 117 118 119
static inline void native_pud_clear(pud_t *pud)
{
	native_set_pud(pud, native_make_pud(0));
}
120

121 122 123 124 125 126 127 128 129 130 131 132 133
static inline pud_t native_pudp_get_and_clear(pud_t *xp)
{
#ifdef CONFIG_SMP
	return native_make_pud(xchg(&xp->pud, 0));
#else
	/* native_local_pudp_get_and_clear,
	 * but duplicated because of cyclic dependency
	 */
	pud_t ret = *xp;

	native_pud_clear(xp);
	return ret;
#endif
134 135
}

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
 * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
 * (8k-aligned and 8k in size).  The kernel one is at the beginning 4k and
 * the user one is in the last 4k.  To switch between them, you
 * just need to flip the 12th bit in their addresses.
 */
#define PTI_PGTABLE_SWITCH_BIT	PAGE_SHIFT

/*
 * This generates better code than the inline assembly in
 * __set_bit().
 */
static inline void *ptr_set_bit(void *ptr, int bit)
{
	unsigned long __ptr = (unsigned long)ptr;

	__ptr |= BIT(bit);
	return (void *)__ptr;
}
static inline void *ptr_clear_bit(void *ptr, int bit)
{
	unsigned long __ptr = (unsigned long)ptr;

	__ptr &= ~BIT(bit);
	return (void *)__ptr;
}

static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
{
	return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
{
	return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
{
	return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}

static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
{
	return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}
#endif /* CONFIG_PAGE_TABLE_ISOLATION */

/*
 * Page table pages are page-aligned.  The lower half of the top
 * level is used for userspace and the top half for the kernel.
 *
 * Returns true for parts of the PGD that map userspace and
 * false for the parts that map the kernel.
 */
static inline bool pgdp_maps_userspace(void *__ptr)
{
	unsigned long ptr = (unsigned long)__ptr;

	return (ptr & ~PAGE_MASK) < (PAGE_SIZE / 2);
}

#ifdef CONFIG_PAGE_TABLE_ISOLATION
pgd_t __pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd);

/*
 * Take a PGD location (pgdp) and a pgd value that needs to be set there.
 * Populates the user and returns the resulting PGD that must be set in
 * the kernel copy of the page tables.
 */
static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
{
	if (!static_cpu_has(X86_FEATURE_PTI))
		return pgd;
	return __pti_set_user_pgd(pgdp, pgd);
}
#else
static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
{
	return pgd;
}
#endif

220 221
static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
{
222 223 224
#if defined(CONFIG_PAGE_TABLE_ISOLATION) && !defined(CONFIG_X86_5LEVEL)
	p4dp->pgd = pti_set_user_pgd(&p4dp->pgd, p4d.pgd);
#else
225
	*p4dp = p4d;
226
#endif
227 228 229 230
}

static inline void native_p4d_clear(p4d_t *p4d)
{
231 232 233
#ifdef CONFIG_X86_5LEVEL
	native_set_p4d(p4d, native_make_p4d(0));
#else
234
	native_set_p4d(p4d, (p4d_t) { .pgd = native_make_pgd(0)});
235
#endif
236 237
}

238 239
static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
{
240 241 242
#ifdef CONFIG_PAGE_TABLE_ISOLATION
	*pgdp = pti_set_user_pgd(pgdp, pgd);
#else
243
	*pgdp = pgd;
244
#endif
245
}
246

247
static inline void native_pgd_clear(pgd_t *pgd)
248
{
249
	native_set_pgd(pgd, native_make_pgd(0));
250 251
}

252
extern void sync_global_pgds(unsigned long start, unsigned long end);
253

Linus Torvalds's avatar
Linus Torvalds committed
254 255 256 257 258 259 260 261
/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 */

/*
 * Level 4 access.
 */
262
static inline int pgd_large(pgd_t pgd) { return 0; }
263
#define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE)
Linus Torvalds's avatar
Linus Torvalds committed
264 265 266 267 268 269 270 271

/* PUD - Level3 access */

/* PMD  - Level 2 access */

/* PTE - Level 1 access. */

/* x86-64 always has all page tables mapped. */
272
#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
273
#define pte_unmap(pte) ((void)(pte))/* NOP */
Linus Torvalds's avatar
Linus Torvalds committed
274

275 276 277
/*
 * Encode and de-code a swap entry
 *
278 279
 * |     ...            | 11| 10|  9|8|7|6|5| 4| 3|2| 1|0| <- bit number
 * |     ...            |SW3|SW2|SW1|G|L|D|A|CD|WT|U| W|P| <- bit names
280
 * | TYPE (59-63) | ~OFFSET (9-58)  |0|0|X|X| X| X|X|SD|0| <- swp entry
281 282 283 284 285 286
 *
 * G (8) is aliased and used as a PROT_NONE indicator for
 * !present ptes.  We need to start storing swap entries above
 * there.  We also need to avoid using A and D because of an
 * erratum where they can be incorrectly set by hardware on
 * non-present PTEs.
287 288 289 290 291 292
 *
 * SD (1) in swp entry is used to store soft dirty bit, which helps us
 * remember soft dirty over page migration
 *
 * Bit 7 in swp entry should be 0 because pmd_present checks not only P,
 * but also L and G.
293 294 295
 *
 * The offset is inverted by a binary not operation to make the high
 * physical bits set.
296
 */
297 298 299 300 301 302
#define SWP_TYPE_BITS		5

#define SWP_OFFSET_FIRST_BIT	(_PAGE_BIT_PROTNONE + 1)

/* We always extract/encode the offset by shifting it all the way up, and then down again */
#define SWP_OFFSET_SHIFT	(SWP_OFFSET_FIRST_BIT+SWP_TYPE_BITS)
303 304 305

#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)

306 307 308 309
/* Extract the high bits for type */
#define __swp_type(x) ((x).val >> (64 - SWP_TYPE_BITS))

/* Shift up (to get rid of type), then down to get value */
310
#define __swp_offset(x) (~(x).val << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT)
311 312 313

/*
 * Shift the offset up "too far" by TYPE bits, then down again
314 315
 * The offset is inverted by a binary not operation to make the high
 * physical bits set.
316 317
 */
#define __swp_entry(type, offset) ((swp_entry_t) { \
318
	(~(unsigned long)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \
319 320
	| ((unsigned long)(type) << (64-SWP_TYPE_BITS)) })

321
#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val((pte)) })
322
#define __pmd_to_swp_entry(pmd)		((swp_entry_t) { pmd_val((pmd)) })
323
#define __swp_entry_to_pte(x)		((pte_t) { .pte = (x).val })
324
#define __swp_entry_to_pmd(x)		((pmd_t) { .pmd = (x).val })
Linus Torvalds's avatar
Linus Torvalds committed
325

326
extern int kern_addr_valid(unsigned long addr);
327
extern void cleanup_highmap(void);
Linus Torvalds's avatar
Linus Torvalds committed
328 329

#define HAVE_ARCH_UNMAPPED_AREA
330
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
Linus Torvalds's avatar
Linus Torvalds committed
331 332

#define pgtable_cache_init()   do { } while (0)
333
#define check_pgt_cache()      do { } while (0)
Linus Torvalds's avatar
Linus Torvalds committed
334 335 336 337 338 339

#define PAGE_AGP    PAGE_KERNEL_NOCACHE
#define HAVE_PAGE_AGP 1

/* fs/proc/kcore.c */
#define	kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK)
340
#define	kc_offset_to_vaddr(o) ((o) | ~__VIRTUAL_MASK)
Linus Torvalds's avatar
Linus Torvalds committed
341 342

#define __HAVE_ARCH_PTE_SAME
343

344 345 346 347 348
#define vmemmap ((struct page *)VMEMMAP_START)

extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);

349 350 351 352 353 354 355 356 357 358 359 360 361 362
#define gup_fast_permitted gup_fast_permitted
static inline bool gup_fast_permitted(unsigned long start, int nr_pages,
		int write)
{
	unsigned long len, end;

	len = (unsigned long)nr_pages << PAGE_SHIFT;
	end = start + len;
	if (end < start)
		return false;
	if (end >> __VIRTUAL_MASK_SHIFT)
		return false;
	return true;
}
363

364 365
#include <asm/pgtable-invert.h>

366
#endif /* !__ASSEMBLY__ */
367
#endif /* _ASM_X86_PGTABLE_64_H */