My Project
Loading...
Searching...
No Matches
mm.h File Reference
#include <stdint.h>
#include <stdbool.h>
#include "annotations.h"
#include "macros.h"
#include "../mtstatus.h"
#include "../intrinsics/intrin.h"
#include "../intrinsics/atomic.h"
#include "ms.h"
#include "core.h"
#include "efi.h"

Go to the source code of this file.

Data Structures

struct  _MMPTE
struct  _PFN_ENTRY
struct  _MM_PFN_LIST
struct  _MM_PFN_DATABASE
struct  _MMVAD
struct  _POOL_HEADER
struct  _POOL_DESCRIPTOR

Macros

#define PML4_INDEX_BITS   9
#define PML4_INDEX_SHIFT   39
#define PML4_INDEX_MASK   ((1ULL << PML4_INDEX_BITS) - 1ULL)
#define PML4_INDEX_FROM_VA(VA)
#define PML4_INDEX_FROM_PHYS(PHYS)
#define VirtualPageSize   4096ULL
#define PhysicalFrameSize   4096ULL
#define KernelVaStart   0xfffff80000000000ULL
#define PhysicalMemoryOffset   0xffff880000000000ULL
#define RECURSIVE_INDEX   0x1FF
#define INDEX_TO_PPFN(Index)
#define PHYSICAL_TO_PPFN(PHYS)
#define PTE_TO_PHYSICAL(PMMPTE)
#define MI_WRITE_PTE(_PtePointer, _Va, _Pa, _Flags)
#define PPFN_TO_INDEX(PPFN)
#define PPFN_TO_PHYSICAL_ADDRESS(PPFN)
#define VA_OFFSET(_VirtualAddress)
#define MM_IS_DEMAND_ZERO_PTE(pte)
#define MM_SET_DEMAND_ZERO_PTE(pte, prot_flags, nx)
#define MM_UNSET_DEMAND_ZERO_PTE(pte)
#define BYTES_TO_PAGES(Bytes)
#define PAGES_TO_BYTES(Pages)
#define MAX_POOL_DESCRIPTORS   7
#define _32KB_POOL   1
#define _64KB_POOL   2
#define _128KB_POOL   3
#define _256KB_POOL   4
#define _512KB_POOL   5
#define _1024KB_POOL   6
#define _2048KB_POOL   7
#define POOL_MIN_ALLOC   32
#define USER_VA_END   0x00007FFFFFFFFFFF
#define USER_VA_START   0x10000
#define POOL_MAX_ALLOC   2048
#define MI_NONPAGED_POOL_SIZE   ((size_t)16ULL * 1024 * 1024 * 1024)
#define MI_PAGED_POOL_SIZE   ((size_t)32ULL * 1024 * 1024 * 1024)
#define NONPAGED_POOL_VA_TOTAL_PAGES   (MI_NONPAGED_POOL_SIZE / VirtualPageSize)
#define PAGED_POOL_VA_TOTAL_PAGES   (MI_PAGED_POOL_SIZE / VirtualPageSize)
#define NONPAGED_POOL_VA_BITMAP_QWORDS   ((NONPAGED_POOL_VA_TOTAL_PAGES + 63) / 64)
#define PAGED_POOL_VA_BITMAP_QWORDS   ((PAGED_POOL_VA_TOTAL_PAGES + 63) / 64)
#define MI_NONPAGED_BITMAP_PAGES_NEEDED   ((NONPAGED_POOL_VA_BITMAP_QWORDS * sizeof(uint64_t) + VirtualPageSize - 1) / VirtualPageSize)
#define MI_PAGED_BITMAP_PAGES_NEEDED   ((PAGED_POOL_VA_BITMAP_QWORDS * sizeof(uint64_t) + VirtualPageSize - 1) / VirtualPageSize)
#define ALIGN_UP(x, align)
#define MI_NONPAGED_BITMAP_BASE   ALIGN_UP(LK_KERNEL_END, VirtualPageSize)
#define MI_NONPAGED_BITMAP_END   (MI_NONPAGED_BITMAP_BASE + MI_NONPAGED_BITMAP_PAGES_NEEDED * VirtualPageSize)
#define MI_PAGED_BITMAP_BASE   ALIGN_UP(MI_NONPAGED_BITMAP_END, VirtualPageSize)
#define MI_PAGED_BITMAP_END   (MI_PAGED_BITMAP_BASE + MI_PAGED_BITMAP_PAGES_NEEDED * VirtualPageSize)
#define MI_NONPAGED_POOL_BASE   ALIGN_UP(MI_NONPAGED_BITMAP_END, VirtualPageSize)
#define MI_NONPAGED_POOL_END   (MI_NONPAGED_POOL_BASE + MI_NONPAGED_POOL_SIZE)
#define MI_PAGED_POOL_BASE   ALIGN_UP(MI_NONPAGED_POOL_END, VirtualPageSize)
#define MI_PAGED_POOL_END   (MI_PAGED_POOL_BASE + MI_PAGED_POOL_SIZE)
#define MI_IS_CANONICAL_ADDR(va)
#define PFN_TO_PHYS(Pfn)
#define PHYS_TO_INDEX(PhysicalAddress)
#define PFN_ERROR   UINT64_T_MAX
#define PROT_KERNEL_READ   0x1
#define PROT_KERNEL_WRITE   0x2
#define MI_DEMAND_ZERO_BIT   (1ULL << 16)
#define MM_POOL_CANARY   'BEKA'
#define MI_STACK_SIZE   0x4000
#define MI_LARGE_STACK_SIZE   0xf000
#define MI_GUARD_PAGE_PROTECTION   (1ULL << 17)
#define MmFullBarrier()
#define MmBarrier()
#define MmIsAddressValid(VirtualAddress)
#define PAGE_PAT   (1ULL << 7)

Typedefs

typedef uint64_t PAGE_INDEX
typedef enum _PFN_STATE PFN_STATE
typedef enum _PFN_FLAGS PFN_FLAGS
typedef enum _VAD_FLAGS VAD_FLAGS
typedef enum _PAGE_FLAGS PAGE_FLAGS
typedef enum _POOL_TYPE POOL_TYPE
typedef enum _FAULT_OPERATION FAULT_OPERATION
typedef enum _FAULT_OPERATIONPFAULT_OPERATION
typedef enum _PRIVILEGE_MODE PRIVILEGE_MODE
typedef enum _PRIVILEGE_MODEPPRIVILEGE_MODE
typedef enum _MEMORY_CACHING_TYPE MEMORY_CACHING_TYPE
typedef enum _SYSTEM_PHASE_ROUTINE SYSTEM_PHASE_ROUTINE
typedef struct _MMPTE MMPTE
typedef struct _MMPTEPMMPTE
typedef struct _PFN_ENTRY PFN_ENTRY
typedef struct _PFN_ENTRYPPFN_ENTRY
typedef struct _MM_PFN_LIST MM_PFN_LIST
typedef struct _MM_PFN_DATABASE MM_PFN_DATABASE
typedef struct _MMVAD MMVAD
typedef struct _MMVADPMMVAD
typedef struct _POOL_HEADER POOL_HEADER
typedef struct _POOL_HEADERPPOOL_HEADER
typedef struct _POOL_DESCRIPTOR POOL_DESCRIPTOR
typedef struct _POOL_DESCRIPTORPPOOL_DESCRIPTOR

Enumerations

enum  _PFN_STATE {
  PfnStateActive , PfnStateStandby , PfnStateModified , PfnStateFree ,
  PfnStateZeroed , PfnStateTransition , PfnStateBad
}
enum  _PFN_FLAGS {
  PFN_FLAG_NONE = 0 , PFN_FLAG_NONPAGED = (1U << 0) , PFN_FLAG_COPY_ON_WRITE = (1U << 1) , PFN_FLAG_MAPPED_FILE = (1U << 2) ,
  PFN_FLAG_LOCKED_FOR_IO = (1U << 3)
}
enum  _VAD_FLAGS {
  VAD_FLAG_NONE = 0 , VAD_FLAG_READ = (1U << 0) , VAD_FLAG_WRITE = (1U << 1) , VAD_FLAG_EXECUTE = (1U << 2) ,
  VAD_FLAG_PRIVATE = (1U << 3) , VAD_FLAG_MAPPED_FILE = (1U << 4) , VAD_FLAG_COPY_ON_WRITE = (1U << 5)
}
enum  _PAGE_FLAGS {
  PAGE_PRESENT = 1 << 0 , PAGE_RW = 1 << 1 , PAGE_USER = 1 << 2 , PAGE_PWT = 0x8 ,
  PAGE_PCD = 0x10 , PAGE_ACCESSED = 0x20 , PAGE_DIRTY = 0x40 , PAGE_PS = 0x80 ,
  PAGE_GLOBAL = 0x100 , PAGE_NX = (1ULL << 63)
}
enum  _POOL_TYPE {
  NonPagedPool = 0 , PagedPool = 1 , NonPagedPoolCacheAligned = 2 , PagedPoolCacheAligned = 3 ,
  NonPagedPoolNx = 4 , PagedPoolNx = 5
}
enum  _FAULT_OPERATION { FaultOpInvalid = -1 , ReadOperation = 0 , WriteOperation = 2 , ExecuteOperation = 10 }
enum  _PRIVILEGE_MODE { KernelMode = 0 , UserMode = 1 }
enum  _MEMORY_CACHING_TYPE {
  MmNonCached = 0 , MmCached , MmWriteCombined , MmWriteThrough ,
  MmNonCachedUnordered , MmUSWCCached , MmHardwareCoherentCached
}
enum  _SYSTEM_PHASE_ROUTINE { SYSTEM_PHASE_INITIALIZE_ALL = 1 , SYSTEM_PHASE_INITIALIZE_PAT_ONLY = 2 }

Functions

uint64_t * pml4_from_recursive (void)
FORCEINLINE void * kmemset (void *dest, int64_t val, uint64_t len)
FORCEINLINE void * kmemcpy (void *dest, const void *src, size_t len)
FORCEINLINE int kmemcmp (const void *s1, const void *s2, size_t n)
void MiReloadTLBs (void)
FORCEINLINE uint64_t MiCacheToFlags (MEMORY_CACHING_TYPE type)
FORCEINLINE FAULT_OPERATION MiRetrieveOperationFromErrorCode (uint64_t ErrorCode)
FORCEINLINE uint64_t MiRetrieveLastFaultyAddress (void)
FORCEINLINE void MiAtomicExchangePte (PMMPTE PtePtr, uint64_t NewPteValue)
void MiInvalidateTlbForVa (IN void *VirtualAddress)
FORCEINLINE bool MiIsValidPfn (IN PAGE_INDEX Pfn)
MTSTATUS MiInitializePfnDatabase (IN PBOOT_INFO BootInfo)
PAGE_INDEX MiRequestPhysicalPage (IN PFN_STATE ListType)
void MiReleasePhysicalPage (IN PAGE_INDEX PfnIndex)
void MiUnlinkPageFromList (PPFN_ENTRY pfn)
PMMPTE MiGetPml4ePointer (IN uintptr_t va)
PMMPTE MiGetPdptePointer (IN uintptr_t va)
PMMPTE MiGetPdePointer (IN uintptr_t va)
PMMPTE MiGetPtePointer (IN uintptr_t va)
uint64_t MiTranslatePteToVa (IN PMMPTE pte)
PAGE_INDEX MiTranslatePteToPfn (IN PMMPTE pte)
uintptr_t MiTranslateVirtualToPhysical (IN void *VirtualAddress)
void MiUnmapPte (IN PMMPTE pte)
bool MmIsAddressPresent (IN uintptr_t VirtualAddress)
void * MiMapPageInHyperspace (IN uint64_t PfnIndex, OUT PIRQL OldIrql)
void MiUnmapHyperSpaceMap (IN IRQL OldIrql)
MTSTATUS MiInitializePoolSystem (void)
void * MmAllocatePoolWithTag (IN enum _POOL_TYPE PoolType, IN size_t NumberOfBytes, IN uint32_t Tag)
void MmFreePool (IN void *buf)
void * MiCreateKernelStack (IN bool LargeStack)
void MiFreeKernelStack (IN void *AllocatedStackTop, IN bool LargeStack)
MTSTATUS MmCreateProcessAddressSpace (OUT void **DirectoryTable)
MTSTATUS MmAllocateVirtualMemory (IN PEPROCESS Process, _In_Opt _Out_Opt void **BaseAddress, IN size_t NumberOfBytes, IN VAD_FLAGS VadFlags)
MTSTATUS MmFreeVirtualMemory (IN PEPROCESS Process, IN void *BaseAddress)
PMMVAD MiFindVad (IN PMMVAD Root, IN uintptr_t VirtualAddress)
void MiMoveUefiDataToHigherHalf (IN PBOOT_INFO BootInfo)
uintptr_t MmFindFreeAddressSpace (IN PEPROCESS Process, IN size_t NumberOfBytes, IN uintptr_t SearchStart, IN uintptr_t SearchEnd)
bool MiInitializePoolVaSpace (void)
uintptr_t MiAllocatePoolVa (IN POOL_TYPE PoolType, IN size_t NumberOfBytes)
void MiFreePoolVaContiguous (IN uintptr_t va, IN size_t NumberOfBytes, IN POOL_TYPE PoolType)
MTSTATUS MmAccessFault (IN uint64_t FaultBits, IN uint64_t VirtualAddress, IN PRIVILEGE_MODE PreviousMode, IN PTRAP_FRAME TrapFrame)
bool MmInvalidAccessAllowed (void)
bool MiCheckForContigiousMemory (IN void *StartAddress, IN size_t NumberOfBytes)
void * MmAllocateContigiousMemory (IN size_t NumberOfBytes, IN uint64_t HighestAcceptableAddress)
void MmFreeContigiousMemory (IN void *BaseAddress, IN size_t NumberOfBytes)
void * MmMapIoSpace (IN uintptr_t PhysicalAddress, IN size_t NumberOfBytes, IN MEMORY_CACHING_TYPE CacheType)
bool MmInitSystem (IN uint8_t Phase, IN PBOOT_INFO BootInformation)

Variables

MM_PFN_DATABASE PfnDatabase
bool MmPfnDatabaseInitialized
PAGE_INDEX MmHighestPfn
uintptr_t MmSystemRangeStart
uintptr_t MmHighestUserAddress
uintptr_t MmUserProbeAddress
uintptr_t MmNonPagedPoolStart
uintptr_t MmNonPagedPoolEnd
uintptr_t MmPagedPoolStart
uintptr_t MmPagedPoolEnd

Macro Definition Documentation

◆ _1024KB_POOL

#define _1024KB_POOL   6

Definition at line 157 of file mm.h.

◆ _128KB_POOL

#define _128KB_POOL   3

Definition at line 154 of file mm.h.

◆ _2048KB_POOL

#define _2048KB_POOL   7

Definition at line 158 of file mm.h.

◆ _256KB_POOL

#define _256KB_POOL   4

Definition at line 155 of file mm.h.

◆ _32KB_POOL

#define _32KB_POOL   1

Definition at line 152 of file mm.h.

◆ _512KB_POOL

#define _512KB_POOL   5

Definition at line 156 of file mm.h.

◆ _64KB_POOL

#define _64KB_POOL   2

Definition at line 153 of file mm.h.

◆ ALIGN_UP

#define ALIGN_UP ( x,
align )
Value:
(((uintptr_t)(x) + ((align)-1)) & ~((uintptr_t)((align)-1)))

Definition at line 181 of file mm.h.

◆ BYTES_TO_PAGES

#define BYTES_TO_PAGES ( Bytes)
Value:
(((Bytes) + VirtualPageSize - 1) / VirtualPageSize)
#define VirtualPageSize
Definition mm.h:53

Definition at line 147 of file mm.h.

◆ INDEX_TO_PPFN

#define INDEX_TO_PPFN ( Index)
Value:
(&(PfnDatabase.PfnEntries[(size_t)(Index)]))
MM_PFN_DATABASE PfnDatabase
Definition pfn.c:29

Definition at line 62 of file mm.h.

◆ KernelVaStart

#define KernelVaStart   0xfffff80000000000ULL

Definition at line 55 of file mm.h.

◆ MAX_POOL_DESCRIPTORS

#define MAX_POOL_DESCRIPTORS   7

Definition at line 151 of file mm.h.

◆ MI_DEMAND_ZERO_BIT

#define MI_DEMAND_ZERO_BIT   (1ULL << 16)

Definition at line 213 of file mm.h.

◆ MI_GUARD_PAGE_PROTECTION

#define MI_GUARD_PAGE_PROTECTION   (1ULL << 17)

Definition at line 221 of file mm.h.

◆ MI_IS_CANONICAL_ADDR

#define MI_IS_CANONICAL_ADDR ( va)
Value:
({ \
uint64_t _va = (uint64_t)(va); \
uint64_t _mask = ~((1ULL << 48) - 1); /* bits 63:48 */ \
((_va & _mask) == 0 || (_va & _mask) == _mask); \
})

Definition at line 198 of file mm.h.

◆ MI_LARGE_STACK_SIZE

#define MI_LARGE_STACK_SIZE   0xf000

Definition at line 220 of file mm.h.

◆ MI_NONPAGED_BITMAP_BASE

#define MI_NONPAGED_BITMAP_BASE   ALIGN_UP(LK_KERNEL_END, VirtualPageSize)

Definition at line 184 of file mm.h.

◆ MI_NONPAGED_BITMAP_END

#define MI_NONPAGED_BITMAP_END   (MI_NONPAGED_BITMAP_BASE + MI_NONPAGED_BITMAP_PAGES_NEEDED * VirtualPageSize)

Definition at line 185 of file mm.h.

◆ MI_NONPAGED_BITMAP_PAGES_NEEDED

#define MI_NONPAGED_BITMAP_PAGES_NEEDED   ((NONPAGED_POOL_VA_BITMAP_QWORDS * sizeof(uint64_t) + VirtualPageSize - 1) / VirtualPageSize)

Definition at line 177 of file mm.h.

◆ MI_NONPAGED_POOL_BASE

#define MI_NONPAGED_POOL_BASE   ALIGN_UP(MI_NONPAGED_BITMAP_END, VirtualPageSize)

Definition at line 191 of file mm.h.

◆ MI_NONPAGED_POOL_END

#define MI_NONPAGED_POOL_END   (MI_NONPAGED_POOL_BASE + MI_NONPAGED_POOL_SIZE)

Definition at line 192 of file mm.h.

◆ MI_NONPAGED_POOL_SIZE

#define MI_NONPAGED_POOL_SIZE   ((size_t)16ULL * 1024 * 1024 * 1024)

Definition at line 165 of file mm.h.

◆ MI_PAGED_BITMAP_BASE

#define MI_PAGED_BITMAP_BASE   ALIGN_UP(MI_NONPAGED_BITMAP_END, VirtualPageSize)

Definition at line 187 of file mm.h.

◆ MI_PAGED_BITMAP_END

#define MI_PAGED_BITMAP_END   (MI_PAGED_BITMAP_BASE + MI_PAGED_BITMAP_PAGES_NEEDED * VirtualPageSize)

Definition at line 188 of file mm.h.

◆ MI_PAGED_BITMAP_PAGES_NEEDED

#define MI_PAGED_BITMAP_PAGES_NEEDED   ((PAGED_POOL_VA_BITMAP_QWORDS * sizeof(uint64_t) + VirtualPageSize - 1) / VirtualPageSize)

Definition at line 178 of file mm.h.

◆ MI_PAGED_POOL_BASE

#define MI_PAGED_POOL_BASE   ALIGN_UP(MI_NONPAGED_POOL_END, VirtualPageSize)

Definition at line 194 of file mm.h.

◆ MI_PAGED_POOL_END

#define MI_PAGED_POOL_END   (MI_PAGED_POOL_BASE + MI_PAGED_POOL_SIZE)

Definition at line 195 of file mm.h.

◆ MI_PAGED_POOL_SIZE

#define MI_PAGED_POOL_SIZE   ((size_t)32ULL * 1024 * 1024 * 1024)

Definition at line 166 of file mm.h.

◆ MI_STACK_SIZE

#define MI_STACK_SIZE   0x4000

Definition at line 219 of file mm.h.

◆ MI_WRITE_PTE

#define MI_WRITE_PTE ( _PtePointer,
_Va,
_Pa,
_Flags )
Value:
do { \
MMPTE* _pte = (MMPTE*)(_PtePointer); \
uint64_t _val = (((uintptr_t)(_Pa)) & ~0xFFFULL) | (uint64_t)(_Flags); \
MiAtomicExchangePte(_pte, _val); \
__asm__ volatile("" ::: "memory"); \
\
/* Only set PFN->PTE link if PFN database is initialized */ \
_pfn->State = PfnStateActive; \
} \
\
invlpg((void*)(uintptr_t)(_Va)); \
\
/* Send IPIs if SMP is initialized */ \
IPI_PARAMS _Params; \
_Params.pageParams.addressToInvalidate = (uint64_t)(_Va); \
MhSendActionToCpusAndWait(CPU_ACTION_PERFORM_TLB_SHOOTDOWN, _Params);\
} \
} while (0)
bool smpInitialized
Definition kernel.c:146
@ CPU_ACTION_PERFORM_TLB_SHOOTDOWN
Definition mh.h:94
struct _IPI_PARAMS IPI_PARAMS
bool allApsInitialized
Definition kernel.c:20
@ PfnStateActive
Definition mm.h:239
#define PHYSICAL_TO_PPFN(PHYS)
Definition mm.h:64
struct _MMPTE * PMMPTE
struct _PFN_ENTRY * PPFN_ENTRY
struct _MMPTE MMPTE
@ PFN_FLAG_NONPAGED
Definition mm.h:251
bool MmPfnDatabaseInitialized
Definition pfn.c:30
struct _PAGE_PARAMETERS pageParams
Definition mh.h:399
uint64_t addressToInvalidate
Definition mh.h:394
union _PFN_ENTRY::@217024126340164016372152071216274230164113211246 Descriptor
struct _PFN_ENTRY::@217024126340164016372152071216274230164113211246::@301110335271023021153236134322146064331241142124 Mapping
uint8_t State
Definition mm.h:430
PMMPTE PteAddress
Definition mm.h:441
uint8_t Flags
Definition mm.h:431

Definition at line 90 of file mm.h.

◆ MM_IS_DEMAND_ZERO_PTE

#define MM_IS_DEMAND_ZERO_PTE ( pte)
Value:
(((pte).Soft.SoftwareFlags & MI_DEMAND_ZERO_BIT) != 0)
#define MI_DEMAND_ZERO_BIT
Definition mm.h:213

Definition at line 120 of file mm.h.

◆ MM_POOL_CANARY

#define MM_POOL_CANARY   'BEKA'

Definition at line 216 of file mm.h.

◆ MM_SET_DEMAND_ZERO_PTE

#define MM_SET_DEMAND_ZERO_PTE ( pte,
prot_flags,
nx )
Value:
do { \
(pte).Value = 0; \
(pte).Soft.SoftwareFlags = (prot_flags) | MI_DEMAND_ZERO_BIT; \
(pte).Soft.NoExecute = (nx); \
} while(0)

Definition at line 122 of file mm.h.

◆ MM_UNSET_DEMAND_ZERO_PTE

#define MM_UNSET_DEMAND_ZERO_PTE ( pte)
Value:
do { \
(pte).Soft.SoftwareFlags &= ~MI_DEMAND_ZERO_BIT; \
} while(0)

Definition at line 128 of file mm.h.

◆ MmBarrier

#define MmBarrier ( )
Value:
__asm__ __volatile__("mfence" ::: "memory")

Definition at line 229 of file mm.h.

◆ MmFullBarrier

#define MmFullBarrier ( )
Value:
__sync_synchronize()

Definition at line 226 of file mm.h.

◆ MmIsAddressValid

#define MmIsAddressValid ( VirtualAddress)
Value:
MmIsAddressPresent(VirtualAddress)
bool MmIsAddressPresent(IN uintptr_t VirtualAddress)
Definition map.c:488

Definition at line 234 of file mm.h.

◆ NONPAGED_POOL_VA_BITMAP_QWORDS

#define NONPAGED_POOL_VA_BITMAP_QWORDS   ((NONPAGED_POOL_VA_TOTAL_PAGES + 63) / 64)

Definition at line 173 of file mm.h.

◆ NONPAGED_POOL_VA_TOTAL_PAGES

#define NONPAGED_POOL_VA_TOTAL_PAGES   (MI_NONPAGED_POOL_SIZE / VirtualPageSize)

Definition at line 169 of file mm.h.

◆ PAGE_PAT

#define PAGE_PAT   (1ULL << 7)

Definition at line 301 of file mm.h.

◆ PAGED_POOL_VA_BITMAP_QWORDS

#define PAGED_POOL_VA_BITMAP_QWORDS   ((PAGED_POOL_VA_TOTAL_PAGES + 63) / 64)

Definition at line 174 of file mm.h.

◆ PAGED_POOL_VA_TOTAL_PAGES

#define PAGED_POOL_VA_TOTAL_PAGES   (MI_PAGED_POOL_SIZE / VirtualPageSize)

Definition at line 170 of file mm.h.

◆ PAGES_TO_BYTES

#define PAGES_TO_BYTES ( Pages)
Value:
((Pages) * VirtualPageSize)

Definition at line 149 of file mm.h.

◆ PFN_ERROR

#define PFN_ERROR   UINT64_T_MAX

Definition at line 208 of file mm.h.

◆ PFN_TO_PHYS

#define PFN_TO_PHYS ( Pfn)
Value:
#define PPFN_TO_PHYSICAL_ADDRESS(PPFN)
Definition mm.h:117
#define INDEX_TO_PPFN(Index)
Definition mm.h:62

Definition at line 205 of file mm.h.

◆ PHYS_TO_INDEX

#define PHYS_TO_INDEX ( PhysicalAddress)
Value:
#define PPFN_TO_INDEX(PPFN)
Definition mm.h:116

Definition at line 206 of file mm.h.

◆ PHYSICAL_TO_PPFN

#define PHYSICAL_TO_PPFN ( PHYS)
Value:
(&PfnDatabase.PfnEntries[(size_t)((PHYS) / (uint64_t)PhysicalFrameSize)])
#define PhysicalFrameSize
Definition mm.h:54

Definition at line 64 of file mm.h.

◆ PhysicalFrameSize

#define PhysicalFrameSize   4096ULL

Definition at line 54 of file mm.h.

◆ PhysicalMemoryOffset

#define PhysicalMemoryOffset   0xffff880000000000ULL

Definition at line 56 of file mm.h.

◆ PML4_INDEX_BITS

#define PML4_INDEX_BITS   9

Definition at line 38 of file mm.h.

◆ PML4_INDEX_FROM_PHYS

#define PML4_INDEX_FROM_PHYS ( PHYS)
Value:
PML4_INDEX_FROM_VA( (uintptr_t)(PHYS) + (uintptr_t)PhysicalMemoryOffset )
#define PhysicalMemoryOffset
Definition mm.h:56
#define PML4_INDEX_FROM_VA(VA)
Definition mm.h:42

Definition at line 46 of file mm.h.

◆ PML4_INDEX_FROM_VA

#define PML4_INDEX_FROM_VA ( VA)
Value:
( ( (uintptr_t)(VA) >> PML4_INDEX_SHIFT ) & PML4_INDEX_MASK )
#define PML4_INDEX_SHIFT
Definition mm.h:39
#define PML4_INDEX_MASK
Definition mm.h:40

Definition at line 42 of file mm.h.

◆ PML4_INDEX_MASK

#define PML4_INDEX_MASK   ((1ULL << PML4_INDEX_BITS) - 1ULL)

Definition at line 40 of file mm.h.

◆ PML4_INDEX_SHIFT

#define PML4_INDEX_SHIFT   39

Definition at line 39 of file mm.h.

◆ POOL_MAX_ALLOC

#define POOL_MAX_ALLOC   2048

Definition at line 163 of file mm.h.

◆ POOL_MIN_ALLOC

#define POOL_MIN_ALLOC   32

Definition at line 159 of file mm.h.

◆ PPFN_TO_INDEX

#define PPFN_TO_INDEX ( PPFN)
Value:
((size_t)((PPFN) - PfnDatabase.PfnEntries))

Definition at line 116 of file mm.h.

◆ PPFN_TO_PHYSICAL_ADDRESS

#define PPFN_TO_PHYSICAL_ADDRESS ( PPFN)
Value:
((uint64_t)((uint64_t)PPFN_TO_INDEX(PPFN) * (uint64_t)PhysicalFrameSize))

Definition at line 117 of file mm.h.

◆ PROT_KERNEL_READ

#define PROT_KERNEL_READ   0x1

Definition at line 211 of file mm.h.

◆ PROT_KERNEL_WRITE

#define PROT_KERNEL_WRITE   0x2

Definition at line 212 of file mm.h.

◆ PTE_TO_PHYSICAL

#define PTE_TO_PHYSICAL ( PMMPTE)
Value:
((PMMPTE)->Value & ~0xFFFULL)

Definition at line 66 of file mm.h.

◆ RECURSIVE_INDEX

#define RECURSIVE_INDEX   0x1FF

Definition at line 57 of file mm.h.

◆ USER_VA_END

#define USER_VA_END   0x00007FFFFFFFFFFF

Definition at line 160 of file mm.h.

◆ USER_VA_START

#define USER_VA_START   0x10000

Definition at line 161 of file mm.h.

◆ VA_OFFSET

#define VA_OFFSET ( _VirtualAddress)
Value:
((uintptr_t)(_VirtualAddress) & 0xFFF)

Definition at line 119 of file mm.h.

◆ VirtualPageSize

#define VirtualPageSize   4096ULL

Definition at line 53 of file mm.h.

Typedef Documentation

◆ FAULT_OPERATION

◆ MEMORY_CACHING_TYPE

◆ MM_PFN_DATABASE

◆ MM_PFN_LIST

typedef struct _MM_PFN_LIST MM_PFN_LIST

◆ MMPTE

typedef struct _MMPTE MMPTE

◆ MMVAD

typedef struct _MMVAD MMVAD

◆ PAGE_FLAGS

typedef enum _PAGE_FLAGS PAGE_FLAGS

◆ PAGE_INDEX

typedef uint64_t PAGE_INDEX

Definition at line 232 of file mm.h.

◆ PFAULT_OPERATION

◆ PFN_ENTRY

typedef struct _PFN_ENTRY PFN_ENTRY

◆ PFN_FLAGS

typedef enum _PFN_FLAGS PFN_FLAGS

◆ PFN_STATE

typedef enum _PFN_STATE PFN_STATE

◆ PMMPTE

typedef struct _MMPTE * PMMPTE

◆ PMMVAD

typedef struct _MMVAD * PMMVAD

◆ POOL_DESCRIPTOR

◆ POOL_HEADER

typedef struct _POOL_HEADER POOL_HEADER

◆ POOL_TYPE

typedef enum _POOL_TYPE POOL_TYPE

◆ PPFN_ENTRY

typedef struct _PFN_ENTRY * PPFN_ENTRY

◆ PPOOL_DESCRIPTOR

◆ PPOOL_HEADER

typedef struct _POOL_HEADER * PPOOL_HEADER

◆ PPRIVILEGE_MODE

◆ PRIVILEGE_MODE

◆ SYSTEM_PHASE_ROUTINE

◆ VAD_FLAGS

typedef enum _VAD_FLAGS VAD_FLAGS

Enumeration Type Documentation

◆ _FAULT_OPERATION

Enumerator
FaultOpInvalid 
ReadOperation 
WriteOperation 
ExecuteOperation 

Definition at line 325 of file mm.h.

◆ _MEMORY_CACHING_TYPE

Enumerator
MmNonCached 
MmCached 
MmWriteCombined 
MmWriteThrough 
MmNonCachedUnordered 
MmUSWCCached 
MmHardwareCoherentCached 

Definition at line 337 of file mm.h.

◆ _PAGE_FLAGS

Enumerator
PAGE_PRESENT 
PAGE_RW 
PAGE_USER 
PAGE_PWT 
PAGE_PCD 
PAGE_ACCESSED 
PAGE_DIRTY 
PAGE_PS 
PAGE_GLOBAL 
PAGE_NX 

Definition at line 267 of file mm.h.

◆ _PFN_FLAGS

enum _PFN_FLAGS
Enumerator
PFN_FLAG_NONE 
PFN_FLAG_NONPAGED 
PFN_FLAG_COPY_ON_WRITE 
PFN_FLAG_MAPPED_FILE 
PFN_FLAG_LOCKED_FOR_IO 

Definition at line 249 of file mm.h.

◆ _PFN_STATE

enum _PFN_STATE
Enumerator
PfnStateActive 
PfnStateStandby 
PfnStateModified 
PfnStateFree 
PfnStateZeroed 
PfnStateTransition 
PfnStateBad 

Definition at line 238 of file mm.h.

◆ _POOL_TYPE

enum _POOL_TYPE
Enumerator
NonPagedPool 
PagedPool 
NonPagedPoolCacheAligned 
PagedPoolCacheAligned 
NonPagedPoolNx 
PagedPoolNx 

Definition at line 315 of file mm.h.

◆ _PRIVILEGE_MODE

Enumerator
KernelMode 
UserMode 

Definition at line 332 of file mm.h.

◆ _SYSTEM_PHASE_ROUTINE

Enumerator
SYSTEM_PHASE_INITIALIZE_ALL 
SYSTEM_PHASE_INITIALIZE_PAT_ONLY 

Definition at line 374 of file mm.h.

◆ _VAD_FLAGS

enum _VAD_FLAGS
Enumerator
VAD_FLAG_NONE 
VAD_FLAG_READ 
VAD_FLAG_WRITE 
VAD_FLAG_EXECUTE 
VAD_FLAG_PRIVATE 
VAD_FLAG_MAPPED_FILE 
VAD_FLAG_COPY_ON_WRITE 

Definition at line 257 of file mm.h.

Function Documentation

◆ kmemcmp()

FORCEINLINE int kmemcmp ( const void * s1,
const void * s2,
size_t n )

Definition at line 566 of file mm.h.

◆ kmemcpy()

FORCEINLINE void * kmemcpy ( void * dest,
const void * src,
size_t len )

Definition at line 554 of file mm.h.

◆ kmemset()

FORCEINLINE void * kmemset ( void * dest,
int64_t val,
uint64_t len )

Definition at line 540 of file mm.h.

◆ MiAllocatePoolVa()

uintptr_t MiAllocatePoolVa ( IN POOL_TYPE PoolType,
IN size_t NumberOfBytes )

Definition at line 213 of file va.c.

◆ MiAtomicExchangePte()

FORCEINLINE void MiAtomicExchangePte ( PMMPTE PtePtr,
uint64_t NewPteValue )

Definition at line 651 of file mm.h.

◆ MiCacheToFlags()

FORCEINLINE uint64_t MiCacheToFlags ( MEMORY_CACHING_TYPE type)

Definition at line 587 of file mm.h.

◆ MiCheckForContigiousMemory()

bool MiCheckForContigiousMemory ( IN void * StartAddress,
IN size_t NumberOfBytes )

Definition at line 25 of file mmio.c.

◆ MiCreateKernelStack()

void * MiCreateKernelStack ( IN bool LargeStack)

Definition at line 25 of file mmproc.c.

◆ MiFindVad()

PMMVAD MiFindVad ( IN PMMVAD Root,
IN uintptr_t VirtualAddress )

Definition at line 354 of file vad.c.

◆ MiFreeKernelStack()

void MiFreeKernelStack ( IN void * AllocatedStackTop,
IN bool LargeStack )

Definition at line 148 of file mmproc.c.

◆ MiFreePoolVaContiguous()

void MiFreePoolVaContiguous ( IN uintptr_t va,
IN size_t NumberOfBytes,
IN POOL_TYPE PoolType )

Definition at line 367 of file va.c.

◆ MiGetPdePointer()

PMMPTE MiGetPdePointer ( IN uintptr_t va)

Definition at line 219 of file map.c.

◆ MiGetPdptePointer()

PMMPTE MiGetPdptePointer ( IN uintptr_t va)

Definition at line 177 of file map.c.

◆ MiGetPml4ePointer()

PMMPTE MiGetPml4ePointer ( IN uintptr_t va)

Definition at line 147 of file map.c.

◆ MiGetPtePointer()

PMMPTE MiGetPtePointer ( IN uintptr_t va)

Definition at line 76 of file map.c.

◆ MiInitializePfnDatabase()

MTSTATUS MiInitializePfnDatabase ( IN PBOOT_INFO BootInfo)

Definition at line 100 of file pfn.c.

◆ MiInitializePoolSystem()

MTSTATUS MiInitializePoolSystem ( void )

Definition at line 36 of file pool.c.

◆ MiInitializePoolVaSpace()

bool MiInitializePoolVaSpace ( void )

Definition at line 35 of file va.c.

◆ MiInvalidateTlbForVa()

void MiInvalidateTlbForVa ( IN void * VirtualAddress)

Definition at line 273 of file map.c.

◆ MiIsValidPfn()

FORCEINLINE bool MiIsValidPfn ( IN PAGE_INDEX Pfn)

Definition at line 667 of file mm.h.

◆ MiMapPageInHyperspace()

void * MiMapPageInHyperspace ( IN uint64_t PfnIndex,
OUT PIRQL OldIrql )

Definition at line 33 of file hypermap.c.

◆ MiMoveUefiDataToHigherHalf()

void MiMoveUefiDataToHigherHalf ( IN PBOOT_INFO BootInfo)

Definition at line 154 of file mminit.c.

◆ MiReleasePhysicalPage()

void MiReleasePhysicalPage ( IN PAGE_INDEX PfnIndex)

Definition at line 421 of file pfn.c.

◆ MiReloadTLBs()

void MiReloadTLBs ( void )

Definition at line 443 of file map.c.

◆ MiRequestPhysicalPage()

PAGE_INDEX MiRequestPhysicalPage ( IN PFN_STATE ListType)

Definition at line 325 of file pfn.c.

◆ MiRetrieveLastFaultyAddress()

FORCEINLINE uint64_t MiRetrieveLastFaultyAddress ( void )

Definition at line 641 of file mm.h.

◆ MiRetrieveOperationFromErrorCode()

FORCEINLINE FAULT_OPERATION MiRetrieveOperationFromErrorCode ( uint64_t ErrorCode)

Definition at line 619 of file mm.h.

◆ MiTranslatePteToPfn()

PAGE_INDEX MiTranslatePteToPfn ( IN PMMPTE pte)

Definition at line 310 of file map.c.

◆ MiTranslatePteToVa()

uint64_t MiTranslatePteToVa ( IN PMMPTE pte)

Definition at line 337 of file map.c.

◆ MiTranslateVirtualToPhysical()

uintptr_t MiTranslateVirtualToPhysical ( IN void * VirtualAddress)

Definition at line 456 of file map.c.

◆ MiUnlinkPageFromList()

void MiUnlinkPageFromList ( PPFN_ENTRY pfn)

Definition at line 486 of file pfn.c.

◆ MiUnmapHyperSpaceMap()

void MiUnmapHyperSpaceMap ( IN IRQL OldIrql)

Definition at line 83 of file hypermap.c.

◆ MiUnmapPte()

void MiUnmapPte ( IN PMMPTE pte)

Definition at line 385 of file map.c.

◆ MmAccessFault()

MTSTATUS MmAccessFault ( IN uint64_t FaultBits,
IN uint64_t VirtualAddress,
IN PRIVILEGE_MODE PreviousMode,
IN PTRAP_FRAME TrapFrame )

Definition at line 27 of file fault.c.

◆ MmAllocateContigiousMemory()

void * MmAllocateContigiousMemory ( IN size_t NumberOfBytes,
IN uint64_t HighestAcceptableAddress )

Definition at line 87 of file mmio.c.

◆ MmAllocatePoolWithTag()

void * MmAllocatePoolWithTag ( IN enum _POOL_TYPE PoolType,
IN size_t NumberOfBytes,
IN uint32_t Tag )

Definition at line 427 of file pool.c.

◆ MmAllocateVirtualMemory()

MTSTATUS MmAllocateVirtualMemory ( IN PEPROCESS Process,
_In_Opt _Out_Opt void ** BaseAddress,
IN size_t NumberOfBytes,
IN VAD_FLAGS VadFlags )

Definition at line 723 of file vad.c.

◆ MmCreateProcessAddressSpace()

MTSTATUS MmCreateProcessAddressSpace ( OUT void ** DirectoryTable)

Definition at line 220 of file mmproc.c.

◆ MmFindFreeAddressSpace()

uintptr_t MmFindFreeAddressSpace ( IN PEPROCESS Process,
IN size_t NumberOfBytes,
IN uintptr_t SearchStart,
IN uintptr_t SearchEnd )

Definition at line 708 of file vad.c.

◆ MmFreeContigiousMemory()

void MmFreeContigiousMemory ( IN void * BaseAddress,
IN size_t NumberOfBytes )

Definition at line 213 of file mmio.c.

◆ MmFreePool()

void MmFreePool ( IN void * buf)

Definition at line 586 of file pool.c.

◆ MmFreeVirtualMemory()

MTSTATUS MmFreeVirtualMemory ( IN PEPROCESS Process,
IN void * BaseAddress )

Definition at line 817 of file vad.c.

◆ MmInitSystem()

bool MmInitSystem ( IN uint8_t Phase,
IN PBOOT_INFO BootInformation )

Definition at line 55 of file mminit.c.

◆ MmInvalidAccessAllowed()

bool MmInvalidAccessAllowed ( void )

Definition at line 292 of file fault.c.

◆ MmIsAddressPresent()

bool MmIsAddressPresent ( IN uintptr_t VirtualAddress)

Definition at line 488 of file map.c.

◆ MmMapIoSpace()

void * MmMapIoSpace ( IN uintptr_t PhysicalAddress,
IN size_t NumberOfBytes,
IN MEMORY_CACHING_TYPE CacheType )

Definition at line 269 of file mmio.c.

◆ pml4_from_recursive()

uint64_t * pml4_from_recursive ( void )

Definition at line 31 of file map.c.

Variable Documentation

◆ MmHighestPfn

PAGE_INDEX MmHighestPfn
extern

Definition at line 31 of file pfn.c.

◆ MmHighestUserAddress

uintptr_t MmHighestUserAddress
extern

Definition at line 27 of file process.c.

◆ MmNonPagedPoolEnd

uintptr_t MmNonPagedPoolEnd
extern

Definition at line 31 of file pool.c.

◆ MmNonPagedPoolStart

uintptr_t MmNonPagedPoolStart
extern

Definition at line 30 of file pool.c.

◆ MmPagedPoolEnd

uintptr_t MmPagedPoolEnd
extern

Definition at line 33 of file pool.c.

◆ MmPagedPoolStart

uintptr_t MmPagedPoolStart
extern

Definition at line 32 of file pool.c.

◆ MmPfnDatabaseInitialized

bool MmPfnDatabaseInitialized
extern

Definition at line 30 of file pfn.c.

◆ MmSystemRangeStart

uintptr_t MmSystemRangeStart
extern

Definition at line 26 of file process.c.

◆ MmUserProbeAddress

uintptr_t MmUserProbeAddress
extern

Definition at line 28 of file process.c.

◆ PfnDatabase

MM_PFN_DATABASE PfnDatabase
extern

Definition at line 29 of file pfn.c.