Source file src/runtime/mem.go
1 // Copyright 2022 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 // OS memory management abstraction layer 10 // 11 // Regions of the address space managed by the runtime may be in one of four 12 // states at any given time: 13 // 1) None - Unreserved and unmapped, the default state of any region. 14 // 2) Reserved - Owned by the runtime, but accessing it would cause a fault. 15 // Does not count against the process' memory footprint. 16 // 3) Prepared - Reserved, intended not to be backed by physical memory (though 17 // an OS may implement this lazily). Can transition efficiently to 18 // Ready. Accessing memory in such a region is undefined (may 19 // fault, may give back unexpected zeroes, etc.). 20 // 4) Ready - may be accessed safely. 21 // 22 // This set of states is more than strictly necessary to support all the 23 // currently supported platforms. One could get by with just None, Reserved, and 24 // Ready. However, the Prepared state gives us flexibility for performance 25 // purposes. For example, on POSIX-y operating systems, Reserved is usually a 26 // private anonymous mmap'd region with PROT_NONE set, and to transition 27 // to Ready would require setting PROT_READ|PROT_WRITE. However the 28 // underspecification of Prepared lets us use just MADV_FREE to transition from 29 // Ready to Prepared. Thus with the Prepared state we can set the permission 30 // bits just once early on, we can efficiently tell the OS that it's free to 31 // take pages away from us when we don't strictly need them. 32 // 33 // This file defines a cross-OS interface for a common set of helpers 34 // that transition memory regions between these states. The helpers call into 35 // OS-specific implementations that handle errors, while the interface boundary 36 // implements cross-OS functionality, like updating runtime accounting. 37 38 // sysAlloc transitions an OS-chosen region of memory from None to Ready. 39 // More specifically, it obtains a large chunk of zeroed memory from the 40 // operating system, typically on the order of a hundred kilobytes 41 // or a megabyte. This memory is always immediately available for use. 42 // 43 // sysStat must be non-nil. 44 // 45 // Don't split the stack as this function may be invoked without a valid G, 46 // which prevents us from allocating more stack. 47 // 48 //go:nosplit 49 func sysAlloc(n uintptr, sysStat *sysMemStat, vmaName string) unsafe.Pointer { 50 sysStat.add(int64(n)) 51 gcController.mappedReady.Add(int64(n)) 52 p := sysAllocOS(n, vmaName) 53 54 // When using ASAN leak detection, we must tell ASAN about 55 // cases where we store pointers in mmapped memory. 56 if asanenabled { 57 lsanregisterrootregion(p, n) 58 } 59 60 return p 61 } 62 63 // sysUnused transitions a memory region from Ready to Prepared. It notifies the 64 // operating system that the physical pages backing this memory region are no 65 // longer needed and can be reused for other purposes. The contents of a 66 // sysUnused memory region are considered forfeit and the region must not be 67 // accessed again until sysUsed is called. 68 func sysUnused(v unsafe.Pointer, n uintptr) { 69 gcController.mappedReady.Add(-int64(n)) 70 sysUnusedOS(v, n) 71 } 72 73 // needZeroAfterSysUnused reports whether memory returned by sysUnused must be 74 // zeroed for use. 75 func needZeroAfterSysUnused() bool { 76 return needZeroAfterSysUnusedOS() 77 } 78 79 // sysUsed transitions a memory region from Prepared to Ready. It notifies the 80 // operating system that the memory region is needed and ensures that the region 81 // may be safely accessed. This is typically a no-op on systems that don't have 82 // an explicit commit step and hard over-commit limits, but is critical on 83 // Windows, for example. 84 // 85 // This operation is idempotent for memory already in the Prepared state, so 86 // it is safe to refer, with v and n, to a range of memory that includes both 87 // Prepared and Ready memory. However, the caller must provide the exact amount 88 // of Prepared memory for accounting purposes. 89 func sysUsed(v unsafe.Pointer, n, prepared uintptr) { 90 gcController.mappedReady.Add(int64(prepared)) 91 sysUsedOS(v, n) 92 } 93 94 // sysHugePage does not transition memory regions, but instead provides a 95 // hint to the OS that it would be more efficient to back this memory region 96 // with pages of a larger size transparently. 97 func sysHugePage(v unsafe.Pointer, n uintptr) { 98 sysHugePageOS(v, n) 99 } 100 101 // sysNoHugePage does not transition memory regions, but instead provides a 102 // hint to the OS that it would be less efficient to back this memory region 103 // with pages of a larger size transparently. 104 func sysNoHugePage(v unsafe.Pointer, n uintptr) { 105 sysNoHugePageOS(v, n) 106 } 107 108 // sysHugePageCollapse attempts to immediately back the provided memory region 109 // with huge pages. It is best-effort and may fail silently. 110 func sysHugePageCollapse(v unsafe.Pointer, n uintptr) { 111 sysHugePageCollapseOS(v, n) 112 } 113 114 // sysFree transitions a memory region from Ready to None. Therefore, it 115 // returns memory unconditionally. 116 // 117 // sysStat must be non-nil. 118 // 119 // The size and start address must exactly match the size and returned address 120 // from the original sysAlloc/sysReserve/sysReserveAligned call. That is, 121 // sysFree cannot be used to free a subset of a memory region. 122 // 123 // Don't split the stack as this function may be invoked without a valid G, 124 // which prevents us from allocating more stack. 125 // 126 //go:nosplit 127 func sysFree(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { 128 // When using ASAN leak detection, the memory being freed is known by 129 // the sanitizer. We need to unregister it so it's not accessed by it. 130 // 131 // lsanunregisterrootregion matches regions by start address and size, 132 // so it is not possible to unregister a subset of the region. This is 133 // why sysFree requires the full region from the initial allocation. 134 if asanenabled { 135 lsanunregisterrootregion(v, n) 136 } 137 138 sysStat.add(-int64(n)) 139 gcController.mappedReady.Add(-int64(n)) 140 sysFreeOS(v, n) 141 } 142 143 // sysFault transitions a memory region from Ready to Reserved. It 144 // marks a region such that it will always fault if accessed. Used only for 145 // debugging the runtime. 146 // 147 // TODO(mknyszek): Currently it's true that all uses of sysFault transition 148 // memory from Ready to Reserved, but this may not be true in the future 149 // since on every platform the operation is much more general than that. 150 // If a transition from Prepared is ever introduced, create a new function 151 // that elides the Ready state accounting. 152 func sysFault(v unsafe.Pointer, n uintptr) { 153 gcController.mappedReady.Add(-int64(n)) 154 sysFaultOS(v, n) 155 } 156 157 // sysReserve transitions a memory region from None to Reserved. It reserves 158 // address space in such a way that it would cause a fatal fault upon access 159 // (either via permissions or not committing the memory). Such a reservation is 160 // thus never backed by physical memory. 161 // 162 // If the pointer passed to it is non-nil, the caller wants the reservation 163 // there, but sysReserve can still choose another location if that one is 164 // unavailable. 165 // 166 // sysReserve returns OS-aligned memory. If a larger alignment is required, use 167 // sysReservedAligned. 168 func sysReserve(v unsafe.Pointer, n uintptr, vmaName string) unsafe.Pointer { 169 p := sysReserveOS(v, n, vmaName) 170 171 // When using ASAN leak detection, we must tell ASAN about 172 // cases where we store pointers in mmapped memory. 173 if asanenabled { 174 lsanregisterrootregion(p, n) 175 } 176 177 return p 178 } 179 180 // sysReserveAligned transitions a memory region from None to Reserved. 181 // 182 // Semantics are equivlent to sysReserve, but the returned pointer is aligned 183 // to align bytes. It may reserve either n or n+align bytes, so it returns the 184 // size that was reserved. 185 func sysReserveAligned(v unsafe.Pointer, size, align uintptr, vmaName string) (unsafe.Pointer, uintptr) { 186 if isSbrkPlatform { 187 if v != nil { 188 throw("unexpected heap arena hint on sbrk platform") 189 } 190 return sysReserveAlignedSbrk(size, align) 191 } 192 // Since the alignment is rather large in uses of this 193 // function, we're not likely to get it by chance, so we ask 194 // for a larger region and remove the parts we don't need. 195 retries := 0 196 retry: 197 p := uintptr(sysReserve(v, size+align, vmaName)) 198 switch { 199 case p == 0: 200 return nil, 0 201 case p&(align-1) == 0: 202 return unsafe.Pointer(p), size + align 203 case GOOS == "windows": 204 // On Windows we can't release pieces of a 205 // reservation, so we release the whole thing and 206 // re-reserve the aligned sub-region. This may race, 207 // so we may have to try again. 208 sysUnreserve(unsafe.Pointer(p), size+align) 209 p = alignUp(p, align) 210 p2 := sysReserve(unsafe.Pointer(p), size, vmaName) 211 if p != uintptr(p2) { 212 // Must have raced. Try again. 213 sysUnreserve(p2, size) 214 if retries++; retries == 100 { 215 throw("failed to allocate aligned heap memory; too many retries") 216 } 217 goto retry 218 } 219 // Success. 220 return p2, size 221 default: 222 // Trim off the unaligned parts. 223 pAligned := alignUp(p, align) 224 end := pAligned + size 225 endLen := (p + size + align) - end 226 227 // sysUnreserve does not allow unreserving a subset of the 228 // region because LSAN does not allow unregistering a subset. 229 // So we can't call sysUnreserve. Instead we simply unregister 230 // the entire region from LSAN and re-register with the smaller 231 // region before freeing the unecessary portions, which does 232 // allow subsets of the region. 233 if asanenabled { 234 lsanunregisterrootregion(unsafe.Pointer(p), size+align) 235 lsanregisterrootregion(unsafe.Pointer(pAligned), size) 236 } 237 sysFreeOS(unsafe.Pointer(p), pAligned-p) 238 if endLen > 0 { 239 sysFreeOS(unsafe.Pointer(end), endLen) 240 } 241 return unsafe.Pointer(pAligned), size 242 } 243 } 244 245 // sysUnreserve transitions a memory region from Reserved to None. 246 // 247 // The size and start address must exactly match the size and returned address 248 // from sysReserve/sysReserveAligned. That is, sysUnreserve cannot be used to 249 // unreserve a subset of a memory region. 250 // 251 // Don't split the stack as this function may be invoked without a valid G, 252 // which prevents us from allocating more stack. 253 // 254 //go:nosplit 255 func sysUnreserve(v unsafe.Pointer, n uintptr) { 256 // When using ASAN leak detection, the memory being freed is known by 257 // the sanitizer. We need to unregister it so it's not accessed by it. 258 // 259 // lsanunregisterrootregion matches regions by start address and size, 260 // so it is not possible to unregister a subset of the region. This is 261 // why sysUnreserve requires the full region from sysReserve. 262 if asanenabled { 263 lsanunregisterrootregion(v, n) 264 } 265 266 sysFreeOS(v, n) 267 } 268 269 // sysMap transitions a memory region from Reserved to Prepared. It ensures the 270 // memory region can be efficiently transitioned to Ready. 271 // 272 // sysStat must be non-nil. 273 func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat, vmaName string) { 274 sysStat.add(int64(n)) 275 sysMapOS(v, n, vmaName) 276 } 277