Source file src/runtime/os_linux.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/goarch"
    10  	"internal/runtime/atomic"
    11  	"internal/runtime/syscall/linux"
    12  	"internal/strconv"
    13  	"unsafe"
    14  )
    15  
    16  // sigPerThreadSyscall is the same signal (SIGSETXID) used by glibc for
    17  // per-thread syscalls on Linux. We use it for the same purpose in non-cgo
    18  // binaries.
    19  const sigPerThreadSyscall = _SIGRTMIN + 1
    20  
    21  type mOS struct {
    22  	// profileTimer holds the ID of the POSIX interval timer for profiling CPU
    23  	// usage on this thread.
    24  	//
    25  	// It is valid when the profileTimerValid field is true. A thread
    26  	// creates and manages its own timer, and these fields are read and written
    27  	// only by this thread. But because some of the reads on profileTimerValid
    28  	// are in signal handling code, this field should be atomic type.
    29  	profileTimer      int32
    30  	profileTimerValid atomic.Bool
    31  
    32  	// needPerThreadSyscall indicates that a per-thread syscall is required
    33  	// for doAllThreadsSyscall.
    34  	needPerThreadSyscall atomic.Uint8
    35  
    36  	// This is a pointer to a chunk of memory allocated with a special
    37  	// mmap invocation in vgetrandomGetState().
    38  	vgetrandomState uintptr
    39  
    40  	waitsema uint32 // semaphore for parking on locks
    41  }
    42  
    43  // Linux futex.
    44  //
    45  //	futexsleep(uint32 *addr, uint32 val)
    46  //	futexwakeup(uint32 *addr)
    47  //
    48  // Futexsleep atomically checks if *addr == val and if so, sleeps on addr.
    49  // Futexwakeup wakes up threads sleeping on addr.
    50  // Futexsleep is allowed to wake up spuriously.
    51  
    52  const (
    53  	_FUTEX_PRIVATE_FLAG = 128
    54  	_FUTEX_WAIT_PRIVATE = 0 | _FUTEX_PRIVATE_FLAG
    55  	_FUTEX_WAKE_PRIVATE = 1 | _FUTEX_PRIVATE_FLAG
    56  )
    57  
    58  // Atomically,
    59  //
    60  //	if(*addr == val) sleep
    61  //
    62  // Might be woken up spuriously; that's allowed.
    63  // Don't sleep longer than ns; ns < 0 means forever.
    64  //
    65  //go:nosplit
    66  func futexsleep(addr *uint32, val uint32, ns int64) {
    67  	// Some Linux kernels have a bug where futex of
    68  	// FUTEX_WAIT returns an internal error code
    69  	// as an errno. Libpthread ignores the return value
    70  	// here, and so can we: as it says a few lines up,
    71  	// spurious wakeups are allowed.
    72  	if ns < 0 {
    73  		futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, nil, nil, 0)
    74  		return
    75  	}
    76  
    77  	var ts timespec
    78  	ts.setNsec(ns)
    79  	futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, &ts, nil, 0)
    80  }
    81  
    82  // If any procs are sleeping on addr, wake up at most cnt.
    83  //
    84  //go:nosplit
    85  func futexwakeup(addr *uint32, cnt uint32) {
    86  	ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0)
    87  	if ret >= 0 {
    88  		return
    89  	}
    90  
    91  	// I don't know that futex wakeup can return
    92  	// EAGAIN or EINTR, but if it does, it would be
    93  	// safe to loop and call futex again.
    94  	systemstack(func() {
    95  		print("futexwakeup addr=", addr, " returned ", ret, "\n")
    96  	})
    97  
    98  	*(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006
    99  }
   100  
   101  func getCPUCount() int32 {
   102  	// This buffer is huge (8 kB) but we are on the system stack
   103  	// and there should be plenty of space (64 kB).
   104  	// Also this is a leaf, so we're not holding up the memory for long.
   105  	// See golang.org/issue/11823.
   106  	// The suggested behavior here is to keep trying with ever-larger
   107  	// buffers, but we don't have a dynamic memory allocator at the
   108  	// moment, so that's a bit tricky and seems like overkill.
   109  	const maxCPUs = 64 * 1024
   110  	var buf [maxCPUs / 8]byte
   111  	r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0])
   112  	if r < 0 {
   113  		return 1
   114  	}
   115  	n := int32(0)
   116  	for _, v := range buf[:r] {
   117  		for v != 0 {
   118  			n += int32(v & 1)
   119  			v >>= 1
   120  		}
   121  	}
   122  	if n == 0 {
   123  		n = 1
   124  	}
   125  	return n
   126  }
   127  
   128  // Clone, the Linux rfork.
   129  const (
   130  	_CLONE_VM             = 0x100
   131  	_CLONE_FS             = 0x200
   132  	_CLONE_FILES          = 0x400
   133  	_CLONE_SIGHAND        = 0x800
   134  	_CLONE_PTRACE         = 0x2000
   135  	_CLONE_VFORK          = 0x4000
   136  	_CLONE_PARENT         = 0x8000
   137  	_CLONE_THREAD         = 0x10000
   138  	_CLONE_NEWNS          = 0x20000
   139  	_CLONE_SYSVSEM        = 0x40000
   140  	_CLONE_SETTLS         = 0x80000
   141  	_CLONE_PARENT_SETTID  = 0x100000
   142  	_CLONE_CHILD_CLEARTID = 0x200000
   143  	_CLONE_UNTRACED       = 0x800000
   144  	_CLONE_CHILD_SETTID   = 0x1000000
   145  	_CLONE_STOPPED        = 0x2000000
   146  	_CLONE_NEWUTS         = 0x4000000
   147  	_CLONE_NEWIPC         = 0x8000000
   148  
   149  	// As of QEMU 2.8.0 (5ea2fc84d), user emulation requires all six of these
   150  	// flags to be set when creating a thread; attempts to share the other
   151  	// five but leave SYSVSEM unshared will fail with -EINVAL.
   152  	//
   153  	// In non-QEMU environments CLONE_SYSVSEM is inconsequential as we do not
   154  	// use System V semaphores.
   155  
   156  	cloneFlags = _CLONE_VM | /* share memory */
   157  		_CLONE_FS | /* share cwd, etc */
   158  		_CLONE_FILES | /* share fd table */
   159  		_CLONE_SIGHAND | /* share sig handler table */
   160  		_CLONE_SYSVSEM | /* share SysV semaphore undo lists (see issue #20763) */
   161  		_CLONE_THREAD /* revisit - okay for now */
   162  )
   163  
   164  //go:noescape
   165  func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32
   166  
   167  // May run with m.p==nil, so write barriers are not allowed.
   168  //
   169  //go:nowritebarrier
   170  func newosproc(mp *m) {
   171  	stk := unsafe.Pointer(mp.g0.stack.hi)
   172  	/*
   173  	 * note: strace gets confused if we use CLONE_PTRACE here.
   174  	 */
   175  	if false {
   176  		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", abi.FuncPCABI0(clone), " id=", mp.id, " ostk=", &mp, "\n")
   177  	}
   178  
   179  	// Disable signals during clone, so that the new thread starts
   180  	// with signals disabled. It will enable them in minit.
   181  	var oset sigset
   182  	sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
   183  	ret := retryOnEAGAIN(func() int32 {
   184  		r := clone(cloneFlags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(abi.FuncPCABI0(mstart)))
   185  		// clone returns positive TID, negative errno.
   186  		// We don't care about the TID.
   187  		if r >= 0 {
   188  			return 0
   189  		}
   190  		return -r
   191  	})
   192  	sigprocmask(_SIG_SETMASK, &oset, nil)
   193  
   194  	if ret != 0 {
   195  		print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", ret, ")\n")
   196  		if ret == _EAGAIN {
   197  			println("runtime: may need to increase max user processes (ulimit -u)")
   198  		}
   199  		throw("newosproc")
   200  	}
   201  }
   202  
   203  // Version of newosproc that doesn't require a valid G.
   204  //
   205  //go:nosplit
   206  func newosproc0(stacksize uintptr, fn unsafe.Pointer) {
   207  	stack := sysAlloc(stacksize, &memstats.stacks_sys, "OS thread stack")
   208  	if stack == nil {
   209  		writeErrStr(failallocatestack)
   210  		exit(1)
   211  	}
   212  	ret := clone(cloneFlags, unsafe.Pointer(uintptr(stack)+stacksize), nil, nil, fn)
   213  	if ret < 0 {
   214  		writeErrStr(failthreadcreate)
   215  		exit(1)
   216  	}
   217  }
   218  
   219  const (
   220  	_AT_NULL     = 0  // End of vector
   221  	_AT_PAGESZ   = 6  // System physical page size
   222  	_AT_PLATFORM = 15 // string identifying platform
   223  	_AT_HWCAP    = 16 // hardware capability bit vector
   224  	_AT_SECURE   = 23 // secure mode boolean
   225  	_AT_RANDOM   = 25 // introduced in 2.6.29
   226  	_AT_HWCAP2   = 26 // hardware capability bit vector 2
   227  )
   228  
   229  var procAuxv = []byte("/proc/self/auxv\x00")
   230  
   231  var addrspace_vec [1]byte
   232  
   233  func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32
   234  
   235  var auxvreadbuf [128]uintptr
   236  
   237  func sysargs(argc int32, argv **byte) {
   238  	n := argc + 1
   239  
   240  	// skip over argv, envp to get to auxv
   241  	for argv_index(argv, n) != nil {
   242  		n++
   243  	}
   244  
   245  	// skip NULL separator
   246  	n++
   247  
   248  	// now argv+n is auxv
   249  	auxvp := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*goarch.PtrSize))
   250  
   251  	if pairs := sysauxv(auxvp[:]); pairs != 0 {
   252  		auxv = auxvp[: pairs*2 : pairs*2]
   253  		return
   254  	}
   255  	// In some situations we don't get a loader-provided
   256  	// auxv, such as when loaded as a library on Android.
   257  	// Fall back to /proc/self/auxv.
   258  	fd := open(&procAuxv[0], 0 /* O_RDONLY */, 0)
   259  	if fd < 0 {
   260  		// On Android, /proc/self/auxv might be unreadable (issue 9229), so we fallback to
   261  		// try using mincore to detect the physical page size.
   262  		// mincore should return EINVAL when address is not a multiple of system page size.
   263  		const size = 256 << 10 // size of memory region to allocate
   264  		p, err := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
   265  		if err != 0 {
   266  			return
   267  		}
   268  		var n uintptr
   269  		for n = 4 << 10; n < size; n <<= 1 {
   270  			err := mincore(unsafe.Pointer(uintptr(p)+n), 1, &addrspace_vec[0])
   271  			if err == 0 {
   272  				physPageSize = n
   273  				break
   274  			}
   275  		}
   276  		if physPageSize == 0 {
   277  			physPageSize = size
   278  		}
   279  		munmap(p, size)
   280  		return
   281  	}
   282  
   283  	n = read(fd, noescape(unsafe.Pointer(&auxvreadbuf[0])), int32(unsafe.Sizeof(auxvreadbuf)))
   284  	closefd(fd)
   285  	if n < 0 {
   286  		return
   287  	}
   288  	// Make sure buf is terminated, even if we didn't read
   289  	// the whole file.
   290  	auxvreadbuf[len(auxvreadbuf)-2] = _AT_NULL
   291  	pairs := sysauxv(auxvreadbuf[:])
   292  	auxv = auxvreadbuf[: pairs*2 : pairs*2]
   293  }
   294  
   295  // secureMode holds the value of AT_SECURE passed in the auxiliary vector.
   296  var secureMode bool
   297  
   298  func sysauxv(auxv []uintptr) (pairs int) {
   299  	// Process the auxiliary vector entries provided by the kernel when the
   300  	// program is executed. See getauxval(3).
   301  	var i int
   302  	for ; auxv[i] != _AT_NULL; i += 2 {
   303  		tag, val := auxv[i], auxv[i+1]
   304  		switch tag {
   305  		case _AT_RANDOM:
   306  			// The kernel provides a pointer to 16 bytes of cryptographically
   307  			// random data. Note that in cgo programs this value may have
   308  			// already been used by libc at this point, and in particular glibc
   309  			// and musl use the value as-is for stack and pointer protector
   310  			// cookies from libc_start_main and/or dl_start. Also, cgo programs
   311  			// may use the value after we do.
   312  			startupRand = (*[16]byte)(unsafe.Pointer(val))[:]
   313  
   314  		case _AT_PAGESZ:
   315  			physPageSize = val
   316  
   317  		case _AT_SECURE:
   318  			secureMode = val == 1
   319  		}
   320  
   321  		archauxv(tag, val)
   322  		vdsoauxv(tag, val)
   323  	}
   324  	return i / 2
   325  }
   326  
   327  var sysTHPSizePath = []byte("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size\x00")
   328  
   329  func getHugePageSize() uintptr {
   330  	var numbuf [20]byte
   331  	fd := open(&sysTHPSizePath[0], 0 /* O_RDONLY */, 0)
   332  	if fd < 0 {
   333  		return 0
   334  	}
   335  	ptr := noescape(unsafe.Pointer(&numbuf[0]))
   336  	n := read(fd, ptr, int32(len(numbuf)))
   337  	closefd(fd)
   338  	if n <= 0 {
   339  		return 0
   340  	}
   341  	n-- // remove trailing newline
   342  	v, err := strconv.Atoi(slicebytetostringtmp((*byte)(ptr), int(n)))
   343  	if err != nil || v < 0 {
   344  		v = 0
   345  	}
   346  	if v&(v-1) != 0 {
   347  		// v is not a power of 2
   348  		return 0
   349  	}
   350  	return uintptr(v)
   351  }
   352  
   353  func osinit() {
   354  	numCPUStartup = getCPUCount()
   355  	physHugePageSize = getHugePageSize()
   356  	vgetrandomInit()
   357  	configure64bitsTimeOn32BitsArchitectures()
   358  }
   359  
   360  var urandom_dev = []byte("/dev/urandom\x00")
   361  
   362  func readRandom(r []byte) int {
   363  	// Note that all supported Linux kernels should provide AT_RANDOM which
   364  	// populates startupRand, so this fallback should be unreachable.
   365  	fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0)
   366  	n := read(fd, unsafe.Pointer(&r[0]), int32(len(r)))
   367  	closefd(fd)
   368  	return int(n)
   369  }
   370  
   371  func goenvs() {
   372  	goenvs_unix()
   373  }
   374  
   375  // Called to do synchronous initialization of Go code built with
   376  // -buildmode=c-archive or -buildmode=c-shared.
   377  // None of the Go runtime is initialized.
   378  //
   379  //go:nosplit
   380  //go:nowritebarrierrec
   381  func libpreinit() {
   382  	initsig(true)
   383  }
   384  
   385  // Called to initialize a new m (including the bootstrap m).
   386  // Called on the parent thread (main thread in case of bootstrap), can allocate memory.
   387  func mpreinit(mp *m) {
   388  	mp.gsignal = malg(32 * 1024) // Linux wants >= 2K
   389  	mp.gsignal.m = mp
   390  }
   391  
   392  func gettid() uint32
   393  
   394  // Called to initialize a new m (including the bootstrap m).
   395  // Called on the new thread, cannot allocate memory.
   396  func minit() {
   397  	minitSignals()
   398  
   399  	// Cgo-created threads and the bootstrap m are missing a
   400  	// procid. We need this for asynchronous preemption and it's
   401  	// useful in debuggers.
   402  	getg().m.procid = uint64(gettid())
   403  }
   404  
   405  // Called from dropm to undo the effect of an minit.
   406  //
   407  //go:nosplit
   408  func unminit() {
   409  	unminitSignals()
   410  	getg().m.procid = 0
   411  }
   412  
   413  // Called from mexit, but not from dropm, to undo the effect of thread-owned
   414  // resources in minit, semacreate, or elsewhere. Do not take locks after calling this.
   415  //
   416  // This always runs without a P, so //go:nowritebarrierrec is required.
   417  //
   418  //go:nowritebarrierrec
   419  func mdestroy(mp *m) {
   420  }
   421  
   422  // #ifdef GOARCH_386
   423  // #define sa_handler k_sa_handler
   424  // #endif
   425  
   426  func sigreturn__sigaction()
   427  func sigtramp() // Called via C ABI
   428  func cgoSigtramp()
   429  
   430  //go:noescape
   431  func sigaltstack(new, old *stackt)
   432  
   433  //go:noescape
   434  func setitimer(mode int32, new, old *itimerval)
   435  
   436  //go:noescape
   437  func timer_create(clockid int32, sevp *sigevent, timerid *int32) int32
   438  
   439  //go:noescape
   440  func timer_delete(timerid int32) int32
   441  
   442  //go:noescape
   443  func rtsigprocmask(how int32, new, old *sigset, size int32)
   444  
   445  //go:nosplit
   446  //go:nowritebarrierrec
   447  func sigprocmask(how int32, new, old *sigset) {
   448  	rtsigprocmask(how, new, old, int32(unsafe.Sizeof(*new)))
   449  }
   450  
   451  func raise(sig uint32)
   452  func raiseproc(sig uint32)
   453  
   454  //go:noescape
   455  func sched_getaffinity(pid, len uintptr, buf *byte) int32
   456  func osyield()
   457  
   458  //go:nosplit
   459  func osyield_no_g() {
   460  	osyield()
   461  }
   462  
   463  func pipe2(flags int32) (r, w int32, errno int32)
   464  
   465  //go:nosplit
   466  func fcntl(fd, cmd, arg int32) (ret int32, errno int32) {
   467  	r, _, err := linux.Syscall6(linux.SYS_FCNTL, uintptr(fd), uintptr(cmd), uintptr(arg), 0, 0, 0)
   468  	return int32(r), int32(err)
   469  }
   470  
   471  const (
   472  	_si_max_size    = 128
   473  	_sigev_max_size = 64
   474  )
   475  
   476  //go:nosplit
   477  //go:nowritebarrierrec
   478  func setsig(i uint32, fn uintptr) {
   479  	var sa sigactiont
   480  	sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER | _SA_RESTART
   481  	sigfillset(&sa.sa_mask)
   482  	// Although Linux manpage says "sa_restorer element is obsolete and
   483  	// should not be used". x86_64 kernel requires it. Only use it on
   484  	// x86. Note that on 386 this is cleared when using the C sigaction
   485  	// function via cgo; see fixSigactionForCgo.
   486  	if GOARCH == "386" || GOARCH == "amd64" {
   487  		sa.sa_restorer = abi.FuncPCABI0(sigreturn__sigaction)
   488  	}
   489  	if fn == abi.FuncPCABIInternal(sighandler) { // abi.FuncPCABIInternal(sighandler) matches the callers in signal_unix.go
   490  		if iscgo {
   491  			fn = abi.FuncPCABI0(cgoSigtramp)
   492  		} else {
   493  			fn = abi.FuncPCABI0(sigtramp)
   494  		}
   495  	}
   496  	sa.sa_handler = fn
   497  	sigaction(i, &sa, nil)
   498  }
   499  
   500  //go:nosplit
   501  //go:nowritebarrierrec
   502  func setsigstack(i uint32) {
   503  	var sa sigactiont
   504  	sigaction(i, nil, &sa)
   505  	if sa.sa_flags&_SA_ONSTACK != 0 {
   506  		return
   507  	}
   508  	sa.sa_flags |= _SA_ONSTACK
   509  	sigaction(i, &sa, nil)
   510  }
   511  
   512  //go:nosplit
   513  //go:nowritebarrierrec
   514  func getsig(i uint32) uintptr {
   515  	var sa sigactiont
   516  	sigaction(i, nil, &sa)
   517  	return sa.sa_handler
   518  }
   519  
   520  // setSignalstackSP sets the ss_sp field of a stackt.
   521  //
   522  //go:nosplit
   523  func setSignalstackSP(s *stackt, sp uintptr) {
   524  	*(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
   525  }
   526  
   527  //go:nosplit
   528  func (c *sigctxt) fixsigcode(sig uint32) {
   529  }
   530  
   531  // sysSigaction calls the rt_sigaction system call.
   532  //
   533  //go:nosplit
   534  func sysSigaction(sig uint32, new, old *sigactiont) {
   535  	if rt_sigaction(uintptr(sig), new, old, unsafe.Sizeof(sigactiont{}.sa_mask)) != 0 {
   536  		// Workaround for bugs in QEMU user mode emulation.
   537  		//
   538  		// QEMU turns calls to the sigaction system call into
   539  		// calls to the C library sigaction call; the C
   540  		// library call rejects attempts to call sigaction for
   541  		// SIGCANCEL (32) or SIGSETXID (33).
   542  		//
   543  		// QEMU rejects calling sigaction on SIGRTMAX (64).
   544  		//
   545  		// Just ignore the error in these case. There isn't
   546  		// anything we can do about it anyhow.
   547  		if sig != 32 && sig != 33 && sig != 64 {
   548  			// Use system stack to avoid split stack overflow on ppc64/ppc64le.
   549  			systemstack(func() {
   550  				throw("sigaction failed")
   551  			})
   552  		}
   553  	}
   554  }
   555  
   556  // rt_sigaction is implemented in assembly.
   557  //
   558  //go:noescape
   559  func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32
   560  
   561  // fixSigactionForCgo is called when we are using cgo to call the
   562  // C sigaction function. On 386 the C function does not expect the
   563  // SA_RESTORER flag to be set, and in some cases will fail if it is set:
   564  // it will pass the SA_RESTORER flag to the kernel without passing
   565  // the sa_restorer field. Since the C function will handle SA_RESTORER
   566  // for us, we need not pass it. See issue #75253.
   567  //
   568  //go:nosplit
   569  func fixSigactionForCgo(new *sigactiont) {
   570  	if GOARCH == "386" && new != nil {
   571  		new.sa_flags &^= _SA_RESTORER
   572  		new.sa_restorer = 0
   573  	}
   574  }
   575  
   576  func getpid() int
   577  func tgkill(tgid, tid, sig int)
   578  
   579  // signalM sends a signal to mp.
   580  func signalM(mp *m, sig int) {
   581  	tgkill(getpid(), int(mp.procid), sig)
   582  }
   583  
   584  // validSIGPROF compares this signal delivery's code against the signal sources
   585  // that the profiler uses, returning whether the delivery should be processed.
   586  // To be processed, a signal delivery from a known profiling mechanism should
   587  // correspond to the best profiling mechanism available to this thread. Signals
   588  // from other sources are always considered valid.
   589  //
   590  //go:nosplit
   591  func validSIGPROF(mp *m, c *sigctxt) bool {
   592  	code := int32(c.sigcode())
   593  	setitimer := code == _SI_KERNEL
   594  	timer_create := code == _SI_TIMER
   595  
   596  	if !(setitimer || timer_create) {
   597  		// The signal doesn't correspond to a profiling mechanism that the
   598  		// runtime enables itself. There's no reason to process it, but there's
   599  		// no reason to ignore it either.
   600  		return true
   601  	}
   602  
   603  	if mp == nil {
   604  		// Since we don't have an M, we can't check if there's an active
   605  		// per-thread timer for this thread. We don't know how long this thread
   606  		// has been around, and if it happened to interact with the Go scheduler
   607  		// at a time when profiling was active (causing it to have a per-thread
   608  		// timer). But it may have never interacted with the Go scheduler, or
   609  		// never while profiling was active. To avoid double-counting, process
   610  		// only signals from setitimer.
   611  		//
   612  		// When a custom cgo traceback function has been registered (on
   613  		// platforms that support runtime.SetCgoTraceback), SIGPROF signals
   614  		// delivered to a thread that cannot find a matching M do this check in
   615  		// the assembly implementations of runtime.cgoSigtramp.
   616  		return setitimer
   617  	}
   618  
   619  	// Having an M means the thread interacts with the Go scheduler, and we can
   620  	// check whether there's an active per-thread timer for this thread.
   621  	if mp.profileTimerValid.Load() {
   622  		// If this M has its own per-thread CPU profiling interval timer, we
   623  		// should track the SIGPROF signals that come from that timer (for
   624  		// accurate reporting of its CPU usage; see issue 35057) and ignore any
   625  		// that it gets from the process-wide setitimer (to not over-count its
   626  		// CPU consumption).
   627  		return timer_create
   628  	}
   629  
   630  	// No active per-thread timer means the only valid profiler is setitimer.
   631  	return setitimer
   632  }
   633  
   634  func setProcessCPUProfiler(hz int32) {
   635  	setProcessCPUProfilerTimer(hz)
   636  }
   637  
   638  func setThreadCPUProfiler(hz int32) {
   639  	mp := getg().m
   640  	mp.profilehz = hz
   641  
   642  	// destroy any active timer
   643  	if mp.profileTimerValid.Load() {
   644  		timerid := mp.profileTimer
   645  		mp.profileTimerValid.Store(false)
   646  		mp.profileTimer = 0
   647  
   648  		ret := timer_delete(timerid)
   649  		if ret != 0 {
   650  			print("runtime: failed to disable profiling timer; timer_delete(", timerid, ") errno=", -ret, "\n")
   651  			throw("timer_delete")
   652  		}
   653  	}
   654  
   655  	if hz == 0 {
   656  		// If the goal was to disable profiling for this thread, then the job's done.
   657  		return
   658  	}
   659  
   660  	// The period of the timer should be 1/Hz. For every "1/Hz" of additional
   661  	// work, the user should expect one additional sample in the profile.
   662  	//
   663  	// But to scale down to very small amounts of application work, to observe
   664  	// even CPU usage of "one tenth" of the requested period, set the initial
   665  	// timing delay in a different way: So that "one tenth" of a period of CPU
   666  	// spend shows up as a 10% chance of one sample (for an expected value of
   667  	// 0.1 samples), and so that "two and six tenths" periods of CPU spend show
   668  	// up as a 60% chance of 3 samples and a 40% chance of 2 samples (for an
   669  	// expected value of 2.6). Set the initial delay to a value in the uniform
   670  	// random distribution between 0 and the desired period. And because "0"
   671  	// means "disable timer", add 1 so the half-open interval [0,period) turns
   672  	// into (0,period].
   673  	//
   674  	// Otherwise, this would show up as a bias away from short-lived threads and
   675  	// from threads that are only occasionally active: for example, when the
   676  	// garbage collector runs on a mostly-idle system, the additional threads it
   677  	// activates may do a couple milliseconds of GC-related work and nothing
   678  	// else in the few seconds that the profiler observes.
   679  	spec := new(itimerspec)
   680  	spec.it_value.setNsec(1 + int64(cheaprandn(uint32(1e9/hz))))
   681  	spec.it_interval.setNsec(1e9 / int64(hz))
   682  
   683  	var timerid int32
   684  	var sevp sigevent
   685  	sevp.notify = _SIGEV_THREAD_ID
   686  	sevp.signo = _SIGPROF
   687  	sevp.sigev_notify_thread_id = int32(mp.procid)
   688  	ret := timer_create(_CLOCK_THREAD_CPUTIME_ID, &sevp, &timerid)
   689  	if ret != 0 {
   690  		// If we cannot create a timer for this M, leave profileTimerValid false
   691  		// to fall back to the process-wide setitimer profiler.
   692  		return
   693  	}
   694  
   695  	ret = timer_settime(timerid, 0, spec, nil)
   696  	if ret != 0 {
   697  		print("runtime: failed to configure profiling timer; timer_settime(", timerid,
   698  			", 0, {interval: {",
   699  			spec.it_interval.tv_sec, "s + ", spec.it_interval.tv_nsec, "ns} value: {",
   700  			spec.it_value.tv_sec, "s + ", spec.it_value.tv_nsec, "ns}}, nil) errno=", -ret, "\n")
   701  		throw("timer_settime")
   702  	}
   703  
   704  	mp.profileTimer = timerid
   705  	mp.profileTimerValid.Store(true)
   706  }
   707  
   708  // perThreadSyscallArgs contains the system call number, arguments, and
   709  // expected return values for a system call to be executed on all threads.
   710  type perThreadSyscallArgs struct {
   711  	trap uintptr
   712  	a1   uintptr
   713  	a2   uintptr
   714  	a3   uintptr
   715  	a4   uintptr
   716  	a5   uintptr
   717  	a6   uintptr
   718  	r1   uintptr
   719  	r2   uintptr
   720  }
   721  
   722  // perThreadSyscall is the system call to execute for the ongoing
   723  // doAllThreadsSyscall.
   724  //
   725  // perThreadSyscall may only be written while mp.needPerThreadSyscall == 0 on
   726  // all Ms.
   727  var perThreadSyscall perThreadSyscallArgs
   728  
   729  // syscall_runtime_doAllThreadsSyscall and executes a specified system call on
   730  // all Ms.
   731  //
   732  // The system call is expected to succeed and return the same value on every
   733  // thread. If any threads do not match, the runtime throws.
   734  //
   735  //go:linkname syscall_runtime_doAllThreadsSyscall syscall.runtime_doAllThreadsSyscall
   736  //go:uintptrescapes
   737  func syscall_runtime_doAllThreadsSyscall(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) {
   738  	if iscgo {
   739  		// In cgo, we are not aware of threads created in C, so this approach will not work.
   740  		panic("doAllThreadsSyscall not supported with cgo enabled")
   741  	}
   742  
   743  	// STW to guarantee that user goroutines see an atomic change to thread
   744  	// state. Without STW, goroutines could migrate Ms while change is in
   745  	// progress and e.g., see state old -> new -> old -> new.
   746  	//
   747  	// N.B. Internally, this function does not depend on STW to
   748  	// successfully change every thread. It is only needed for user
   749  	// expectations, per above.
   750  	stw := stopTheWorld(stwAllThreadsSyscall)
   751  
   752  	// This function depends on several properties:
   753  	//
   754  	// 1. All OS threads that already exist are associated with an M in
   755  	//    allm. i.e., we won't miss any pre-existing threads.
   756  	// 2. All Ms listed in allm will eventually have an OS thread exist.
   757  	//    i.e., they will set procid and be able to receive signals.
   758  	// 3. OS threads created after we read allm will clone from a thread
   759  	//    that has executed the system call. i.e., they inherit the
   760  	//    modified state.
   761  	//
   762  	// We achieve these through different mechanisms:
   763  	//
   764  	// 1. Addition of new Ms to allm in allocm happens before clone of its
   765  	//    OS thread later in newm.
   766  	// 2. newm does acquirem to avoid being preempted, ensuring that new Ms
   767  	//    created in allocm will eventually reach OS thread clone later in
   768  	//    newm.
   769  	// 3. We take allocmLock for write here to prevent allocation of new Ms
   770  	//    while this function runs. Per (1), this prevents clone of OS
   771  	//    threads that are not yet in allm.
   772  	allocmLock.lock()
   773  
   774  	// Disable preemption, preventing us from changing Ms, as we handle
   775  	// this M specially.
   776  	//
   777  	// N.B. STW and lock() above do this as well, this is added for extra
   778  	// clarity.
   779  	acquirem()
   780  
   781  	// N.B. allocmLock also prevents concurrent execution of this function,
   782  	// serializing use of perThreadSyscall, mp.needPerThreadSyscall, and
   783  	// ensuring all threads execute system calls from multiple calls in the
   784  	// same order.
   785  
   786  	r1, r2, errno := linux.Syscall6(trap, a1, a2, a3, a4, a5, a6)
   787  	if GOARCH == "ppc64" || GOARCH == "ppc64le" {
   788  		// TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2.
   789  		r2 = 0
   790  	}
   791  	if errno != 0 {
   792  		releasem(getg().m)
   793  		allocmLock.unlock()
   794  		startTheWorld(stw)
   795  		return r1, r2, errno
   796  	}
   797  
   798  	perThreadSyscall = perThreadSyscallArgs{
   799  		trap: trap,
   800  		a1:   a1,
   801  		a2:   a2,
   802  		a3:   a3,
   803  		a4:   a4,
   804  		a5:   a5,
   805  		a6:   a6,
   806  		r1:   r1,
   807  		r2:   r2,
   808  	}
   809  
   810  	// Wait for all threads to start.
   811  	//
   812  	// As described above, some Ms have been added to allm prior to
   813  	// allocmLock, but not yet completed OS clone and set procid.
   814  	//
   815  	// At minimum we must wait for a thread to set procid before we can
   816  	// send it a signal.
   817  	//
   818  	// We take this one step further and wait for all threads to start
   819  	// before sending any signals. This prevents system calls from getting
   820  	// applied twice: once in the parent and once in the child, like so:
   821  	//
   822  	//          A                     B                  C
   823  	//                         add C to allm
   824  	// doAllThreadsSyscall
   825  	//   allocmLock.lock()
   826  	//   signal B
   827  	//                         <receive signal>
   828  	//                         execute syscall
   829  	//                         <signal return>
   830  	//                         clone C
   831  	//                                             <thread start>
   832  	//                                             set procid
   833  	//   signal C
   834  	//                                             <receive signal>
   835  	//                                             execute syscall
   836  	//                                             <signal return>
   837  	//
   838  	// In this case, thread C inherited the syscall-modified state from
   839  	// thread B and did not need to execute the syscall, but did anyway
   840  	// because doAllThreadsSyscall could not be sure whether it was
   841  	// required.
   842  	//
   843  	// Some system calls may not be idempotent, so we ensure each thread
   844  	// executes the system call exactly once.
   845  	for mp := allm; mp != nil; mp = mp.alllink {
   846  		for atomic.Load64(&mp.procid) == 0 {
   847  			// Thread is starting.
   848  			osyield()
   849  		}
   850  	}
   851  
   852  	// Signal every other thread, where they will execute perThreadSyscall
   853  	// from the signal handler.
   854  	gp := getg()
   855  	tid := gp.m.procid
   856  	for mp := allm; mp != nil; mp = mp.alllink {
   857  		if atomic.Load64(&mp.procid) == tid {
   858  			// Our thread already performed the syscall.
   859  			continue
   860  		}
   861  		mp.needPerThreadSyscall.Store(1)
   862  		signalM(mp, sigPerThreadSyscall)
   863  	}
   864  
   865  	// Wait for all threads to complete.
   866  	for mp := allm; mp != nil; mp = mp.alllink {
   867  		if mp.procid == tid {
   868  			continue
   869  		}
   870  		for mp.needPerThreadSyscall.Load() != 0 {
   871  			osyield()
   872  		}
   873  	}
   874  
   875  	perThreadSyscall = perThreadSyscallArgs{}
   876  
   877  	releasem(getg().m)
   878  	allocmLock.unlock()
   879  	startTheWorld(stw)
   880  
   881  	return r1, r2, errno
   882  }
   883  
   884  // runPerThreadSyscall runs perThreadSyscall for this M if required.
   885  //
   886  // This function throws if the system call returns with anything other than the
   887  // expected values.
   888  //
   889  //go:nosplit
   890  func runPerThreadSyscall() {
   891  	gp := getg()
   892  	if gp.m.needPerThreadSyscall.Load() == 0 {
   893  		return
   894  	}
   895  
   896  	args := perThreadSyscall
   897  	r1, r2, errno := linux.Syscall6(args.trap, args.a1, args.a2, args.a3, args.a4, args.a5, args.a6)
   898  	if GOARCH == "ppc64" || GOARCH == "ppc64le" {
   899  		// TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2.
   900  		r2 = 0
   901  	}
   902  	if errno != 0 || r1 != args.r1 || r2 != args.r2 {
   903  		print("trap:", args.trap, ", a123456=[", args.a1, ",", args.a2, ",", args.a3, ",", args.a4, ",", args.a5, ",", args.a6, "]\n")
   904  		print("results: got {r1=", r1, ",r2=", r2, ",errno=", errno, "}, want {r1=", args.r1, ",r2=", args.r2, ",errno=0}\n")
   905  		fatal("AllThreadsSyscall6 results differ between threads; runtime corrupted")
   906  	}
   907  
   908  	gp.m.needPerThreadSyscall.Store(0)
   909  }
   910  
   911  const (
   912  	_SI_USER     = 0
   913  	_SI_TKILL    = -6
   914  	_SYS_SECCOMP = 1
   915  )
   916  
   917  // sigFromUser reports whether the signal was sent because of a call
   918  // to kill or tgkill.
   919  //
   920  //go:nosplit
   921  func (c *sigctxt) sigFromUser() bool {
   922  	code := int32(c.sigcode())
   923  	return code == _SI_USER || code == _SI_TKILL
   924  }
   925  
   926  // sigFromSeccomp reports whether the signal was sent from seccomp.
   927  //
   928  //go:nosplit
   929  func (c *sigctxt) sigFromSeccomp() bool {
   930  	code := int32(c.sigcode())
   931  	return code == _SYS_SECCOMP
   932  }
   933  
   934  //go:nosplit
   935  func mprotect(addr unsafe.Pointer, n uintptr, prot int32) (ret int32, errno int32) {
   936  	r, _, err := linux.Syscall6(linux.SYS_MPROTECT, uintptr(addr), n, uintptr(prot), 0, 0, 0)
   937  	return int32(r), int32(err)
   938  }
   939  
   940  type kernelVersion struct {
   941  	major int
   942  	minor int
   943  }
   944  
   945  // getKernelVersion returns major and minor kernel version numbers
   946  // parsed from the uname release field.
   947  func getKernelVersion() kernelVersion {
   948  	var buf linux.Utsname
   949  	if e := linux.Uname(&buf); e != 0 {
   950  		throw("uname failed")
   951  	}
   952  
   953  	rel := gostringnocopy(&buf.Release[0])
   954  	major, minor, _, ok := parseRelease(rel)
   955  	if !ok {
   956  		throw("failed to parse kernel version from uname")
   957  	}
   958  	return kernelVersion{major: major, minor: minor}
   959  }
   960  
   961  // parseRelease parses a dot-separated version number. It follows the
   962  // semver syntax, but allows the minor and patch versions to be
   963  // elided.
   964  func parseRelease(rel string) (major, minor, patch int, ok bool) {
   965  	// Strip anything after a dash or plus.
   966  	for i := 0; i < len(rel); i++ {
   967  		if rel[i] == '-' || rel[i] == '+' {
   968  			rel = rel[:i]
   969  			break
   970  		}
   971  	}
   972  
   973  	next := func() (int, bool) {
   974  		for i := 0; i < len(rel); i++ {
   975  			if rel[i] == '.' {
   976  				ver, err := strconv.Atoi(rel[:i])
   977  				rel = rel[i+1:]
   978  				return ver, err == nil
   979  			}
   980  		}
   981  		ver, err := strconv.Atoi(rel)
   982  		rel = ""
   983  		return ver, err == nil
   984  	}
   985  	if major, ok = next(); !ok || rel == "" {
   986  		return
   987  	}
   988  	if minor, ok = next(); !ok || rel == "" {
   989  		return
   990  	}
   991  	patch, ok = next()
   992  	return
   993  }
   994  
   995  // GE checks if the running kernel version
   996  // is greater than or equal to the provided version.
   997  func (kv kernelVersion) GE(x, y int) bool {
   998  	return kv.major > x || (kv.major == x && kv.minor >= y)
   999  }
  1000  

View as plain text