crash_test.go

     1  // Copyright 2024 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build goexperiment.runtimesecret && linux
     6  
     7  package secret
     8  
     9  import (
    10  	"bytes"
    11  	"debug/elf"
    12  	"fmt"
    13  	"internal/asan"
    14  	"internal/msan"
    15  	"internal/race"
    16  	"internal/testenv"
    17  	"io"
    18  	"os"
    19  	"os/exec"
    20  	"path/filepath"
    21  	"runtime"
    22  	"strings"
    23  	"syscall"
    24  	"testing"
    25  )
    26  
    27  // Copied from runtime/runtime-gdb_unix_test.go
    28  func canGenerateCore(t *testing.T) bool {
    29  	// Ensure there is enough RLIMIT_CORE available to generate a full core.
    30  	var lim syscall.Rlimit
    31  	err := syscall.Getrlimit(syscall.RLIMIT_CORE, &lim)
    32  	if err != nil {
    33  		t.Fatalf("error getting rlimit: %v", err)
    34  	}
    35  	// Minimum RLIMIT_CORE max to allow. This is a conservative estimate.
    36  	// Most systems allow infinity.
    37  	const minRlimitCore = 100 << 20 // 100 MB
    38  	if lim.Max < minRlimitCore {
    39  		t.Skipf("RLIMIT_CORE max too low: %#+v", lim)
    40  	}
    41  
    42  	// Make sure core pattern will send core to the current directory.
    43  	b, err := os.ReadFile("/proc/sys/kernel/core_pattern")
    44  	if err != nil {
    45  		t.Fatalf("error reading core_pattern: %v", err)
    46  	}
    47  	if string(b) != "core\n" {
    48  		t.Skipf("Unexpected core pattern %q", string(b))
    49  	}
    50  
    51  	coreUsesPID := false
    52  	b, err = os.ReadFile("/proc/sys/kernel/core_uses_pid")
    53  	if err == nil {
    54  		switch string(bytes.TrimSpace(b)) {
    55  		case "0":
    56  		case "1":
    57  			coreUsesPID = true
    58  		default:
    59  			t.Skipf("unexpected core_uses_pid value %q", string(b))
    60  		}
    61  	}
    62  	return coreUsesPID
    63  }
    64  
    65  func TestCore(t *testing.T) {
    66  	// use secret, grab a coredump, rummage through
    67  	// it, trying to find our secret.
    68  
    69  	switch runtime.GOARCH {
    70  	case "amd64", "arm64":
    71  	default:
    72  		t.Skip("unsupported arch")
    73  	}
    74  	coreUsesPid := canGenerateCore(t)
    75  
    76  	// Build our crashing program
    77  	// Because we need assembly files to properly dirty our state
    78  	// we need to construct a package in our temporary directory.
    79  	tmpDir := t.TempDir()
    80  	// copy our base source
    81  	err := copyToDir("./testdata/crash.go", tmpDir, nil)
    82  	if err != nil {
    83  		t.Fatalf("error copying directory %v", err)
    84  	}
    85  	// Copy our testing assembly files. Use the ones from the package
    86  	// to assure that they are always in sync
    87  	err = copyToDir("./asm_amd64.s", tmpDir, func(s string) string {
    88  		return strings.ReplaceAll(s, "runtime∕secret·", "main·")
    89  	})
    90  	if err != nil {
    91  		t.Fatalf("error copying file %v", err)
    92  	}
    93  	err = copyToDir("./asm_arm64.s", tmpDir, func(s string) string {
    94  		return strings.ReplaceAll(s, "runtime∕secret·", "main·")
    95  	})
    96  	if err != nil {
    97  		t.Fatalf("error copying file %v", err)
    98  	}
    99  	err = copyToDir("./stubs.go", tmpDir, func(s string) string {
   100  		return strings.Replace(s, "package secret", "package main", 1)
   101  	})
   102  	if err != nil {
   103  		t.Fatalf("error copying file %v", err)
   104  	}
   105  
   106  	// the crashing package will live out of tree, so its source files
   107  	// cannot refer to our internal packages. However, the assembly files
   108  	// can refer to internal names and we can pass the missing offsets as
   109  	// a small generated file
   110  	offsets := `
   111  	package main
   112  	const (
   113  		offsetX86HasAVX    = %v
   114  		offsetX86HasAVX512 = %v
   115  	)
   116  	`
   117  	err = os.WriteFile(filepath.Join(tmpDir, "offsets.go"), []byte(fmt.Sprintf(offsets, offsetX86HasAVX, offsetX86HasAVX512)), 0666)
   118  	if err != nil {
   119  		t.Fatalf("error writing offset file %v", err)
   120  	}
   121  
   122  	// generate go.mod file
   123  	cmd := exec.Command(testenv.GoToolPath(t), "mod", "init", "crashtest")
   124  	cmd.Dir = tmpDir
   125  	out, err := testenv.CleanCmdEnv(cmd).CombinedOutput()
   126  	if err != nil {
   127  		t.Fatalf("error initing module %v\n%s", err, out)
   128  	}
   129  
   130  	// Pass through the flags for any of the memory validating modes
   131  	args := []string{"build", "-o", filepath.Join(tmpDir, "a.exe")}
   132  	if msan.Enabled {
   133  		args = append(args, "-msan")
   134  	}
   135  	if asan.Enabled {
   136  		args = append(args, "-asan")
   137  	}
   138  	if race.Enabled {
   139  		args = append(args, "-race")
   140  	}
   141  	cmd = exec.Command(testenv.GoToolPath(t), args...)
   142  	cmd.Dir = tmpDir
   143  	out, err = testenv.CleanCmdEnv(cmd).CombinedOutput()
   144  	if err != nil {
   145  		t.Fatalf("error building source %v\n%s", err, out)
   146  	}
   147  
   148  	// Start the test binary.
   149  	cmd = testenv.CommandContext(t, t.Context(), "./a.exe")
   150  	cmd.Dir = tmpDir
   151  	var stdout strings.Builder
   152  	cmd.Stdout = &stdout
   153  	cmd.Stderr = &stdout
   154  
   155  	err = cmd.Run()
   156  	// For debugging.
   157  	t.Logf("\n\n\n--- START SUBPROCESS ---\n\n\n%s\n\n--- END SUBPROCESS ---\n\n\n", stdout.String())
   158  	if err == nil {
   159  		t.Fatalf("test binary did not crash")
   160  	}
   161  	eErr, ok := err.(*exec.ExitError)
   162  	if !ok {
   163  		t.Fatalf("error is not exit error: %v", err)
   164  	}
   165  	if eErr.Exited() {
   166  		t.Fatalf("process exited instead of being terminated: %v", eErr)
   167  	}
   168  
   169  	rummage(t, tmpDir, eErr.Pid(), coreUsesPid)
   170  }
   171  
   172  func copyToDir(name string, dir string, replace func(string) string) error {
   173  	f, err := os.ReadFile(name)
   174  	if err != nil {
   175  		return err
   176  	}
   177  	if replace != nil {
   178  		f = []byte(replace(string(f)))
   179  	}
   180  	return os.WriteFile(filepath.Join(dir, filepath.Base(name)), f, 0666)
   181  }
   182  
   183  type violation struct {
   184  	id  byte   // secret ID
   185  	off uint64 // offset in core dump
   186  }
   187  
   188  // A secret value that should never appear in a core dump,
   189  // except for this global variable itself.
   190  // The first byte of the secret is variable, to track
   191  // different instances of it.
   192  //
   193  // If this value is changed, update ./internal/crashsecret/main.go
   194  // TODO: this is little-endian specific.
   195  var secretStore = [8]byte{
   196  	0x00,
   197  	0x81,
   198  	0xa0,
   199  	0xc6,
   200  	0xb3,
   201  	0x01,
   202  	0x66,
   203  	0x53,
   204  }
   205  
   206  func rummage(t *testing.T, tmpDir string, pid int, coreUsesPid bool) {
   207  	coreFileName := "core"
   208  	if coreUsesPid {
   209  		coreFileName += fmt.Sprintf(".%d", pid)
   210  	}
   211  	core, err := os.Open(filepath.Join(tmpDir, coreFileName))
   212  	if err != nil {
   213  		t.Fatalf("core file not found: %v", err)
   214  	}
   215  	b, err := io.ReadAll(core)
   216  	if err != nil {
   217  		t.Fatalf("can't read core file: %v", err)
   218  	}
   219  
   220  	// Open elf view onto core file.
   221  	coreElf, err := elf.NewFile(core)
   222  	if err != nil {
   223  		t.Fatalf("can't parse core file: %v", err)
   224  	}
   225  
   226  	// Look for any places that have the secret.
   227  	var violations []violation // core file offsets where we found a secret
   228  	i := 0
   229  	for {
   230  		j := bytes.Index(b[i:], secretStore[1:])
   231  		if j < 0 {
   232  			break
   233  		}
   234  		j--
   235  		i += j
   236  
   237  		t.Errorf("secret %d found at offset %x in core file", b[i], i)
   238  		violations = append(violations, violation{
   239  			id:  b[i],
   240  			off: uint64(i),
   241  		})
   242  
   243  		i += len(secretStore)
   244  	}
   245  
   246  	// Get more specific data about where in the core we found the secrets.
   247  	regions := elfRegions(t, core, coreElf)
   248  	for _, r := range regions {
   249  		for _, v := range violations {
   250  			if v.off >= r.min && v.off < r.max {
   251  				var addr string
   252  				if r.addrMin != 0 {
   253  					addr = fmt.Sprintf(" addr=%x", r.addrMin+(v.off-r.min))
   254  				}
   255  				t.Logf("additional info: secret %d at offset %x in %s%s", v.id, v.off-r.min, r.name, addr)
   256  			}
   257  		}
   258  	}
   259  }
   260  
   261  type elfRegion struct {
   262  	name             string
   263  	min, max         uint64 // core file offset range
   264  	addrMin, addrMax uint64 // inferior address range (or 0,0 if no address, like registers)
   265  }
   266  
   267  func elfRegions(t *testing.T, core *os.File, coreElf *elf.File) []elfRegion {
   268  	var regions []elfRegion
   269  	for _, p := range coreElf.Progs {
   270  		regions = append(regions, elfRegion{
   271  			name:    fmt.Sprintf("%s[%s]", p.Type, p.Flags),
   272  			min:     p.Off,
   273  			max:     p.Off + min(p.Filesz, p.Memsz),
   274  			addrMin: p.Vaddr,
   275  			addrMax: p.Vaddr + min(p.Filesz, p.Memsz),
   276  		})
   277  	}
   278  
   279  	// TODO(dmo): parse thread regions for arm64.
   280  	// This doesn't invalidate the test, it just makes it harder to figure
   281  	// out where we're leaking stuff.
   282  	if runtime.GOARCH == "amd64" {
   283  		regions = append(regions, threadRegions(t, core, coreElf)...)
   284  	}
   285  
   286  	for i, r1 := range regions {
   287  		for j, r2 := range regions {
   288  			if i == j {
   289  				continue
   290  			}
   291  			if r1.max <= r2.min || r2.max <= r1.min {
   292  				continue
   293  			}
   294  			t.Fatalf("overlapping regions %v %v", r1, r2)
   295  		}
   296  	}
   297  
   298  	return regions
   299  }
   300  
   301  func threadRegions(t *testing.T, core *os.File, coreElf *elf.File) []elfRegion {
   302  	var regions []elfRegion
   303  
   304  	for _, prog := range coreElf.Progs {
   305  		if prog.Type != elf.PT_NOTE {
   306  			continue
   307  		}
   308  
   309  		b := make([]byte, prog.Filesz)
   310  		_, err := core.ReadAt(b, int64(prog.Off))
   311  		if err != nil {
   312  			t.Fatalf("can't read core file %v", err)
   313  		}
   314  		prefix := "unk"
   315  		b0 := b
   316  		for len(b) > 0 {
   317  			namesz := coreElf.ByteOrder.Uint32(b)
   318  			b = b[4:]
   319  			descsz := coreElf.ByteOrder.Uint32(b)
   320  			b = b[4:]
   321  			typ := elf.NType(coreElf.ByteOrder.Uint32(b))
   322  			b = b[4:]
   323  			name := string(b[:namesz-1])
   324  			b = b[(namesz+3)/4*4:]
   325  			off := prog.Off + uint64(len(b0)-len(b))
   326  			desc := b[:descsz]
   327  			b = b[(descsz+3)/4*4:]
   328  
   329  			if name != "CORE" && name != "LINUX" {
   330  				continue
   331  			}
   332  			end := off + uint64(len(desc))
   333  			// Note: amd64 specific
   334  			// See /usr/include/x86_64-linux-gnu/bits/sigcontext.h
   335  			//
   336  			//   struct _fpstate
   337  			switch typ {
   338  			case elf.NT_PRSTATUS:
   339  				pid := coreElf.ByteOrder.Uint32(desc[32:36])
   340  				prefix = fmt.Sprintf("thread%d: ", pid)
   341  				regions = append(regions, elfRegion{
   342  					name: prefix + "prstatus header",
   343  					min:  off,
   344  					max:  off + 112,
   345  				})
   346  				off += 112
   347  				greg := []string{
   348  					"r15",
   349  					"r14",
   350  					"r13",
   351  					"r12",
   352  					"rbp",
   353  					"rbx",
   354  					"r11",
   355  					"r10",
   356  					"r9",
   357  					"r8",
   358  					"rax",
   359  					"rcx",
   360  					"rdx",
   361  					"rsi",
   362  					"rdi",
   363  					"orig_rax",
   364  					"rip",
   365  					"cs",
   366  					"eflags",
   367  					"rsp",
   368  					"ss",
   369  					"fs_base",
   370  					"gs_base",
   371  					"ds",
   372  					"es",
   373  					"fs",
   374  					"gs",
   375  				}
   376  				for _, r := range greg {
   377  					regions = append(regions, elfRegion{
   378  						name: prefix + r,
   379  						min:  off,
   380  						max:  off + 8,
   381  					})
   382  					off += 8
   383  				}
   384  				regions = append(regions, elfRegion{
   385  					name: prefix + "prstatus footer",
   386  					min:  off,
   387  					max:  off + 8,
   388  				})
   389  				off += 8
   390  			case elf.NT_FPREGSET:
   391  				regions = append(regions, elfRegion{
   392  					name: prefix + "fpregset header",
   393  					min:  off,
   394  					max:  off + 32,
   395  				})
   396  				off += 32
   397  				for i := 0; i < 8; i++ {
   398  					regions = append(regions, elfRegion{
   399  						name: prefix + fmt.Sprintf("mmx%d", i),
   400  						min:  off,
   401  						max:  off + 16,
   402  					})
   403  					off += 16
   404  					// They are long double (10 bytes), but
   405  					// stored in 16-byte slots.
   406  				}
   407  				for i := 0; i < 16; i++ {
   408  					regions = append(regions, elfRegion{
   409  						name: prefix + fmt.Sprintf("xmm%d", i),
   410  						min:  off,
   411  						max:  off + 16,
   412  					})
   413  					off += 16
   414  				}
   415  				regions = append(regions, elfRegion{
   416  					name: prefix + "fpregset footer",
   417  					min:  off,
   418  					max:  off + 96,
   419  				})
   420  				off += 96
   421  				/*
   422  					case NT_X86_XSTATE: // aka NT_PRPSINFO+511
   423  						// legacy: 512 bytes
   424  						// xsave header: 64 bytes
   425  						fmt.Printf("hdr %v\n", desc[512:][:64])
   426  						// ymm high128: 256 bytes
   427  
   428  						println(len(desc))
   429  						fallthrough
   430  				*/
   431  			default:
   432  				regions = append(regions, elfRegion{
   433  					name: fmt.Sprintf("%s/%s", name, typ),
   434  					min:  off,
   435  					max:  off + uint64(len(desc)),
   436  				})
   437  				off += uint64(len(desc))
   438  			}
   439  			if off != end {
   440  				t.Fatalf("note section incomplete")
   441  			}
   442  		}
   443  	}
   444  	return regions
   445  }
   446
View as plain text