unified.go

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package noder
     6  
     7  import (
     8  	"cmp"
     9  	"fmt"
    10  	"internal/pkgbits"
    11  	"internal/types/errors"
    12  	"io"
    13  	"runtime"
    14  	"slices"
    15  	"strings"
    16  
    17  	"cmd/compile/internal/base"
    18  	"cmd/compile/internal/inline"
    19  	"cmd/compile/internal/ir"
    20  	"cmd/compile/internal/pgoir"
    21  	"cmd/compile/internal/typecheck"
    22  	"cmd/compile/internal/types"
    23  	"cmd/compile/internal/types2"
    24  	"cmd/internal/src"
    25  )
    26  
    27  // uirVersion is the unified IR version to use for encoding/decoding.
    28  // Use V3 for promoted struct field support in composite literals.
    29  const uirVersion = pkgbits.V3
    30  
    31  // localPkgReader holds the package reader used for reading the local
    32  // package. It exists so the unified IR linker can refer back to it
    33  // later.
    34  var localPkgReader *pkgReader
    35  
    36  // LookupFunc returns the ir.Func for an arbitrary full symbol name if
    37  // that function exists in the set of available export data.
    38  //
    39  // This allows lookup of arbitrary functions and methods that aren't otherwise
    40  // referenced by the local package and thus haven't been read yet.
    41  //
    42  // TODO(prattmic): Does not handle instantiation of generic types. Currently
    43  // profiles don't contain the original type arguments, so we won't be able to
    44  // create the runtime dictionaries.
    45  //
    46  // TODO(prattmic): Hit rate of this function is usually fairly low, and errors
    47  // are only used when debug logging is enabled. Consider constructing cheaper
    48  // errors by default.
    49  func LookupFunc(fullName string) (*ir.Func, error) {
    50  	pkgPath, symName, err := ir.ParseLinkFuncName(fullName)
    51  	if err != nil {
    52  		return nil, fmt.Errorf("error parsing symbol name %q: %v", fullName, err)
    53  	}
    54  
    55  	pkg, ok := types.PkgMap()[pkgPath]
    56  	if !ok {
    57  		return nil, fmt.Errorf("pkg %s doesn't exist in %v", pkgPath, types.PkgMap())
    58  	}
    59  
    60  	// Symbol naming is ambiguous. We can't necessarily distinguish between
    61  	// a method and a closure. e.g., is foo.Bar.func1 a closure defined in
    62  	// function Bar, or a method on type Bar? Thus we must simply attempt
    63  	// to lookup both.
    64  
    65  	fn, err := lookupFunction(pkg, symName)
    66  	if err == nil {
    67  		return fn, nil
    68  	}
    69  
    70  	fn, mErr := lookupMethod(pkg, symName)
    71  	if mErr == nil {
    72  		return fn, nil
    73  	}
    74  
    75  	return nil, fmt.Errorf("%s is not a function (%v) or method (%v)", fullName, err, mErr)
    76  }
    77  
    78  // PostLookupCleanup performs cleanup operations needed
    79  // after a series of calls to LookupFunc, specifically invoking
    80  // readBodies to post-process any funcs on the "todoBodies" list
    81  // that were added as a result of the lookup operations.
    82  func PostLookupCleanup() {
    83  	readBodies(typecheck.Target, false)
    84  }
    85  
    86  func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {
    87  	sym := pkg.Lookup(symName)
    88  
    89  	// TODO(prattmic): Enclosed functions (e.g., foo.Bar.func1) are not
    90  	// present in objReader, only as OCLOSURE nodes in the enclosing
    91  	// function.
    92  	pri, ok := objReader[sym]
    93  	if !ok {
    94  		return nil, fmt.Errorf("func sym %v missing objReader", sym)
    95  	}
    96  
    97  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
    98  	if err != nil {
    99  		return nil, fmt.Errorf("func sym %v lookup error: %w", sym, err)
   100  	}
   101  	name := node.(*ir.Name)
   102  	if name.Op() != ir.ONAME || name.Class != ir.PFUNC {
   103  		return nil, fmt.Errorf("func sym %v refers to non-function name: %v", sym, name)
   104  	}
   105  	return name.Func, nil
   106  }
   107  
   108  func lookupMethod(pkg *types.Pkg, symName string) (*ir.Func, error) {
   109  	// N.B. readPackage creates a Sym for every object in the package to
   110  	// initialize objReader and importBodyReader, even if the object isn't
   111  	// read.
   112  	//
   113  	// However, objReader is only initialized for top-level objects, so we
   114  	// must first lookup the type and use that to find the method rather
   115  	// than looking for the method directly.
   116  	typ, meth, err := ir.LookupMethodSelector(pkg, symName)
   117  	if err != nil {
   118  		return nil, fmt.Errorf("error looking up method symbol %q: %v", symName, err)
   119  	}
   120  
   121  	pri, ok := objReader[typ]
   122  	if !ok {
   123  		return nil, fmt.Errorf("type sym %v missing objReader", typ)
   124  	}
   125  
   126  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
   127  	if err != nil {
   128  		return nil, fmt.Errorf("func sym %v lookup error: %w", typ, err)
   129  	}
   130  	name := node.(*ir.Name)
   131  	if name.Op() != ir.OTYPE {
   132  		return nil, fmt.Errorf("type sym %v refers to non-type name: %v", typ, name)
   133  	}
   134  	if name.Alias() {
   135  		return nil, fmt.Errorf("type sym %v refers to alias", typ)
   136  	}
   137  	if name.Type().IsInterface() {
   138  		return nil, fmt.Errorf("type sym %v refers to interface type", typ)
   139  	}
   140  
   141  	for _, m := range name.Type().Methods() {
   142  		if m.Sym == meth {
   143  			fn := m.Nname.(*ir.Name).Func
   144  			return fn, nil
   145  		}
   146  	}
   147  
   148  	return nil, fmt.Errorf("method %s missing from method set of %v", symName, typ)
   149  }
   150  
   151  // unified constructs the local package's Internal Representation (IR)
   152  // from its syntax tree (AST).
   153  //
   154  // The pipeline contains 2 steps:
   155  //
   156  //  1. Generate the export data "stub".
   157  //
   158  //  2. Generate the IR from the export data above.
   159  //
   160  // The package data "stub" at step (1) contains everything from the local package,
   161  // but nothing that has been imported. When we're actually writing out export data
   162  // to the output files (see writeNewExport), we run the "linker", which:
   163  //
   164  //   - Updates compiler extensions data (e.g. inlining cost, escape analysis results).
   165  //
   166  //   - Handles re-exporting any transitive dependencies.
   167  //
   168  //   - Prunes out any unnecessary details (e.g. non-inlineable functions, because any
   169  //     downstream importers only care about inlinable functions).
   170  //
   171  // The source files are typechecked twice: once before writing the export data
   172  // using types2, and again after reading the export data using gc/typecheck.
   173  // The duplication of work will go away once we only use the types2 type checker,
   174  // removing the gc/typecheck step. For now, it is kept because:
   175  //
   176  //   - It reduces the engineering costs in maintaining a fork of typecheck
   177  //     (e.g. no need to backport fixes like CL 327651).
   178  //
   179  //   - It makes it easier to pass toolstash -cmp.
   180  //
   181  //   - Historically, we would always re-run the typechecker after importing a package,
   182  //     even though we know the imported data is valid. It's not ideal, but it's
   183  //     not causing any problems either.
   184  //
   185  //   - gc/typecheck is still in charge of some transformations, such as rewriting
   186  //     multi-valued function calls or transforming ir.OINDEX to ir.OINDEXMAP.
   187  //
   188  // Using the syntax tree with types2, which has a complete representation of generics,
   189  // the unified IR has the full typed AST needed for introspection during step (1).
   190  // In other words, we have all the necessary information to build the generic IR form
   191  // (see writer.captureVars for an example).
   192  func unified(m posMap, noders []*noder) {
   193  	inline.InlineCall = unifiedInlineCall
   194  	typecheck.HaveInlineBody = unifiedHaveInlineBody
   195  	pgoir.LookupFunc = LookupFunc
   196  	pgoir.PostLookupCleanup = PostLookupCleanup
   197  
   198  	data := writePkgStub(m, noders)
   199  
   200  	target := typecheck.Target
   201  
   202  	localPkgReader = newPkgReader(pkgbits.NewPkgDecoder(types.LocalPkg.Path, data))
   203  	readPackage(localPkgReader, types.LocalPkg, true)
   204  
   205  	r := localPkgReader.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   206  	r.pkgInit(types.LocalPkg, target)
   207  
   208  	readBodies(target, false)
   209  
   210  	// Check that nothing snuck past typechecking.
   211  	for _, fn := range target.Funcs {
   212  		if fn.Typecheck() == 0 {
   213  			base.FatalfAt(fn.Pos(), "missed typecheck: %v", fn)
   214  		}
   215  
   216  		// For functions, check that at least their first statement (if
   217  		// any) was typechecked too.
   218  		if len(fn.Body) != 0 {
   219  			if stmt := fn.Body[0]; stmt.Typecheck() == 0 {
   220  				base.FatalfAt(stmt.Pos(), "missed typecheck: %v", stmt)
   221  			}
   222  		}
   223  	}
   224  
   225  	// For functions originally came from package runtime,
   226  	// mark as norace to prevent instrumenting, see issue #60439.
   227  	for _, fn := range target.Funcs {
   228  		if !base.Flag.CompilingRuntime && types.RuntimeSymName(fn.Sym()) != "" {
   229  			fn.Pragma |= ir.Norace
   230  		}
   231  	}
   232  
   233  	base.ExitIfErrors() // just in case
   234  }
   235  
   236  // readBodies iteratively expands all pending dictionaries and
   237  // function bodies.
   238  //
   239  // If duringInlining is true, then the inline.InlineDecls is called as
   240  // necessary on instantiations of imported generic functions, so their
   241  // inlining costs can be computed.
   242  func readBodies(target *ir.Package, duringInlining bool) {
   243  	var inlDecls []*ir.Func
   244  
   245  	// Don't use range--bodyIdx can add closures to todoBodies.
   246  	for {
   247  		// The order we expand dictionaries and bodies doesn't matter, so
   248  		// pop from the end to reduce todoBodies reallocations if it grows
   249  		// further.
   250  		//
   251  		// However, we do at least need to flush any pending dictionaries
   252  		// before reading bodies, because bodies might reference the
   253  		// dictionaries.
   254  
   255  		if len(todoDicts) > 0 {
   256  			fn := todoDicts[len(todoDicts)-1]
   257  			todoDicts = todoDicts[:len(todoDicts)-1]
   258  			fn()
   259  			continue
   260  		}
   261  
   262  		if len(todoBodies) > 0 {
   263  			fn := todoBodies[len(todoBodies)-1]
   264  			todoBodies = todoBodies[:len(todoBodies)-1]
   265  
   266  			pri, ok := bodyReader[fn]
   267  			assert(ok)
   268  			pri.funcBody(fn)
   269  
   270  			// Instantiated generic function: add to Decls for typechecking
   271  			// and compilation.
   272  			if fn.OClosure == nil && len(pri.dict.targs) != 0 {
   273  				// cmd/link does not support a type symbol referencing a method symbol
   274  				// across DSO boundary, so force re-compiling methods on a generic type
   275  				// even it was seen from imported package in linkshared mode, see #58966.
   276  				canSkipNonGenericMethod := !(base.Ctxt.Flag_linkshared && ir.IsMethod(fn))
   277  				if duringInlining && canSkipNonGenericMethod {
   278  					inlDecls = append(inlDecls, fn)
   279  				} else {
   280  					target.Funcs = append(target.Funcs, fn)
   281  				}
   282  			}
   283  
   284  			continue
   285  		}
   286  
   287  		break
   288  	}
   289  
   290  	todoDicts = nil
   291  	todoBodies = nil
   292  
   293  	if len(inlDecls) != 0 {
   294  		// If we instantiated any generic functions during inlining, we need
   295  		// to call CanInline on them so they'll be transitively inlined
   296  		// correctly (#56280).
   297  		//
   298  		// We know these functions were already compiled in an imported
   299  		// package though, so we don't need to actually apply InlineCalls or
   300  		// save the function bodies any further than this.
   301  		//
   302  		// We can also lower the -m flag to 0, to suppress duplicate "can
   303  		// inline" diagnostics reported against the imported package. Again,
   304  		// we already reported those diagnostics in the original package, so
   305  		// it's pointless repeating them here.
   306  
   307  		oldLowerM := base.Flag.LowerM
   308  		base.Flag.LowerM = 0
   309  		inline.CanInlineFuncs(inlDecls, nil)
   310  		base.Flag.LowerM = oldLowerM
   311  
   312  		for _, fn := range inlDecls {
   313  			fn.Body = nil // free memory
   314  		}
   315  	}
   316  }
   317  
   318  // writePkgStub type checks the given parsed source files,
   319  // writes an export data package stub representing them,
   320  // and returns the result.
   321  func writePkgStub(m posMap, noders []*noder) string {
   322  	pkg, info, otherInfo := checkFiles(m, noders)
   323  
   324  	pw := newPkgWriter(m, pkg, info, otherInfo)
   325  
   326  	pw.collectDecls(noders)
   327  
   328  	publicRootWriter := pw.newWriter(pkgbits.SectionMeta, pkgbits.SyncPublic)
   329  	privateRootWriter := pw.newWriter(pkgbits.SectionMeta, pkgbits.SyncPrivate)
   330  
   331  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   332  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   333  
   334  	{
   335  		w := publicRootWriter
   336  		w.pkg(pkg)
   337  
   338  		if w.Version().Has(pkgbits.HasInit) {
   339  			w.Bool(false)
   340  		}
   341  
   342  		scope := pkg.Scope()
   343  		names := scope.Names()
   344  		w.Len(len(names))
   345  		for _, name := range names {
   346  			w.obj(scope.Lookup(name), nil)
   347  		}
   348  
   349  		w.Sync(pkgbits.SyncEOF)
   350  		w.Flush()
   351  	}
   352  
   353  	{
   354  		w := privateRootWriter
   355  		w.pkgInit(noders)
   356  		w.Flush()
   357  	}
   358  
   359  	var sb strings.Builder
   360  	pw.DumpTo(&sb)
   361  
   362  	// At this point, we're done with types2. Make sure the package is
   363  	// garbage collected.
   364  	freePackage(pkg)
   365  
   366  	return sb.String()
   367  }
   368  
   369  // freePackage ensures the given package is garbage collected.
   370  func freePackage(pkg *types2.Package) {
   371  	// The GC test below relies on a precise GC that runs finalizers as
   372  	// soon as objects are unreachable. Our implementation provides
   373  	// this, but other/older implementations may not (e.g., Go 1.4 does
   374  	// not because of #22350). To avoid imposing unnecessary
   375  	// restrictions on the GOROOT_BOOTSTRAP toolchain, we skip the test
   376  	// during bootstrapping.
   377  	if base.CompilerBootstrap || base.Debug.GCCheck == 0 {
   378  		*pkg = types2.Package{}
   379  		return
   380  	}
   381  
   382  	// Set a finalizer on pkg so we can detect if/when it's collected.
   383  	done := make(chan struct{})
   384  	runtime.SetFinalizer(pkg, func(*types2.Package) { close(done) })
   385  
   386  	// Important: objects involved in cycles are not finalized, so zero
   387  	// out pkg to break its cycles and allow the finalizer to run.
   388  	*pkg = types2.Package{}
   389  
   390  	// It typically takes just 1 or 2 cycles to release pkg, but it
   391  	// doesn't hurt to try a few more times.
   392  	for i := 0; i < 10; i++ {
   393  		select {
   394  		case <-done:
   395  			return
   396  		default:
   397  			runtime.GC()
   398  		}
   399  	}
   400  
   401  	base.Fatalf("package never finalized")
   402  }
   403  
   404  // readPackage reads package export data from pr to populate
   405  // importpkg.
   406  //
   407  // localStub indicates whether pr is reading the stub export data for
   408  // the local package, as opposed to relocated export data for an
   409  // import.
   410  func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {
   411  	{
   412  		r := pr.newReader(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   413  
   414  		pkg := r.pkg()
   415  		// This error can happen if "go tool compile" is called with wrong "-p" flag, see issue #54542.
   416  		if pkg != importpkg {
   417  			base.ErrorfAt(base.AutogeneratedPos, errors.BadImportPath, "mismatched import path, have %q (%p), want %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)
   418  			base.ErrorExit()
   419  		}
   420  
   421  		if r.Version().Has(pkgbits.HasInit) {
   422  			r.Bool()
   423  		}
   424  
   425  		for i, n := 0, r.Len(); i < n; i++ {
   426  			r.Sync(pkgbits.SyncObject)
   427  			if r.Version().Has(pkgbits.DerivedFuncInstance) {
   428  				assert(!r.Bool())
   429  			}
   430  			idx := r.Reloc(pkgbits.SectionObj)
   431  			assert(r.Len() == 0)
   432  
   433  			path, name, code := r.p.PeekObj(idx)
   434  			if code != pkgbits.ObjStub {
   435  				objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil, nil, nil}
   436  			}
   437  		}
   438  
   439  		r.Sync(pkgbits.SyncEOF)
   440  	}
   441  
   442  	if !localStub {
   443  		r := pr.newReader(pkgbits.SectionMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   444  
   445  		if r.Bool() {
   446  			sym := importpkg.Lookup(".inittask")
   447  			task := ir.NewNameAt(src.NoXPos, sym, nil)
   448  			task.Class = ir.PEXTERN
   449  			sym.Def = task
   450  		}
   451  
   452  		for i, n := 0, r.Len(); i < n; i++ {
   453  			path := r.String()
   454  			name := r.String()
   455  			idx := r.Reloc(pkgbits.SectionBody)
   456  
   457  			sym := types.NewPkg(path, "").Lookup(name)
   458  			if _, ok := importBodyReader[sym]; !ok {
   459  				importBodyReader[sym] = pkgReaderIndex{pr, idx, nil, nil, nil}
   460  			}
   461  		}
   462  
   463  		r.Sync(pkgbits.SyncEOF)
   464  	}
   465  }
   466  
   467  // writeUnifiedExport writes to `out` the finalized, self-contained
   468  // Unified IR export data file for the current compilation unit.
   469  func writeUnifiedExport(out io.Writer) {
   470  	l := linker{
   471  		pw: pkgbits.NewPkgEncoder(uirVersion, base.Debug.SyncFrames),
   472  
   473  		pkgs:   make(map[string]index),
   474  		decls:  make(map[*types.Sym]index),
   475  		bodies: make(map[*types.Sym]index),
   476  	}
   477  
   478  	publicRootWriter := l.pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPublic)
   479  	privateRootWriter := l.pw.NewEncoder(pkgbits.SectionMeta, pkgbits.SyncPrivate)
   480  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   481  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   482  
   483  	var selfPkgIdx index
   484  
   485  	{
   486  		pr := localPkgReader
   487  		r := pr.NewDecoder(pkgbits.SectionMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   488  
   489  		r.Sync(pkgbits.SyncPkg)
   490  		selfPkgIdx = l.relocIdx(pr, pkgbits.SectionPkg, r.Reloc(pkgbits.SectionPkg))
   491  
   492  		// Versions must match.
   493  		// TODO: It seems that we should be able to use r.Version() for NewPkgEncoder
   494  		// instead of passing uirVersion, but NewPkgEncoder is created before r.
   495  		// If that is correct, we should make that happen.
   496  		assert(r.Version() == uirVersion)
   497  
   498  		if r.Version().Has(pkgbits.HasInit) {
   499  			r.Bool()
   500  		}
   501  
   502  		for i, n := 0, r.Len(); i < n; i++ {
   503  			r.Sync(pkgbits.SyncObject)
   504  			if r.Version().Has(pkgbits.DerivedFuncInstance) {
   505  				assert(!r.Bool())
   506  			}
   507  			idx := r.Reloc(pkgbits.SectionObj)
   508  			assert(r.Len() == 0)
   509  
   510  			xpath, xname, xtag := pr.PeekObj(idx)
   511  			assert(xpath == pr.PkgPath())
   512  			assert(xtag != pkgbits.ObjStub)
   513  
   514  			if types.IsExported(xname) {
   515  				l.relocIdx(pr, pkgbits.SectionObj, idx)
   516  			}
   517  		}
   518  
   519  		r.Sync(pkgbits.SyncEOF)
   520  	}
   521  
   522  	{
   523  		var idxs []index
   524  		for _, idx := range l.decls {
   525  			idxs = append(idxs, idx)
   526  		}
   527  		slices.Sort(idxs)
   528  
   529  		w := publicRootWriter
   530  
   531  		w.Sync(pkgbits.SyncPkg)
   532  		w.Reloc(pkgbits.SectionPkg, selfPkgIdx)
   533  
   534  		if w.Version().Has(pkgbits.HasInit) {
   535  			w.Bool(false)
   536  		}
   537  
   538  		w.Len(len(idxs))
   539  		for _, idx := range idxs {
   540  			w.Sync(pkgbits.SyncObject)
   541  			if w.Version().Has(pkgbits.DerivedFuncInstance) {
   542  				w.Bool(false)
   543  			}
   544  			w.Reloc(pkgbits.SectionObj, idx)
   545  			w.Len(0)
   546  		}
   547  
   548  		w.Sync(pkgbits.SyncEOF)
   549  		w.Flush()
   550  	}
   551  
   552  	{
   553  		type symIdx struct {
   554  			sym *types.Sym
   555  			idx index
   556  		}
   557  		var bodies []symIdx
   558  		for sym, idx := range l.bodies {
   559  			bodies = append(bodies, symIdx{sym, idx})
   560  		}
   561  		slices.SortFunc(bodies, func(a, b symIdx) int { return cmp.Compare(a.idx, b.idx) })
   562  
   563  		w := privateRootWriter
   564  
   565  		w.Bool(typecheck.Lookup(".inittask").Def != nil)
   566  
   567  		w.Len(len(bodies))
   568  		for _, body := range bodies {
   569  			w.String(body.sym.Pkg.Path)
   570  			w.String(body.sym.Name)
   571  			w.Reloc(pkgbits.SectionBody, body.idx)
   572  		}
   573  
   574  		w.Sync(pkgbits.SyncEOF)
   575  		w.Flush()
   576  	}
   577  
   578  	base.Ctxt.Fingerprint = l.pw.DumpTo(out)
   579  }
   580
View as plain text