Source file src/cmd/go/internal/modfetch/cache.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"encoding/json"
    11  	"errors"
    12  	"fmt"
    13  	"io"
    14  	"io/fs"
    15  	"math/rand"
    16  	"os"
    17  	"path/filepath"
    18  	"runtime"
    19  	"strconv"
    20  	"strings"
    21  	"sync"
    22  	"syscall"
    23  
    24  	"cmd/go/internal/base"
    25  	"cmd/go/internal/cfg"
    26  	"cmd/go/internal/gover"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/modfetch/codehost"
    29  	"cmd/internal/par"
    30  	"cmd/internal/robustio"
    31  	"cmd/internal/telemetry/counter"
    32  
    33  	"golang.org/x/mod/module"
    34  	"golang.org/x/mod/semver"
    35  )
    36  
    37  func cacheDir(ctx context.Context, path string) (string, error) {
    38  	if err := checkCacheDir(ctx); err != nil {
    39  		return "", err
    40  	}
    41  	enc, err := module.EscapePath(path)
    42  	if err != nil {
    43  		return "", err
    44  	}
    45  	return filepath.Join(cfg.GOMODCACHE, "cache/download", enc, "/@v"), nil
    46  }
    47  
    48  func CachePath(ctx context.Context, m module.Version, suffix string) (string, error) {
    49  	if gover.IsToolchain(m.Path) {
    50  		return "", ErrToolchain
    51  	}
    52  	dir, err := cacheDir(ctx, m.Path)
    53  	if err != nil {
    54  		return "", err
    55  	}
    56  	if !gover.ModIsValid(m.Path, m.Version) {
    57  		return "", fmt.Errorf("non-semver module version %q", m.Version)
    58  	}
    59  	if module.CanonicalVersion(m.Version) != m.Version {
    60  		return "", fmt.Errorf("non-canonical module version %q", m.Version)
    61  	}
    62  	encVer, err := module.EscapeVersion(m.Version)
    63  	if err != nil {
    64  		return "", err
    65  	}
    66  	return filepath.Join(dir, encVer+"."+suffix), nil
    67  }
    68  
    69  // DownloadDir returns the directory to which m should have been downloaded.
    70  // An error will be returned if the module path or version cannot be escaped.
    71  // An error satisfying errors.Is(err, fs.ErrNotExist) will be returned
    72  // along with the directory if the directory does not exist or if the directory
    73  // is not completely populated.
    74  func DownloadDir(ctx context.Context, m module.Version) (string, error) {
    75  	if gover.IsToolchain(m.Path) {
    76  		return "", ErrToolchain
    77  	}
    78  	if err := checkCacheDir(ctx); err != nil {
    79  		return "", err
    80  	}
    81  	enc, err := module.EscapePath(m.Path)
    82  	if err != nil {
    83  		return "", err
    84  	}
    85  	if !gover.ModIsValid(m.Path, m.Version) {
    86  		return "", fmt.Errorf("non-semver module version %q", m.Version)
    87  	}
    88  	if module.CanonicalVersion(m.Version) != m.Version {
    89  		return "", fmt.Errorf("non-canonical module version %q", m.Version)
    90  	}
    91  	encVer, err := module.EscapeVersion(m.Version)
    92  	if err != nil {
    93  		return "", err
    94  	}
    95  
    96  	// Check whether the directory itself exists.
    97  	dir := filepath.Join(cfg.GOMODCACHE, enc+"@"+encVer)
    98  	if fi, err := os.Stat(dir); os.IsNotExist(err) {
    99  		return dir, err
   100  	} else if err != nil {
   101  		return dir, &DownloadDirPartialError{dir, err}
   102  	} else if !fi.IsDir() {
   103  		return dir, &DownloadDirPartialError{dir, errors.New("not a directory")}
   104  	}
   105  
   106  	// Check if a .partial file exists. This is created at the beginning of
   107  	// a download and removed after the zip is extracted.
   108  	partialPath, err := CachePath(ctx, m, "partial")
   109  	if err != nil {
   110  		return dir, err
   111  	}
   112  	if _, err := os.Stat(partialPath); err == nil {
   113  		return dir, &DownloadDirPartialError{dir, errors.New("not completely extracted")}
   114  	} else if !os.IsNotExist(err) {
   115  		return dir, err
   116  	}
   117  
   118  	// Special case: ziphash is not required for the golang.org/fips140 module,
   119  	// because it is unpacked from a file in GOROOT, not downloaded.
   120  	// We've already checked that it's not a partial unpacking, so we're happy.
   121  	if m.Path == "golang.org/fips140" {
   122  		return dir, nil
   123  	}
   124  
   125  	// Check if a .ziphash file exists. It should be created before the
   126  	// zip is extracted, but if it was deleted (by another program?), we need
   127  	// to re-calculate it. Note that checkMod will repopulate the ziphash
   128  	// file if it doesn't exist, but if the module is excluded by checks
   129  	// through GONOSUMDB or GOPRIVATE, that check and repopulation won't happen.
   130  	ziphashPath, err := CachePath(ctx, m, "ziphash")
   131  	if err != nil {
   132  		return dir, err
   133  	}
   134  	if _, err := os.Stat(ziphashPath); os.IsNotExist(err) {
   135  		return dir, &DownloadDirPartialError{dir, errors.New("ziphash file is missing")}
   136  	} else if err != nil {
   137  		return dir, err
   138  	}
   139  	return dir, nil
   140  }
   141  
   142  // DownloadDirPartialError is returned by DownloadDir if a module directory
   143  // exists but was not completely populated.
   144  //
   145  // DownloadDirPartialError is equivalent to fs.ErrNotExist.
   146  type DownloadDirPartialError struct {
   147  	Dir string
   148  	Err error
   149  }
   150  
   151  func (e *DownloadDirPartialError) Error() string     { return fmt.Sprintf("%s: %v", e.Dir, e.Err) }
   152  func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist }
   153  
   154  // lockVersion locks a file within the module cache that guards the downloading
   155  // and extraction of the zipfile for the given module version.
   156  func lockVersion(ctx context.Context, mod module.Version) (unlock func(), err error) {
   157  	path, err := CachePath(ctx, mod, "lock")
   158  	if err != nil {
   159  		return nil, err
   160  	}
   161  	if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil {
   162  		return nil, err
   163  	}
   164  	return lockedfile.MutexAt(path).Lock()
   165  }
   166  
   167  // SideLock locks a file within the module cache that previously guarded
   168  // edits to files outside the cache, such as go.sum and go.mod files in the
   169  // user's working directory.
   170  // If err is nil, the caller MUST eventually call the unlock function.
   171  func SideLock(ctx context.Context) (unlock func(), err error) {
   172  	if err := checkCacheDir(ctx); err != nil {
   173  		return nil, err
   174  	}
   175  
   176  	path := filepath.Join(cfg.GOMODCACHE, "cache", "lock")
   177  	if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil {
   178  		return nil, fmt.Errorf("failed to create cache directory: %w", err)
   179  	}
   180  
   181  	return lockedfile.MutexAt(path).Lock()
   182  }
   183  
   184  // A cachingRepo is a cache around an underlying Repo,
   185  // avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not CheckReuse or Zip).
   186  // It is also safe for simultaneous use by multiple goroutines
   187  // (so that it can be returned from Lookup multiple times).
   188  // It serializes calls to the underlying Repo.
   189  type cachingRepo struct {
   190  	path          string
   191  	versionsCache par.ErrCache[string, *Versions]
   192  	statCache     par.ErrCache[string, *RevInfo]
   193  	latestCache   par.ErrCache[struct{}, *RevInfo]
   194  	gomodCache    par.ErrCache[string, []byte]
   195  
   196  	once     sync.Once
   197  	initRepo func(context.Context) (Repo, error)
   198  	r        Repo
   199  	fetcher  *Fetcher
   200  }
   201  
   202  func newCachingRepo(ctx context.Context, fetcher *Fetcher, path string, initRepo func(context.Context) (Repo, error)) *cachingRepo {
   203  	return &cachingRepo{
   204  		path:     path,
   205  		initRepo: initRepo,
   206  		fetcher:  fetcher,
   207  	}
   208  }
   209  
   210  func (r *cachingRepo) repo(ctx context.Context) Repo {
   211  	r.once.Do(func() {
   212  		var err error
   213  		r.r, err = r.initRepo(ctx)
   214  		if err != nil {
   215  			r.r = errRepo{r.path, err}
   216  		}
   217  	})
   218  	return r.r
   219  }
   220  
   221  func (r *cachingRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error {
   222  	return r.repo(ctx).CheckReuse(ctx, old)
   223  }
   224  
   225  func (r *cachingRepo) ModulePath() string {
   226  	return r.path
   227  }
   228  
   229  func (r *cachingRepo) Versions(ctx context.Context, prefix string) (*Versions, error) {
   230  	v, err := r.versionsCache.Do(prefix, func() (*Versions, error) {
   231  		return r.repo(ctx).Versions(ctx, prefix)
   232  	})
   233  	if err != nil {
   234  		return nil, err
   235  	}
   236  	return &Versions{
   237  		Origin: v.Origin,
   238  		List:   append([]string(nil), v.List...),
   239  	}, nil
   240  }
   241  
   242  type cachedInfo struct {
   243  	info *RevInfo
   244  	err  error
   245  }
   246  
   247  func (r *cachingRepo) Stat(ctx context.Context, rev string) (*RevInfo, error) {
   248  	if gover.IsToolchain(r.path) {
   249  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   250  		return r.repo(ctx).Stat(ctx, rev)
   251  	}
   252  	info, err := r.statCache.Do(rev, func() (*RevInfo, error) {
   253  		file, info, err := readDiskStat(ctx, r.path, rev)
   254  		if err == nil {
   255  			return info, err
   256  		}
   257  
   258  		info, err = r.repo(ctx).Stat(ctx, rev)
   259  		if err == nil {
   260  			// If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78,
   261  			// then save the information under the proper version, for future use.
   262  			if info.Version != rev {
   263  				file, _ = CachePath(ctx, module.Version{Path: r.path, Version: info.Version}, "info")
   264  				r.statCache.Do(info.Version, func() (*RevInfo, error) {
   265  					return info, nil
   266  				})
   267  			}
   268  
   269  			if err := writeDiskStat(ctx, file, info); err != nil && !isErrReadOnlyFS(err) {
   270  				fmt.Fprintf(os.Stderr, "go: writing stat cache: %v\n", err)
   271  			}
   272  		}
   273  		return info, err
   274  	})
   275  	if info != nil {
   276  		copy := *info
   277  		info = &copy
   278  	}
   279  	return info, err
   280  }
   281  
   282  func (r *cachingRepo) Latest(ctx context.Context) (*RevInfo, error) {
   283  	if gover.IsToolchain(r.path) {
   284  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   285  		return r.repo(ctx).Latest(ctx)
   286  	}
   287  	info, err := r.latestCache.Do(struct{}{}, func() (*RevInfo, error) {
   288  		info, err := r.repo(ctx).Latest(ctx)
   289  
   290  		// Save info for likely future Stat call.
   291  		if err == nil {
   292  			r.statCache.Do(info.Version, func() (*RevInfo, error) {
   293  				return info, nil
   294  			})
   295  			if file, _, err := readDiskStat(ctx, r.path, info.Version); err != nil {
   296  				writeDiskStat(ctx, file, info)
   297  			}
   298  		}
   299  
   300  		return info, err
   301  	})
   302  	if info != nil {
   303  		copy := *info
   304  		info = &copy
   305  	}
   306  	return info, err
   307  }
   308  
   309  func (r *cachingRepo) GoMod(ctx context.Context, version string) ([]byte, error) {
   310  	if gover.IsToolchain(r.path) {
   311  		// Skip disk cache; the underlying golang.org/toolchain repo is cached instead.
   312  		return r.repo(ctx).GoMod(ctx, version)
   313  	}
   314  	text, err := r.gomodCache.Do(version, func() ([]byte, error) {
   315  		file, text, err := r.fetcher.readDiskGoMod(ctx, r.path, version)
   316  		if err == nil {
   317  			// Note: readDiskGoMod already called checkGoMod.
   318  			return text, nil
   319  		}
   320  
   321  		text, err = r.repo(ctx).GoMod(ctx, version)
   322  		if err == nil {
   323  			if err := checkGoMod(r.fetcher, r.path, version, text); err != nil {
   324  				return text, err
   325  			}
   326  			if err := writeDiskGoMod(ctx, file, text); err != nil && !isErrReadOnlyFS(err) {
   327  				fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err)
   328  			}
   329  		}
   330  		return text, err
   331  	})
   332  	if err != nil {
   333  		return nil, err
   334  	}
   335  	return append([]byte(nil), text...), nil
   336  }
   337  
   338  func (r *cachingRepo) Zip(ctx context.Context, dst io.Writer, version string) error {
   339  	if gover.IsToolchain(r.path) {
   340  		return ErrToolchain
   341  	}
   342  	return r.repo(ctx).Zip(ctx, dst, version)
   343  }
   344  
   345  // InfoFile is like Lookup(ctx, path).Stat(version) but also returns the name of the file
   346  // containing the cached information.
   347  func (f *Fetcher) InfoFile(ctx context.Context, path, version string) (*RevInfo, string, error) {
   348  	if !gover.ModIsValid(path, version) {
   349  		return nil, "", fmt.Errorf("invalid version %q", version)
   350  	}
   351  
   352  	if file, info, err := readDiskStat(ctx, path, version); err == nil {
   353  		return info, file, nil
   354  	}
   355  
   356  	var info *RevInfo
   357  	var err2info map[error]*RevInfo
   358  	err := TryProxies(func(proxy string) error {
   359  		i, err := f.Lookup(ctx, proxy, path).Stat(ctx, version)
   360  		if err == nil {
   361  			info = i
   362  		} else {
   363  			if err2info == nil {
   364  				err2info = make(map[error]*RevInfo)
   365  			}
   366  			err2info[err] = info
   367  		}
   368  		return err
   369  	})
   370  	if err != nil {
   371  		return err2info[err], "", err
   372  	}
   373  
   374  	// Stat should have populated the disk cache for us.
   375  	file, err := CachePath(ctx, module.Version{Path: path, Version: version}, "info")
   376  	if err != nil {
   377  		return nil, "", err
   378  	}
   379  	return info, file, nil
   380  }
   381  
   382  // GoMod is like Lookup(ctx, path).GoMod(rev) but avoids the
   383  // repository path resolution in Lookup if the result is
   384  // already cached on local disk.
   385  func (f *Fetcher) GoMod(ctx context.Context, path, rev string) ([]byte, error) {
   386  	// Convert commit hash to pseudo-version
   387  	// to increase cache hit rate.
   388  	if !gover.ModIsValid(path, rev) {
   389  		if _, info, err := readDiskStat(ctx, path, rev); err == nil {
   390  			rev = info.Version
   391  		} else {
   392  			if errors.Is(err, statCacheErr) {
   393  				return nil, err
   394  			}
   395  			err := TryProxies(func(proxy string) error {
   396  				info, err := f.Lookup(ctx, proxy, path).Stat(ctx, rev)
   397  				if err == nil {
   398  					rev = info.Version
   399  				}
   400  				return err
   401  			})
   402  			if err != nil {
   403  				return nil, err
   404  			}
   405  		}
   406  	}
   407  
   408  	_, data, err := f.readDiskGoMod(ctx, path, rev)
   409  	if err == nil {
   410  		return data, nil
   411  	}
   412  
   413  	err = TryProxies(func(proxy string) (err error) {
   414  		data, err = f.Lookup(ctx, proxy, path).GoMod(ctx, rev)
   415  		return err
   416  	})
   417  	return data, err
   418  }
   419  
   420  // GoModFile is like GoMod but returns the name of the file containing
   421  // the cached information.
   422  func (f *Fetcher) GoModFile(ctx context.Context, path, version string) (string, error) {
   423  	if !gover.ModIsValid(path, version) {
   424  		return "", fmt.Errorf("invalid version %q", version)
   425  	}
   426  	if _, err := f.GoMod(ctx, path, version); err != nil {
   427  		return "", err
   428  	}
   429  	// GoMod should have populated the disk cache for us.
   430  	file, err := CachePath(ctx, module.Version{Path: path, Version: version}, "mod")
   431  	if err != nil {
   432  		return "", err
   433  	}
   434  	return file, nil
   435  }
   436  
   437  // GoModSum returns the go.sum entry for the module version's go.mod file.
   438  // (That is, it returns the entry listed in go.sum as "path version/go.mod".)
   439  func (f *Fetcher) GoModSum(ctx context.Context, path, version string) (string, error) {
   440  	if !gover.ModIsValid(path, version) {
   441  		return "", fmt.Errorf("invalid version %q", version)
   442  	}
   443  	data, err := f.GoMod(ctx, path, version)
   444  	if err != nil {
   445  		return "", err
   446  	}
   447  	sum, err := goModSum(data)
   448  	if err != nil {
   449  		return "", err
   450  	}
   451  	return sum, nil
   452  }
   453  
   454  var errNotCached = fmt.Errorf("not in cache")
   455  
   456  // readDiskStat reads a cached stat result from disk,
   457  // returning the name of the cache file and the result.
   458  // If the read fails, the caller can use
   459  // writeDiskStat(file, info) to write a new cache entry.
   460  func readDiskStat(ctx context.Context, path, rev string) (file string, info *RevInfo, err error) {
   461  	if gover.IsToolchain(path) {
   462  		return "", nil, errNotCached
   463  	}
   464  	file, data, err := readDiskCache(ctx, path, rev, "info")
   465  	if err != nil {
   466  		// If the cache already contains a pseudo-version with the given hash, we
   467  		// would previously return that pseudo-version without checking upstream.
   468  		// However, that produced an unfortunate side-effect: if the author added a
   469  		// tag to the repository, 'go get' would not pick up the effect of that new
   470  		// tag on the existing commits, and 'go' commands that referred to those
   471  		// commits would use the previous name instead of the new one.
   472  		//
   473  		// That's especially problematic if the original pseudo-version starts with
   474  		// v0.0.0-, as was the case for all pseudo-versions during vgo development,
   475  		// since a v0.0.0- pseudo-version has lower precedence than pretty much any
   476  		// tagged version.
   477  		//
   478  		// In practice, we're only looking up by hash during initial conversion of a
   479  		// legacy config and during an explicit 'go get', and a little extra latency
   480  		// for those operations seems worth the benefit of picking up more accurate
   481  		// versions.
   482  		//
   483  		// Fall back to this resolution scheme only if the GOPROXY setting prohibits
   484  		// us from resolving upstream tags.
   485  		if cfg.GOPROXY == "off" {
   486  			if file, info, err := readDiskStatByHash(ctx, path, rev); err == nil {
   487  				return file, info, nil
   488  			}
   489  		}
   490  		return file, nil, err
   491  	}
   492  	info = new(RevInfo)
   493  	if err := json.Unmarshal(data, info); err != nil {
   494  		return file, nil, errNotCached
   495  	}
   496  	// The disk might have stale .info files that have Name and Short fields set.
   497  	// We want to canonicalize to .info files with those fields omitted.
   498  	// Remarshal and update the cache file if needed.
   499  	data2, err := json.Marshal(info)
   500  	if err == nil && !bytes.Equal(data2, data) {
   501  		writeDiskCache(ctx, file, data)
   502  	}
   503  	return file, info, nil
   504  }
   505  
   506  // readDiskStatByHash is a fallback for readDiskStat for the case
   507  // where rev is a commit hash instead of a proper semantic version.
   508  // In that case, we look for a cached pseudo-version that matches
   509  // the commit hash. If we find one, we use it.
   510  // This matters most for converting legacy package management
   511  // configs, when we are often looking up commits by full hash.
   512  // Without this check we'd be doing network I/O to the remote repo
   513  // just to find out about a commit we already know about
   514  // (and have cached under its pseudo-version).
   515  func readDiskStatByHash(ctx context.Context, path, rev string) (file string, info *RevInfo, err error) {
   516  	if gover.IsToolchain(path) {
   517  		return "", nil, errNotCached
   518  	}
   519  	if cfg.GOMODCACHE == "" {
   520  		// Do not download to current directory.
   521  		return "", nil, errNotCached
   522  	}
   523  
   524  	if !codehost.AllHex(rev) || len(rev) < 12 {
   525  		return "", nil, errNotCached
   526  	}
   527  	rev = rev[:12]
   528  	cdir, err := cacheDir(ctx, path)
   529  	if err != nil {
   530  		return "", nil, errNotCached
   531  	}
   532  	dir, err := os.Open(cdir)
   533  	if err != nil {
   534  		return "", nil, errNotCached
   535  	}
   536  	names, err := dir.Readdirnames(-1)
   537  	dir.Close()
   538  	if err != nil {
   539  		return "", nil, errNotCached
   540  	}
   541  
   542  	// A given commit hash may map to more than one pseudo-version,
   543  	// depending on which tags are present on the repository.
   544  	// Take the highest such version.
   545  	var maxVersion string
   546  	suffix := "-" + rev + ".info"
   547  	err = errNotCached
   548  	for _, name := range names {
   549  		if strings.HasSuffix(name, suffix) {
   550  			v := strings.TrimSuffix(name, ".info")
   551  			if module.IsPseudoVersion(v) && semver.Compare(v, maxVersion) > 0 {
   552  				maxVersion = v
   553  				file, info, err = readDiskStat(ctx, path, strings.TrimSuffix(name, ".info"))
   554  			}
   555  		}
   556  	}
   557  	return file, info, err
   558  }
   559  
   560  // oldVgoPrefix is the prefix in the old auto-generated cached go.mod files.
   561  // We stopped trying to auto-generate the go.mod files. Now we use a trivial
   562  // go.mod with only a module line, and we've dropped the version prefix
   563  // entirely. If we see a version prefix, that means we're looking at an old copy
   564  // and should ignore it.
   565  var oldVgoPrefix = []byte("//vgo 0.0.")
   566  
   567  // readDiskGoMod reads a cached go.mod file from disk,
   568  // returning the name of the cache file and the result.
   569  // If the read fails, the caller can use
   570  // writeDiskGoMod(file, data) to write a new cache entry.
   571  func (f *Fetcher) readDiskGoMod(ctx context.Context, path, rev string) (file string, data []byte, err error) {
   572  	if gover.IsToolchain(path) {
   573  		return "", nil, errNotCached
   574  	}
   575  	file, data, err = readDiskCache(ctx, path, rev, "mod")
   576  
   577  	// If the file has an old auto-conversion prefix, pretend it's not there.
   578  	if bytes.HasPrefix(data, oldVgoPrefix) {
   579  		err = errNotCached
   580  		data = nil
   581  	}
   582  
   583  	if err == nil {
   584  		if err := checkGoMod(f, path, rev, data); err != nil {
   585  			return "", nil, err
   586  		}
   587  	}
   588  
   589  	return file, data, err
   590  }
   591  
   592  // readDiskCache is the generic "read from a cache file" implementation.
   593  // It takes the revision and an identifying suffix for the kind of data being cached.
   594  // It returns the name of the cache file and the content of the file.
   595  // If the read fails, the caller can use
   596  // writeDiskCache(file, data) to write a new cache entry.
   597  func readDiskCache(ctx context.Context, path, rev, suffix string) (file string, data []byte, err error) {
   598  	if gover.IsToolchain(path) {
   599  		return "", nil, errNotCached
   600  	}
   601  	file, err = CachePath(ctx, module.Version{Path: path, Version: rev}, suffix)
   602  	if err != nil {
   603  		return "", nil, errNotCached
   604  	}
   605  	data, err = robustio.ReadFile(file)
   606  	if err != nil {
   607  		return file, nil, errNotCached
   608  	}
   609  	return file, data, nil
   610  }
   611  
   612  // writeDiskStat writes a stat result cache entry.
   613  // The file name must have been returned by a previous call to readDiskStat.
   614  func writeDiskStat(ctx context.Context, file string, info *RevInfo) error {
   615  	if file == "" {
   616  		return nil
   617  	}
   618  
   619  	if info.Origin != nil {
   620  		// Clean the origin information, which might have too many
   621  		// validation criteria, for example if we are saving the result of
   622  		// m@master as m@pseudo-version.
   623  		clean := *info
   624  		info = &clean
   625  		o := *info.Origin
   626  		info.Origin = &o
   627  
   628  		// Tags and RepoSum never matter if you are starting with a semver version,
   629  		// as we would be when finding this cache entry.
   630  		o.TagSum = ""
   631  		o.TagPrefix = ""
   632  		o.RepoSum = ""
   633  		// Ref doesn't matter if you have a pseudoversion.
   634  		if module.IsPseudoVersion(info.Version) {
   635  			o.Ref = ""
   636  		}
   637  	}
   638  
   639  	js, err := json.Marshal(info)
   640  	if err != nil {
   641  		return err
   642  	}
   643  	return writeDiskCache(ctx, file, js)
   644  }
   645  
   646  // writeDiskGoMod writes a go.mod cache entry.
   647  // The file name must have been returned by a previous call to readDiskGoMod.
   648  func writeDiskGoMod(ctx context.Context, file string, text []byte) error {
   649  	return writeDiskCache(ctx, file, text)
   650  }
   651  
   652  // writeDiskCache is the generic "write to a cache file" implementation.
   653  // The file must have been returned by a previous call to readDiskCache.
   654  func writeDiskCache(ctx context.Context, file string, data []byte) error {
   655  	if file == "" {
   656  		return nil
   657  	}
   658  	// Make sure directory for file exists.
   659  	if err := os.MkdirAll(filepath.Dir(file), 0o777); err != nil {
   660  		return err
   661  	}
   662  
   663  	// Write the file to a temporary location, and then rename it to its final
   664  	// path to reduce the likelihood of a corrupt file existing at that final path.
   665  	f, err := tempFile(ctx, filepath.Dir(file), filepath.Base(file), 0o666)
   666  	if err != nil {
   667  		return err
   668  	}
   669  	defer func() {
   670  		// Only call os.Remove on f.Name() if we failed to rename it: otherwise,
   671  		// some other process may have created a new file with the same name after
   672  		// the rename completed.
   673  		if err != nil {
   674  			f.Close()
   675  			os.Remove(f.Name())
   676  		}
   677  	}()
   678  
   679  	if _, err := f.Write(data); err != nil {
   680  		return err
   681  	}
   682  	if err := f.Close(); err != nil {
   683  		return err
   684  	}
   685  	if err := robustio.Rename(f.Name(), file); err != nil {
   686  		return err
   687  	}
   688  
   689  	if strings.HasSuffix(file, ".mod") {
   690  		rewriteVersionList(ctx, filepath.Dir(file))
   691  	}
   692  	return nil
   693  }
   694  
   695  // isErrReadOnlyFS reports whether err is a read-only filesystem error
   696  // (syscall.EROFS on Unix/wasip1, ERROR_NOT_SUPPORTED on Windows).
   697  func isErrReadOnlyFS(err error) bool {
   698  	switch runtime.GOOS {
   699  	case "plan9":
   700  		return false
   701  	case "windows":
   702  		const ERROR_NOT_SUPPORTED = 50
   703  		return errors.Is(err, syscall.Errno(ERROR_NOT_SUPPORTED))
   704  	case "wasip1":
   705  		const EROFS = 69
   706  		return errors.Is(err, syscall.Errno(EROFS))
   707  	default: // unix and js
   708  		const EROFS = 30
   709  		return errors.Is(err, syscall.Errno(EROFS))
   710  	}
   711  }
   712  
   713  // tempFile creates a new temporary file with given permission bits.
   714  func tempFile(ctx context.Context, dir, prefix string, perm fs.FileMode) (f *os.File, err error) {
   715  	for i := 0; i < 10000; i++ {
   716  		name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+".tmp")
   717  		f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
   718  		if os.IsExist(err) {
   719  			if ctx.Err() != nil {
   720  				return nil, ctx.Err()
   721  			}
   722  			continue
   723  		}
   724  		break
   725  	}
   726  	return
   727  }
   728  
   729  // rewriteVersionList rewrites the version list in dir
   730  // after a new *.mod file has been written.
   731  func rewriteVersionList(ctx context.Context, dir string) (err error) {
   732  	if filepath.Base(dir) != "@v" {
   733  		base.Fatalf("go: internal error: misuse of rewriteVersionList")
   734  	}
   735  
   736  	listFile := filepath.Join(dir, "list")
   737  
   738  	// Lock listfile when writing to it to try to avoid corruption to the file.
   739  	// Under rare circumstances, for instance, if the system loses power in the
   740  	// middle of a write it is possible for corrupt data to be written. This is
   741  	// not a problem for the go command itself, but may be an issue if the
   742  	// cache is being served by a GOPROXY HTTP server. This will be corrected
   743  	// the next time a new version of the module is fetched and the file is rewritten.
   744  	// TODO(matloob): golang.org/issue/43313 covers adding a go mod verify
   745  	// command that removes module versions that fail checksums. It should also
   746  	// remove list files that are detected to be corrupt.
   747  	f, err := lockedfile.Edit(listFile)
   748  	if err != nil {
   749  		return err
   750  	}
   751  	defer func() {
   752  		if cerr := f.Close(); cerr != nil && err == nil {
   753  			err = cerr
   754  		}
   755  	}()
   756  	infos, err := os.ReadDir(dir)
   757  	if err != nil {
   758  		return err
   759  	}
   760  	var list []string
   761  	for _, info := range infos {
   762  		// We look for *.mod files on the theory that if we can't supply
   763  		// the .mod file then there's no point in listing that version,
   764  		// since it's unusable. (We can have *.info without *.mod.)
   765  		// We don't require *.zip files on the theory that for code only
   766  		// involved in module graph construction, many *.zip files
   767  		// will never be requested.
   768  		name := info.Name()
   769  		if v, found := strings.CutSuffix(name, ".mod"); found {
   770  			if v != "" && module.CanonicalVersion(v) == v {
   771  				list = append(list, v)
   772  			}
   773  		}
   774  	}
   775  	semver.Sort(list)
   776  
   777  	var buf bytes.Buffer
   778  	for _, v := range list {
   779  		buf.WriteString(v)
   780  		buf.WriteString("\n")
   781  	}
   782  	if fi, err := f.Stat(); err == nil && int(fi.Size()) == buf.Len() {
   783  		old := make([]byte, buf.Len()+1)
   784  		if n, err := f.ReadAt(old, 0); err == io.EOF && n == buf.Len() && bytes.Equal(buf.Bytes(), old) {
   785  			return nil // No edit needed.
   786  		}
   787  	}
   788  	// Remove existing contents, so that when we truncate to the actual size it will zero-fill,
   789  	// and we will be able to detect (some) incomplete writes as files containing trailing NUL bytes.
   790  	if err := f.Truncate(0); err != nil {
   791  		return err
   792  	}
   793  	// Reserve the final size and zero-fill.
   794  	if err := f.Truncate(int64(buf.Len())); err != nil {
   795  		return err
   796  	}
   797  	// Write the actual contents. If this fails partway through,
   798  	// the remainder of the file should remain as zeroes.
   799  	if _, err := f.Write(buf.Bytes()); err != nil {
   800  		f.Truncate(0)
   801  		return err
   802  	}
   803  
   804  	return nil
   805  }
   806  
   807  var (
   808  	statCacheOnce sync.Once
   809  	statCacheErr  error
   810  
   811  	counterErrorsGOMODCACHEEntryRelative = counter.New("go/errors:gomodcache-entry-relative")
   812  )
   813  
   814  // checkCacheDir checks if the directory specified by GOMODCACHE exists. An
   815  // error is returned if it does not.
   816  func checkCacheDir(ctx context.Context) error {
   817  	if cfg.GOMODCACHE == "" {
   818  		// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
   819  		// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
   820  		return fmt.Errorf("module cache not found: neither GOMODCACHE nor GOPATH is set")
   821  	}
   822  	if !filepath.IsAbs(cfg.GOMODCACHE) {
   823  		counterErrorsGOMODCACHEEntryRelative.Inc()
   824  		return fmt.Errorf("GOMODCACHE entry is relative; must be absolute path: %q.\n", cfg.GOMODCACHE)
   825  	}
   826  
   827  	// os.Stat is slow on Windows, so we only call it once to prevent unnecessary
   828  	// I/O every time this function is called.
   829  	statCacheOnce.Do(func() {
   830  		fi, err := os.Stat(cfg.GOMODCACHE)
   831  		if err != nil {
   832  			if !os.IsNotExist(err) {
   833  				statCacheErr = fmt.Errorf("could not create module cache: %w", err)
   834  				return
   835  			}
   836  			if err := os.MkdirAll(cfg.GOMODCACHE, 0o777); err != nil {
   837  				statCacheErr = fmt.Errorf("could not create module cache: %w", err)
   838  				return
   839  			}
   840  			return
   841  		}
   842  		if !fi.IsDir() {
   843  			statCacheErr = fmt.Errorf("could not create module cache: %q is not a directory", cfg.GOMODCACHE)
   844  			return
   845  		}
   846  	})
   847  	return statCacheErr
   848  }
   849  

View as plain text