proxy.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"context"
     9  	"encoding/json"
    10  	"errors"
    11  	"fmt"
    12  	"io"
    13  	"io/fs"
    14  	"net/url"
    15  	pathpkg "path"
    16  	"path/filepath"
    17  	"strings"
    18  	"sync"
    19  	"time"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/modfetch/codehost"
    24  	"cmd/go/internal/web"
    25  
    26  	"golang.org/x/mod/module"
    27  	"golang.org/x/mod/semver"
    28  )
    29  
    30  var HelpGoproxy = &base.Command{
    31  	UsageLine: "goproxy",
    32  	Short:     "module proxy protocol",
    33  	Long: `
    34  A Go module proxy is any web server that can respond to GET requests for
    35  URLs of a specified form. The requests have no query parameters, so even
    36  a site serving from a fixed file system (including a file:/// URL)
    37  can be a module proxy.
    38  
    39  For details on the GOPROXY protocol, see
    40  https://go.dev/ref/mod#goproxy-protocol.
    41  `,
    42  }
    43  
    44  var proxyOnce struct {
    45  	sync.Once
    46  	list []proxySpec
    47  	err  error
    48  }
    49  
    50  type proxySpec struct {
    51  	// url is the proxy URL or one of "off", "direct", "noproxy".
    52  	url string
    53  
    54  	// fallBackOnError is true if a request should be attempted on the next proxy
    55  	// in the list after any error from this proxy. If fallBackOnError is false,
    56  	// the request will only be attempted on the next proxy if the error is
    57  	// equivalent to os.ErrNotFound, which is true for 404 and 410 responses.
    58  	fallBackOnError bool
    59  }
    60  
    61  func proxyList() ([]proxySpec, error) {
    62  	proxyOnce.Do(func() {
    63  		if cfg.GONOPROXY != "" && cfg.GOPROXY != "direct" {
    64  			proxyOnce.list = append(proxyOnce.list, proxySpec{url: "noproxy"})
    65  		}
    66  
    67  		goproxy := cfg.GOPROXY
    68  		for goproxy != "" {
    69  			var url string
    70  			fallBackOnError := false
    71  			if i := strings.IndexAny(goproxy, ",|"); i >= 0 {
    72  				url = goproxy[:i]
    73  				fallBackOnError = goproxy[i] == '|'
    74  				goproxy = goproxy[i+1:]
    75  			} else {
    76  				url = goproxy
    77  				goproxy = ""
    78  			}
    79  
    80  			url = strings.TrimSpace(url)
    81  			if url == "" {
    82  				continue
    83  			}
    84  			if url == "off" {
    85  				// "off" always fails hard, so can stop walking list.
    86  				proxyOnce.list = append(proxyOnce.list, proxySpec{url: "off"})
    87  				break
    88  			}
    89  			if url == "direct" {
    90  				proxyOnce.list = append(proxyOnce.list, proxySpec{url: "direct"})
    91  				// For now, "direct" is the end of the line. We may decide to add some
    92  				// sort of fallback behavior for them in the future, so ignore
    93  				// subsequent entries for forward-compatibility.
    94  				break
    95  			}
    96  
    97  			// Single-word tokens are reserved for built-in behaviors, and anything
    98  			// containing the string ":/" or matching an absolute file path must be a
    99  			// complete URL. For all other paths, implicitly add "https://".
   100  			if strings.ContainsAny(url, ".:/") && !strings.Contains(url, ":/") && !filepath.IsAbs(url) && !pathpkg.IsAbs(url) {
   101  				url = "https://" + url
   102  			}
   103  
   104  			// Check that newProxyRepo accepts the URL.
   105  			// It won't do anything with the path.
   106  			if _, err := newProxyRepo(url, "golang.org/x/text"); err != nil {
   107  				proxyOnce.err = err
   108  				return
   109  			}
   110  
   111  			proxyOnce.list = append(proxyOnce.list, proxySpec{
   112  				url:             url,
   113  				fallBackOnError: fallBackOnError,
   114  			})
   115  		}
   116  
   117  		if len(proxyOnce.list) == 0 ||
   118  			len(proxyOnce.list) == 1 && proxyOnce.list[0].url == "noproxy" {
   119  			// There were no proxies, other than the implicit "noproxy" added when
   120  			// GONOPROXY is set. This can happen if GOPROXY is a non-empty string
   121  			// like "," or " ".
   122  			proxyOnce.err = fmt.Errorf("GOPROXY list is not the empty string, but contains no entries")
   123  		}
   124  	})
   125  
   126  	return proxyOnce.list, proxyOnce.err
   127  }
   128  
   129  // TryProxies iterates f over each configured proxy (including "noproxy" and
   130  // "direct" if applicable) until f returns no error or until f returns an
   131  // error that is not equivalent to fs.ErrNotExist on a proxy configured
   132  // not to fall back on errors.
   133  //
   134  // TryProxies then returns that final error.
   135  //
   136  // If GOPROXY is set to "off", TryProxies invokes f once with the argument
   137  // "off".
   138  func TryProxies(f func(proxy string) error) error {
   139  	proxies, err := proxyList()
   140  	if err != nil {
   141  		return err
   142  	}
   143  	if len(proxies) == 0 {
   144  		panic("GOPROXY list is empty")
   145  	}
   146  
   147  	// We try to report the most helpful error to the user. "direct" and "noproxy"
   148  	// errors are best, followed by proxy errors other than ErrNotExist, followed
   149  	// by ErrNotExist.
   150  	//
   151  	// Note that errProxyOff, errNoproxy, and errUseProxy are equivalent to
   152  	// ErrNotExist. errUseProxy should only be returned if "noproxy" is the only
   153  	// proxy. errNoproxy should never be returned, since there should always be a
   154  	// more useful error from "noproxy" first.
   155  	const (
   156  		notExistRank = iota
   157  		proxyRank
   158  		directRank
   159  	)
   160  	var bestErr error
   161  	bestErrRank := notExistRank
   162  	for _, proxy := range proxies {
   163  		err := f(proxy.url)
   164  		if err == nil {
   165  			return nil
   166  		}
   167  		isNotExistErr := errors.Is(err, fs.ErrNotExist)
   168  
   169  		if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) {
   170  			bestErr = err
   171  			bestErrRank = directRank
   172  		} else if bestErrRank <= proxyRank && !isNotExistErr {
   173  			bestErr = err
   174  			bestErrRank = proxyRank
   175  		} else if bestErrRank == notExistRank {
   176  			bestErr = err
   177  		}
   178  
   179  		if !proxy.fallBackOnError && !isNotExistErr {
   180  			break
   181  		}
   182  	}
   183  	return bestErr
   184  }
   185  
   186  type proxyRepo struct {
   187  	url          *url.URL // The combined module proxy URL joined with the module path.
   188  	path         string   // The module path (unescaped).
   189  	redactedBase string   // The base module proxy URL in [url.URL.Redacted] form.
   190  
   191  	listLatestOnce sync.Once
   192  	listLatest     *RevInfo
   193  	listLatestErr  error
   194  }
   195  
   196  func newProxyRepo(baseURL, path string) (Repo, error) {
   197  	// Parse the base proxy URL.
   198  	base, err := url.Parse(baseURL)
   199  	if err != nil {
   200  		return nil, err
   201  	}
   202  	redactedBase := base.Redacted()
   203  	switch base.Scheme {
   204  	case "http", "https":
   205  		// ok
   206  	case "file":
   207  		if *base != (url.URL{Scheme: base.Scheme, Path: base.Path, RawPath: base.RawPath}) {
   208  			return nil, fmt.Errorf("invalid file:// proxy URL with non-path elements: %s", redactedBase)
   209  		}
   210  	case "":
   211  		return nil, fmt.Errorf("invalid proxy URL missing scheme: %s", redactedBase)
   212  	default:
   213  		return nil, fmt.Errorf("invalid proxy URL scheme (must be https, http, file): %s", redactedBase)
   214  	}
   215  
   216  	// Append the module path to the URL.
   217  	url := base
   218  	enc, err := module.EscapePath(path)
   219  	if err != nil {
   220  		return nil, err
   221  	}
   222  	url.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc
   223  	url.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc)
   224  
   225  	return &proxyRepo{url, path, redactedBase, sync.Once{}, nil, nil}, nil
   226  }
   227  
   228  func (p *proxyRepo) ModulePath() string {
   229  	return p.path
   230  }
   231  
   232  var errProxyReuse = fmt.Errorf("proxy does not support CheckReuse")
   233  
   234  func (p *proxyRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error {
   235  	return errProxyReuse
   236  }
   237  
   238  // versionError returns err wrapped in a ModuleError for p.path.
   239  func (p *proxyRepo) versionError(version string, err error) error {
   240  	if version != "" && version != module.CanonicalVersion(version) {
   241  		var iv *module.InvalidVersionError
   242  		if !errors.As(err, &iv) {
   243  			iv = &module.InvalidVersionError{
   244  				Version: version,
   245  				Pseudo:  module.IsPseudoVersion(version),
   246  				Err:     err,
   247  			}
   248  		}
   249  		return &module.ModuleError{
   250  			Path: p.path,
   251  			Err:  iv,
   252  		}
   253  	}
   254  
   255  	return &module.ModuleError{
   256  		Path:    p.path,
   257  		Version: version,
   258  		Err:     err,
   259  	}
   260  }
   261  
   262  func (p *proxyRepo) getBytes(ctx context.Context, path string) ([]byte, error) {
   263  	body, redactedURL, err := p.getBody(ctx, path)
   264  	if err != nil {
   265  		return nil, err
   266  	}
   267  	defer body.Close()
   268  
   269  	b, err := io.ReadAll(body)
   270  	if err != nil {
   271  		// net/http doesn't add context to Body read errors, so add it here.
   272  		// (See https://go.dev/issue/52727.)
   273  		return b, &url.Error{Op: "read", URL: redactedURL, Err: err}
   274  	}
   275  	return b, nil
   276  }
   277  
   278  func (p *proxyRepo) getBody(ctx context.Context, path string) (r io.ReadCloser, redactedURL string, err error) {
   279  	fullPath := pathpkg.Join(p.url.Path, path)
   280  
   281  	target := *p.url
   282  	target.Path = fullPath
   283  	target.RawPath = pathpkg.Join(target.RawPath, pathEscape(path))
   284  
   285  	resp, err := web.Get(web.DefaultSecurity, &target)
   286  	if err != nil {
   287  		return nil, "", err
   288  	}
   289  	if err := resp.Err(); err != nil {
   290  		resp.Body.Close()
   291  		return nil, "", err
   292  	}
   293  	return resp.Body, resp.URL, nil
   294  }
   295  
   296  func (p *proxyRepo) Versions(ctx context.Context, prefix string) (*Versions, error) {
   297  	data, err := p.getBytes(ctx, "@v/list")
   298  	if err != nil {
   299  		p.listLatestOnce.Do(func() {
   300  			p.listLatest, p.listLatestErr = nil, p.versionError("", err)
   301  		})
   302  		return nil, p.versionError("", err)
   303  	}
   304  	var list []string
   305  	allLine := strings.Split(string(data), "\n")
   306  	for _, line := range allLine {
   307  		f := strings.Fields(line)
   308  		if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) && !module.IsPseudoVersion(f[0]) {
   309  			list = append(list, f[0])
   310  		}
   311  	}
   312  	p.listLatestOnce.Do(func() {
   313  		p.listLatest, p.listLatestErr = p.latestFromList(ctx, allLine)
   314  	})
   315  	semver.Sort(list)
   316  	return &Versions{List: list}, nil
   317  }
   318  
   319  func (p *proxyRepo) latest(ctx context.Context) (*RevInfo, error) {
   320  	p.listLatestOnce.Do(func() {
   321  		data, err := p.getBytes(ctx, "@v/list")
   322  		if err != nil {
   323  			p.listLatestErr = p.versionError("", err)
   324  			return
   325  		}
   326  		list := strings.Split(string(data), "\n")
   327  		p.listLatest, p.listLatestErr = p.latestFromList(ctx, list)
   328  	})
   329  	return p.listLatest, p.listLatestErr
   330  }
   331  
   332  func (p *proxyRepo) latestFromList(ctx context.Context, allLine []string) (*RevInfo, error) {
   333  	var (
   334  		bestTime    time.Time
   335  		bestVersion string
   336  	)
   337  	for _, line := range allLine {
   338  		f := strings.Fields(line)
   339  		if len(f) >= 1 && semver.IsValid(f[0]) {
   340  			// If the proxy includes timestamps, prefer the timestamp it reports.
   341  			// Otherwise, derive the timestamp from the pseudo-version.
   342  			var (
   343  				ft time.Time
   344  			)
   345  			if len(f) >= 2 {
   346  				ft, _ = time.Parse(time.RFC3339, f[1])
   347  			} else if module.IsPseudoVersion(f[0]) {
   348  				ft, _ = module.PseudoVersionTime(f[0])
   349  			} else {
   350  				// Repo.Latest promises that this method is only called where there are
   351  				// no tagged versions. Ignore any tagged versions that were added in the
   352  				// meantime.
   353  				continue
   354  			}
   355  			if bestTime.Before(ft) {
   356  				bestTime = ft
   357  				bestVersion = f[0]
   358  			}
   359  		}
   360  	}
   361  	if bestVersion == "" {
   362  		return nil, p.versionError("", codehost.ErrNoCommits)
   363  	}
   364  
   365  	// Call Stat to get all the other fields, including Origin information.
   366  	return p.Stat(ctx, bestVersion)
   367  }
   368  
   369  func (p *proxyRepo) Stat(ctx context.Context, rev string) (*RevInfo, error) {
   370  	encRev, err := module.EscapeVersion(rev)
   371  	if err != nil {
   372  		return nil, p.versionError(rev, err)
   373  	}
   374  	data, err := p.getBytes(ctx, "@v/"+encRev+".info")
   375  	if err != nil {
   376  		return nil, p.versionError(rev, err)
   377  	}
   378  	info := new(RevInfo)
   379  	if err := json.Unmarshal(data, info); err != nil {
   380  		return nil, p.versionError(rev, fmt.Errorf("invalid response from proxy %q: %w", p.redactedBase, err))
   381  	}
   382  	if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil {
   383  		// If we request a correct, appropriate version for the module path, the
   384  		// proxy must return either exactly that version or an error — not some
   385  		// arbitrary other version.
   386  		return nil, p.versionError(rev, fmt.Errorf("proxy returned info for version %s instead of requested version", info.Version))
   387  	}
   388  	return info, nil
   389  }
   390  
   391  func (p *proxyRepo) Latest(ctx context.Context) (*RevInfo, error) {
   392  	data, err := p.getBytes(ctx, "@latest")
   393  	if err != nil {
   394  		if !errors.Is(err, fs.ErrNotExist) {
   395  			return nil, p.versionError("", err)
   396  		}
   397  		return p.latest(ctx)
   398  	}
   399  	info := new(RevInfo)
   400  	if err := json.Unmarshal(data, info); err != nil {
   401  		return nil, p.versionError("", fmt.Errorf("invalid response from proxy %q: %w", p.redactedBase, err))
   402  	}
   403  	return info, nil
   404  }
   405  
   406  func (p *proxyRepo) GoMod(ctx context.Context, version string) ([]byte, error) {
   407  	if version != module.CanonicalVersion(version) {
   408  		return nil, p.versionError(version, fmt.Errorf("internal error: version passed to GoMod is not canonical"))
   409  	}
   410  
   411  	encVer, err := module.EscapeVersion(version)
   412  	if err != nil {
   413  		return nil, p.versionError(version, err)
   414  	}
   415  	data, err := p.getBytes(ctx, "@v/"+encVer+".mod")
   416  	if err != nil {
   417  		return nil, p.versionError(version, err)
   418  	}
   419  	return data, nil
   420  }
   421  
   422  func (p *proxyRepo) Zip(ctx context.Context, dst io.Writer, version string) error {
   423  	if version != module.CanonicalVersion(version) {
   424  		return p.versionError(version, fmt.Errorf("internal error: version passed to Zip is not canonical"))
   425  	}
   426  
   427  	encVer, err := module.EscapeVersion(version)
   428  	if err != nil {
   429  		return p.versionError(version, err)
   430  	}
   431  	path := "@v/" + encVer + ".zip"
   432  	body, redactedURL, err := p.getBody(ctx, path)
   433  	if err != nil {
   434  		return p.versionError(version, err)
   435  	}
   436  	defer body.Close()
   437  
   438  	lr := &io.LimitedReader{R: body, N: codehost.MaxZipFile + 1}
   439  	if _, err := io.Copy(dst, lr); err != nil {
   440  		// net/http doesn't add context to Body read errors, so add it here.
   441  		// (See https://go.dev/issue/52727.)
   442  		err = &url.Error{Op: "read", URL: redactedURL, Err: err}
   443  		return p.versionError(version, err)
   444  	}
   445  	if lr.N <= 0 {
   446  		return p.versionError(version, fmt.Errorf("downloaded zip file too large"))
   447  	}
   448  	return nil
   449  }
   450  
   451  // pathEscape escapes s so it can be used in a path.
   452  // That is, it escapes things like ? and # (which really shouldn't appear anyway).
   453  // It does not escape / to %2F: our REST API is designed so that / can be left as is.
   454  func pathEscape(s string) string {
   455  	return strings.ReplaceAll(url.PathEscape(s), "%2F", "/")
   456  }
   457
View as plain text