teleport/lib/versioncontrol/github/github.go

/*
Copyright 2022 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package github

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/gravitational/teleport"
	vc "github.com/gravitational/teleport/lib/versioncontrol"

	"github.com/gravitational/trace"
	"github.com/sirupsen/logrus"

	"golang.org/x/mod/semver"
)

// NOTE: when making modifications to package, make sure to run tests with
// `TEST_GITHUB_API=yes`. this will enable some additional tests that are not
// run as part of normal CI.

var log = logrus.WithFields(logrus.Fields{
	trace.Component: teleport.ComponentVersionControl,
})

// Visit uses the supplied visitor to aggregate release info from the github releases api.
func Visit(visitor *vc.Visitor) error {
	return visit(Iterator{}, visitor)
}

// visit is the business logic of Visit, broken out for testing purposes.
func visit(iter Iterator, visitor *vc.Visitor) error {
	if !visitor.Current.Ok() {
		return trace.BadParameter("cannot scrape github releases, invalid 'current' target: %+v", visitor.Current)
	}

	// we only care about newer releaseas, so set halting point to the version of 'current'
	iter.halt = visitor.Current.Version()

	for iter.Next() {
		for _, target := range iter.Page() {
			visitor.Visit(target)
		}
	}
	return trace.Wrap(iter.Error())
}

// defaultHaltingPoint represents the default cutoff for the iterator. this value
// is expanded/truncated to `<major>.<minor>` so e.g. `v1` would halt iteration on the
// first `v1.0.X` release observed, and `v1.2.3` would halt iteration on the first
// `v1.2.X` release observed.
const defaultHaltingPoint = "v8"

const defaultPageSize = 100

// Release represents a github release.
type Release struct {
	// Version is the semver version from the git tag of the release.
	Version string

	// OtherLabels are the labels extracted from the release notes.
	OtherLabels map[string]string

	// TODO(fspmarshall): get rid of this type in favor of a common attribute-based
	// release representation based in lib/versioncontrol.
}

// release is the representation of a release returned
// by the github API.
type release struct {
	TagName string `json:"tag_name"`
	Body    string `json:"body"`
}

// getter loads a page of releases. we override the standard page loading logic
// via this type in order to avoid issues with rate-limiting of the unauthenticated
// releases API.
type getter func(n, size int) ([]release, error)

// getPage loads the specified page from the unauthenticated github releases API
// using the provided http client.
func getPage(clt http.Client, n, size int) ([]release, error) {
	if n == 0 {
		return nil, trace.BadParameter("unspecified page number for teleport/releases (this is a bug)")
	}
	if size == 0 {
		return nil, trace.BadParameter("unspecified page size for teleport/releases (this is a bug)")
	}

	// get page from the github api. see https://docs.github.com/rest/releases/releases for details.
	res, err := clt.Get(fmt.Sprintf("https://api.github.com/repos/gravitational/teleport/releases?page=%d&per_page=%d", n, size))
	if err != nil {
		return nil, trace.Errorf("failed to get teleport/releases: %v", err)
	}

	body, err := io.ReadAll(res.Body)
	res.Body.Close()

	if res.StatusCode > 299 {
		// attempt to extract pretty error message
		var gherr ghError
		if json.Unmarshal(body, &gherr) == nil {
			if gherr.Message != "" {
				return nil, trace.Errorf("teleport/releases request failed with message: %s (%d)", gherr.Message, res.StatusCode)
			}
		}
		// fallback to status code and quoted body
		return nil, trace.Errorf("teleport/releases request failed with code=%d, body=%q", res.StatusCode, body)
	}

	// handle error from rsp body read
	if err != nil {
		return nil, trace.Wrap(err)
	}

	return parsePage(body)
}

// deserialization helper for github api errors
type ghError struct {
	Message string `json:"message"`
}

func parsePage(page []byte) ([]release, error) {
	var releases []release
	if err := json.Unmarshal(page, &releases); err != nil {
		return nil, trace.Errorf("failed to unmarshal releases page: %v", err)
	}
	return releases, nil
}

// Iterator allows lazy consumption of github release pages. Not safe for
// concurrent use. Always check for err after iteration, even if one or more
// pages were loaded successfully. If a release being added concurrently with
// iteration may cause duplicate releases to be observed.
type Iterator struct {
	getPage getter
	n       int
	size    int
	page    []vc.Target
	halt    string
	done    bool
	err     error
}

func (i *Iterator) setDefaults() {
	if i.getPage == nil {
		var clt http.Client
		i.getPage = func(n, size int) ([]release, error) {
			return getPage(clt, n, size)
		}
	}

	if i.size == 0 {
		i.size = defaultPageSize
	}

	if i.halt == "" {
		i.halt = defaultHaltingPoint
	}
}

// Next attempts to load the next page.
func (i *Iterator) Next() bool {
	i.setDefaults()
	if i.done {
		return false
	}
	i.n++
	var page []release
	page, i.err = i.getPage(i.n, i.size)
	if len(page) < i.size {
		// check unfiltered page size for halt condition
		i.done = true
	}
	i.page = make([]vc.Target, 0, len(page))
	for _, r := range page {
		if !semver.IsValid(r.TagName) {
			log.Debugf("Skipping non-semver release tag: %q\n", r.TagName)
			continue
		}
		labels := parseReleaseNoteLabels(r.Body)
		labels[vc.LabelVersion] = r.TagName
		i.page = append(i.page, labels)

		// only match `<major>.<minor>` when finding halt point. theoretically
		// unnecessary, but this feels a bit less brittle than halting on a specific
		// tag, and is more consistent than using a gt/lt rule, since we occasionally
		// release patches for very old versions.
		if semver.MajorMinor(r.TagName) == semver.MajorMinor(i.halt) {
			// set 'done' so that this is the last page we end up processing
			i.done = true
		}
	}
	return i.err == nil && len(page) != 0
}

// Page loads the current page. Must not be called until after Next() has been
// called. Subsequent calls between calls to Next() return the same value.
func (i *Iterator) Page() []vc.Target {
	return i.page
}

// Error checks if an error occurred during iteration.
func (i *Iterator) Error() error {
	return i.err
}

// parseReleaseNoteLabels attempts to extract labels from github release notes.
// Invalid values are skipped/ignored in order to ensure that future extensions of
// the label format can be reasonably backwards-compatible. Labels are encoded
// in the form '\nlabels:<key>=<val>[,<key>=<val>]'. Characters are normalize to
// lowercase and spaces between keypairs are stripped. See TestLabelParse for
// examples.
func parseReleaseNoteLabels(notes string) map[string]string {
	const labelPrefix = "labels:"

	labels := make(map[string]string)

	scanner := bufio.NewScanner(strings.NewReader(notes))

	for scanner.Scan() {
		l := strings.ToLower(strings.TrimSpace(scanner.Text()))
		if !strings.HasPrefix(l, labelPrefix) {
			continue
		}
		l = strings.TrimPrefix(l, labelPrefix)
		for _, kv := range strings.Split(l, ",") {
			if !strings.Contains(kv, "=") {
				log.Debugf("Skipping invalid release label keypair: %q", kv)
				continue
			}

			parts := strings.SplitN(kv, "=", 2)
			if len(parts) != 2 {
				log.Debugf("Skipping invalid release label keypair: %q", kv)
				continue
			}

			key := strings.TrimSpace(parts[0])
			val := strings.TrimSpace(parts[1])

			if !vc.IsValidTargetKey(key) || !vc.IsValidTargetVal(val) {
				log.Debugf("Skipping invalid release label keypair: %q", kv)
				// NOTE: we are skipping invalid keypairs for github release scraping
				// because github releases are using a generally simplistic release representation.
				// The TUF implementation will not skip invalid keypairs, preferring to
				// preserve them in the backend in order to ensure that backend representations
				// are forward-compatible with future versions auth version.
				continue
			}

			labels[key] = val
		}
	}

	return labels
}