ci: build release targets on independent matrix runners

This commit is contained in:
ranxi2001
2026-09-20 16:36:24 +08:00
parent acdf3c54da
commit 0892ef3a9a
7 changed files with 627 additions and 115 deletions
+30
View File
@@ -0,0 +1,30 @@
# Release matrix
The release workflow builds the frontend once, then runs each configured Go target on its own Linux runner. `CGO_ENABLED=0` permits cross-compilation. The target matrix is read from `.goreleaser.yaml`, including its exclusions; simple releases select Linux amd64 only.
Each build uses GoReleaser OSS in snapshot mode with the selected release version and one target. Archive naming, bundled files, Go flags and release templates remain in the existing GoReleaser configurations. Every archive is accompanied by its source commit, target, version and SHA256. The publishing job verifies the complete matrix before building images or publishing. It uses GoReleaser's `extra_files` support to publish existing archives and checksums, with builds disabled. No Pro license is needed.
Go caches are isolated by target and refreshed on each source commit, with fallback to the preceding target cache. Matrix jobs upload uniquely named artifacts. The publishing job extracts only the regular Linux binary from each verified archive and restores its executable permission before constructing Docker contexts. QEMU remains limited to runtime-image instructions. DockerHub images are omitted when its credentials are absent; GHCR is always retained. Simple mode still publishes only the amd64 GHCR image and the simple release description.
All build jobs use the commit resolved by `prepare`, including a manual release's selected tag. The workflow serializes release runs to prevent simultaneous updates to moving image tags.
## Validate without publication
From a branch containing this workflow:
```bash
gh workflow run release.yml --ref <branch> \
-f tag=<branch> -f dry_run=true -f simple_release=false
```
A dry run builds all selected archives and both runtime images, verifies artifact provenance and produces the final checksum file. It exports images locally as OCI archives instead of pushing them. It skips registry logins, GitHub Release publication, DockerHub description updates, Telegram notifications and VERSION synchronization. Test the simple path separately with `simple_release=true`.
Dry-run artifacts are available in the Actions run, including `release-dry-run-report`. Compare job start/end times, GoReleaser's build duration and cache restore results. Do not present an initial cold-cache run as a warmed-cache benchmark; publishing network time is not measured by dry runs.
Helper checks:
```bash
python -m pip install -r .github/release-tools/requirements-release.txt
python -m unittest discover -s .github/release -p 'test_release_matrix.py'
bash -n .github/release-tools/release-images.sh
```
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -euo pipefail
: "${RELEASE_VERSION:?}" "${RELEASE_SHA:?}" "${GITHUB_REPOSITORY:?}" "${RUNNER_TEMP:?}"
owner=${GITHUB_REPOSITORY%%/*}
registries=("ghcr.io/${owner,,}/sub2api")
if [[ ${SIMPLE_RELEASE:-false} != true && ${DOCKERHUB_USERNAME:-skip} != skip ]]; then
registries+=("${DOCKERHUB_USERNAME}/sub2api")
fi
arches=(amd64 arm64)
if [[ ${SIMPLE_RELEASE:-false} == true ]]; then arches=(amd64); fi
for arch in "${arches[@]}"; do
args=(--platform "linux/$arch" --file ".release-context/$arch/Dockerfile"
--label "org.opencontainers.image.version=$RELEASE_VERSION"
--label "org.opencontainers.image.revision=$RELEASE_SHA"
--label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY")
for registry in "${registries[@]}"; do
args+=(--tag "$registry:$RELEASE_VERSION-$arch")
if [[ ${SIMPLE_RELEASE:-false} == true ]]; then
args+=(--tag "$registry:$RELEASE_VERSION" --tag "$registry:latest")
fi
done
if [[ ${DRY_RUN:-false} == true ]]; then
args+=(--output "type=oci,dest=$RUNNER_TEMP/sub2api-$arch.oci.tar")
else
args+=(--push)
fi
docker buildx build "${args[@]}" ".release-context/$arch"
done
if [[ ${DRY_RUN:-false} != true && ${SIMPLE_RELEASE:-false} != true ]]; then
major=${RELEASE_VERSION%%.*}
minor=${RELEASE_VERSION#*.}; minor=${minor%%.*}
for registry in "${registries[@]}"; do
docker buildx imagetools create \
--tag "$registry:$RELEASE_VERSION" --tag "$registry:latest" \
--tag "$registry:$major.$minor" --tag "$registry:$major" \
"$registry:$RELEASE_VERSION-amd64" "$registry:$RELEASE_VERSION-arm64"
done
fi
+189
View File
@@ -0,0 +1,189 @@
#!/usr/bin/env python3
"""Split GoReleaser builds across runners without requiring GoReleaser Pro."""
import argparse
import hashlib
import itertools
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import tarfile
from datetime import datetime, timezone
import yaml
FULL_CONFIG = Path('.goreleaser.yaml')
SIMPLE_CONFIG = Path('.goreleaser.simple.yaml')
VERSION_FILE = Path('backend/cmd/server/VERSION')
VERSION_RE = re.compile(r'\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?')
def config(simple=False):
return yaml.safe_load((SIMPLE_CONFIG if simple else FULL_CONFIG).read_text())
def targets(simple=False):
build = config()['builds'][0]
result = []
for goos, goarch in itertools.product(build['goos'], build['goarch']):
item = {'goos': goos, 'goarch': goarch}
if any(all(item.get(k) == v for k, v in rule.items()) for rule in build.get('ignore', [])):
continue
if not simple or item == {'goos': 'linux', 'goarch': 'amd64'}:
result.append(item)
if not result:
raise ValueError('empty release target matrix')
return result
def archive_name(version, target):
if not VERSION_RE.fullmatch(version) or target not in targets():
raise ValueError('invalid release version or target')
suffix = 'zip' if target['goos'] == 'windows' else 'tar.gz'
return f"sub2api_{version}_{target['goos']}_{target['goarch']}.{suffix}"
def sha256(path):
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
def plan(args):
sha = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()
if args.dry_run:
version = VERSION_FILE.read_text().strip()
tag = 'v' + version
else:
tag = args.ref
version = tag.removeprefix('v')
if not tag.startswith('v') or not VERSION_RE.fullmatch(version):
raise ValueError('publishing requires a v-prefixed release version tag')
tagged_sha = subprocess.check_output(['git', 'rev-parse', '--verify', f'refs/tags/{tag}^{{commit}}'], text=True).strip()
if sha != tagged_sha:
raise ValueError('checkout does not match the selected release tag')
if not VERSION_RE.fullmatch(version):
raise ValueError('invalid VERSION')
VERSION_FILE.write_text(version + '\n')
result = {'sha': sha, 'tag': tag, 'version': version,
'owner_lower': os.environ.get('GITHUB_REPOSITORY_OWNER', '').lower(),
'simple': str(args.simple).lower(), 'dry_run': str(args.dry_run).lower(),
'date': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
'matrix': json.dumps({'include': targets(args.simple)}, separators=(',', ':'))}
with Path(os.environ['GITHUB_OUTPUT']).open('a') as output:
for key, value in result.items():
output.write(f'{key}={value}\n')
def generate_config(args):
data = config(args.simple if args.mode == 'publish' else False)
data['snapshot'] = {'version_template': '{{ .Env.RELEASE_VERSION }}'}
data['dockers'] = []
data['docker_manifests'] = []
if args.mode == 'build':
target = {'goos': args.goos, 'goarch': args.goarch}
if target not in targets():
raise ValueError('unsupported build target')
for build in data['builds']:
build['goos'], build['goarch'], build['ignore'] = [args.goos], [args.goarch], []
build['ldflags'] = [re.sub(r'{{\s*\.Date\s*}}', '{{ .Env.RELEASE_DATE }}', flag)
for flag in build.get('ldflags', [])]
else:
# Artifacts are supplied through the OSS extra_files mechanism. No build
# is repeated on the publishing runner, and release templates stay intact.
data['before'] = {'hooks': []}
data['builds'] = [{'id': 'sub2api', 'skip': True}]
data['archives'] = []
extra = [{'glob': 'release-input/sub2api_*.tar.gz'}, {'glob': 'release-input/sub2api_*.zip'}]
if args.simple:
data['checksum'] = {'disable': True}
else:
data['release']['extra_files'] = extra
data['checksum'] = {'name_template': 'checksums.txt', 'algorithm': 'sha256', 'extra_files': extra}
Path(args.output).write_text(yaml.safe_dump(data, sort_keys=False, allow_unicode=True))
def collect(args):
target = {'goos': args.goos, 'goarch': args.goarch}
name = archive_name(args.version, target)
source = Path('dist') / name
checksums = {line.split()[1].lstrip('*'): line.split()[0] for line in Path('dist/checksums.txt').read_text().splitlines()}
digest = sha256(source)
if checksums.get(name) != digest:
raise ValueError('archive does not match the build checksum')
output = Path(args.output)
output.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, output / name)
manifest = {'sha': args.sha, 'version': args.version, 'target': target, 'archive': name, 'sha256': digest}
(output / f"manifest-{args.goos}-{args.goarch}.json").write_text(json.dumps(manifest) + '\n')
def verify(args):
directory = Path(args.input)
expected = set()
for target in targets(args.simple):
name = archive_name(args.version, target)
manifest_name = f"manifest-{target['goos']}-{target['goarch']}.json"
expected.update((name, manifest_name))
manifest = json.loads((directory / manifest_name).read_text())
if manifest != {'sha': args.sha, 'version': args.version, 'target': target,
'archive': name, 'sha256': sha256(directory / name)}:
raise ValueError(f'build provenance or checksum mismatch: {name}')
if {p.name for p in directory.iterdir()} != expected:
raise ValueError('missing or unexpected release artifacts')
def contexts(args):
verify(args)
for target in targets(args.simple):
if target['goos'] != 'linux':
continue
dest = Path(args.output) / target['goarch']
dest.mkdir(parents=True, exist_ok=True)
with tarfile.open(Path(args.input) / archive_name(args.version, target), 'r:gz') as archive:
members = [member for member in archive.getmembers() if member.name in ('sub2api', './sub2api')]
if len(members) != 1 or not members[0].isfile():
raise ValueError('archive must contain one regular sub2api binary')
with archive.extractfile(members[0]) as source, (dest / 'sub2api').open('wb') as output:
shutil.copyfileobj(source, output)
(dest / 'sub2api').chmod(0o755)
shutil.copy2('Dockerfile.goreleaser', dest / 'Dockerfile')
(dest / 'deploy').mkdir(exist_ok=True)
shutil.copy2('deploy/docker-entrypoint.sh', dest / 'deploy/docker-entrypoint.sh')
shutil.copytree('backend/resources', dest / 'backend/resources', dirs_exist_ok=True)
def main():
parser = argparse.ArgumentParser(description=__doc__)
commands = parser.add_subparsers(dest='command', required=True)
p = commands.add_parser('plan')
p.add_argument('--ref', required=True)
p.add_argument('--simple', action='store_true')
p.add_argument('--dry-run', action='store_true')
p.set_defaults(run=plan)
p = commands.add_parser('config')
p.add_argument('mode', choices=['build', 'publish'])
p.add_argument('--goos')
p.add_argument('--goarch')
p.add_argument('--simple', action='store_true')
p.add_argument('--output', required=True)
p.set_defaults(run=generate_config)
p = commands.add_parser('collect')
for arg in ('version', 'sha', 'goos', 'goarch', 'output'):
p.add_argument('--' + arg, required=True)
p.set_defaults(run=collect)
for command, handler in [('verify', verify), ('contexts', contexts)]:
p = commands.add_parser(command)
for arg in ('version', 'sha', 'input'):
p.add_argument('--' + arg, required=True)
p.add_argument('--simple', action='store_true')
if command == 'contexts':
p.add_argument('--output', required=True)
p.set_defaults(run=handler)
args = parser.parse_args()
args.run(args)
if __name__ == '__main__':
main()
@@ -0,0 +1 @@
PyYAML==6.0.3
@@ -0,0 +1,163 @@
import argparse
import hashlib
import importlib.util
import io
import json
import os
from pathlib import Path
import shutil
import subprocess
import tarfile
import tempfile
import unittest
from unittest.mock import patch
import yaml
ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location('release_matrix', Path(__file__).with_name('release_matrix.py'))
release = importlib.util.module_from_spec(spec)
spec.loader.exec_module(release)
class ReleaseMatrixTest(unittest.TestCase):
def setUp(self):
self.previous = Path.cwd()
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
os.chdir(self.temp.name)
self.addCleanup(os.chdir, self.previous)
for name in ('.goreleaser.yaml', '.goreleaser.simple.yaml'):
shutil.copyfile(ROOT / name, name)
Path('backend/cmd/server').mkdir(parents=True)
release.VERSION_FILE.write_text('9.8.7\n')
def fixture_artifacts(self, simple=False):
directory = Path('release-input')
directory.mkdir()
for target in release.targets(simple):
name = release.archive_name('9.8.7', target)
archive = directory / name
if target['goos'] == 'linux':
with tarfile.open(archive, 'w:gz') as out:
info = tarfile.TarInfo('sub2api')
info.size = 7
info.mode = 0o755
out.addfile(info, io.BytesIO(b'fixture'))
else:
archive.write_bytes(b'fixture archive')
metadata = {'version': '9.8.7', 'sha': 'a' * 40, 'target': target,
'archive': name, 'sha256': release.sha256(archive)}
(directory / f"manifest-{target['goos']}-{target['goarch']}.json").write_text(json.dumps(metadata))
return argparse.Namespace(input='release-input', version='9.8.7', sha='a' * 40, simple=simple, output='contexts')
def test_full_and_simple_matrix_match_existing_targets(self):
full = release.targets()
self.assertEqual(len(full), 5)
self.assertNotIn({'goos': 'windows', 'goarch': 'arm64'}, full)
self.assertEqual(release.targets(True), [{'goos': 'linux', 'goarch': 'amd64'}])
def test_leaf_keeps_packaging_and_selects_only_one_target(self):
original = release.config()
release.generate_config(argparse.Namespace(mode='build', simple=False, goos='darwin', goarch='arm64', output='leaf.yaml'))
leaf = yaml.safe_load(Path('leaf.yaml').read_text())
self.assertEqual(leaf['builds'][0]['goos'], ['darwin'])
self.assertEqual(leaf['builds'][0]['goarch'], ['arm64'])
self.assertEqual(leaf['builds'][0]['ignore'], [])
self.assertEqual(leaf['archives'], original['archives'])
self.assertEqual(leaf['release'], original['release'])
self.assertFalse(leaf['dockers'])
self.assertIn('{{ .Env.RELEASE_DATE }}', '\n'.join(leaf['builds'][0]['ldflags']))
def test_publication_config_has_no_compilation_or_docker_work(self):
for simple in (False, True):
with self.subTest(simple=simple):
original = release.config(simple)
release.generate_config(argparse.Namespace(mode='publish', simple=simple, output='publisher.yaml'))
data = yaml.safe_load(Path('publisher.yaml').read_text())
self.assertTrue(data['builds'][0]['skip'])
self.assertFalse(data['archives'])
self.assertFalse(data['dockers'])
self.assertEqual(data['release']['header'], original['release']['header'])
self.assertEqual(data['release']['footer'], original['release']['footer'])
if simple:
self.assertTrue(data['checksum']['disable'])
self.assertTrue(data['release']['skip_upload'])
else:
self.assertEqual(data['checksum']['extra_files'], data['release']['extra_files'])
def test_collect_and_verify_hash_and_source_binding(self):
args = self.fixture_artifacts()
release.verify(args)
file = next(Path(args.input).glob('*.tar.gz'))
file.write_bytes(b'corrupted')
with self.assertRaisesRegex(ValueError, 'checksum mismatch'):
release.verify(args)
def test_missing_extra_and_wrong_commit_artifacts_are_rejected(self):
args = self.fixture_artifacts(True)
args.sha = 'b' * 40
with self.assertRaises(ValueError):
release.verify(args)
args.sha = 'a' * 40
Path('release-input/unexpected').write_text('not an asset')
with self.assertRaises(ValueError):
release.verify(args)
Path('release-input/unexpected').unlink()
next(Path('release-input').glob('*.tar.gz')).unlink()
with self.assertRaises(FileNotFoundError):
release.verify(args)
def test_linux_context_preserves_binary_executable_mode(self):
args = self.fixture_artifacts()
Path('Dockerfile.goreleaser').write_text('FROM scratch\nCOPY sub2api /sub2api\n')
Path('deploy').mkdir()
Path('deploy/docker-entrypoint.sh').write_text('#!/bin/sh\nexec /app/sub2api\n')
Path('backend/resources').mkdir()
Path('backend/resources/data').write_text('fixture')
release.contexts(args)
for arch in ('amd64', 'arm64'):
binary = Path('contexts') / arch / 'sub2api'
self.assertEqual(binary.read_bytes(), b'fixture')
self.assertEqual(binary.stat().st_mode & 0o777, 0o755)
def test_plan_requires_a_tag_for_publication(self):
args = argparse.Namespace(ref='main', dry_run=False, simple=False)
with patch.object(subprocess, 'check_output', return_value='a' * 40 + '\n'):
with self.assertRaisesRegex(ValueError, 'version tag'):
release.plan(args)
args.ref = 'v9.8.7'
with patch.object(subprocess, 'check_output', side_effect=['a' * 40 + '\n', 'b' * 40 + '\n']):
with self.assertRaisesRegex(ValueError, 'does not match'):
release.plan(args)
def test_dry_run_plan_resolves_matrix_without_a_new_tag(self):
with patch.dict(os.environ, {'GITHUB_OUTPUT': 'outputs', 'GITHUB_REPOSITORY_OWNER': 'ExampleOwner'}), patch.object(subprocess, 'check_output', return_value='a' * 40 + '\n'):
release.plan(argparse.Namespace(ref='feature/matrix', dry_run=True, simple=False))
output = dict(line.split('=', 1) for line in Path('outputs').read_text().splitlines())
self.assertEqual(output['dry_run'], 'true')
self.assertEqual(output['owner_lower'], 'exampleowner')
self.assertEqual(len(json.loads(output['matrix'])['include']), 5)
def test_docker_commands_do_not_publish_during_dry_run(self):
fake_bin = Path('bin')
fake_bin.mkdir()
docker = fake_bin / 'docker'
docker.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$DOCKER_LOG"\n')
docker.chmod(0o755)
env = {**os.environ, 'PATH': str(fake_bin.resolve()) + os.pathsep + os.environ['PATH'],
'DOCKER_LOG': str(Path('docker.log').resolve()), 'RUNNER_TEMP': self.temp.name,
'RELEASE_VERSION': '9.8.7', 'RELEASE_SHA': 'a' * 40, 'GITHUB_REPOSITORY': 'ExampleOwner/sub2api',
'DRY_RUN': 'true', 'SIMPLE_RELEASE': 'false', 'DOCKERHUB_USERNAME': 'skip'}
subprocess.run(['bash', str(ROOT / '.github/release-tools/release-images.sh')], env=env, check=True)
log = Path('docker.log').read_text()
self.assertEqual(log.count('buildx build'), 2)
self.assertIn('linux/arm64', log)
self.assertNotIn('--push', log)
self.assertNotIn('imagetools', log)
self.assertNotIn('skip/sub2api', log)
self.assertIn('ghcr.io/exampleowner/sub2api', log)
if __name__ == '__main__':
unittest.main()