From 20213eb2f268178ecec8c9bd1ec40005301c7328 Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Thu, 10 Sep 2026 16:35:50 +0800 Subject: [PATCH] fix: bind release staging to exact inputs and license metadata --- .github/workflows/managed-wren-release-ci.yml | 3 + .github/workflows/managed-wren-runtime.yml | 31 ++++++--- .../scripts/managed-wren-release.test.mjs | 63 ++++++++++++++++++- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/.github/workflows/managed-wren-release-ci.yml b/.github/workflows/managed-wren-release-ci.yml index aa7e204e8..90181ea03 100644 --- a/.github/workflows/managed-wren-release-ci.yml +++ b/.github/workflows/managed-wren-release-ci.yml @@ -30,6 +30,9 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 24 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' - name: Check standalone helper syntax run: node --check apps/genbi/scripts/managed-wren-release.mjs - name: Test exact inventory, approval and workflow boundaries diff --git a/.github/workflows/managed-wren-runtime.yml b/.github/workflows/managed-wren-runtime.yml index 3bbd5e7d7..9c42c6885 100644 --- a/.github/workflows/managed-wren-runtime.yml +++ b/.github/workflows/managed-wren-runtime.yml @@ -37,26 +37,39 @@ jobs: env: RUNTIME_TAG: ${{ inputs.runtime_tag }} run: | + set -euo pipefail + PYTHON_URL="$(node -p "require('./apps/genbi/managed-wren/release-inputs.json').python.url")" + PYTHON_SHA256="$(node -p "require('./apps/genbi/managed-wren/release-inputs.json').python.sha256")" + WRENAI_VERSION="$(node -p "require('./apps/genbi/managed-wren/release-inputs.json').wrenai.version")" + export PYTHON_SHA256 mkdir -p release/wheels release/runtime - python -m pip download --only-binary=:all: --platform macosx_11_0_arm64 --python-version 3.11 --implementation cp --dest release/wheels 'wrenai==0.13.0' + python -m pip download --only-binary=:all: --platform macosx_11_0_arm64 --python-version 3.11 --implementation cp --dest release/wheels "wrenai==$WRENAI_VERSION" python - <<'PY' - import hashlib, json, pathlib, urllib.request, zipfile + import email.parser, hashlib, json, pathlib, urllib.request, zipfile rows = [] for wheel in sorted(pathlib.Path('release/wheels').glob('*.whl')): with zipfile.ZipFile(wheel) as z: - meta = next(n for n in z.namelist() if n.endswith('.dist-info/METADATA')) - values = dict(line.split(': ', 1) for line in z.read(meta).decode('utf8', 'replace').splitlines() if ': ' in line) + entries = [n for n in z.namelist() if n.endswith('.dist-info/METADATA')] + if len(entries) != 1: raise SystemExit('wheel must have exactly one METADATA entry') + meta = email.parser.BytesParser().parsebytes(z.read(entries[0])) + if len(meta.get_all('Name', [])) != 1 or len(meta.get_all('Version', [])) != 1: + raise SystemExit('wheel must declare one Name and Version') + expressions = meta.get_all('License-Expression', []) + if len(expressions) > 1: raise SystemExit('wheel has ambiguous License-Expression') + licenses = [v for v in meta.get_all('License', []) if v.strip() and v.strip() != 'UNKNOWN'] + classifiers = [v for v in meta.get_all('Classifier', []) if v.startswith('License ::')] + license_value = '; '.join(expressions or licenses or classifiers) or 'UNKNOWN' sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() - release = json.load(urllib.request.urlopen('https://pypi.org/pypi/{}/{}/json'.format(values['Name'], values['Version']))) + release = json.load(urllib.request.urlopen('https://pypi.org/pypi/{}/{}/json'.format(meta['Name'], meta['Version']), timeout=30)) source = next((item['url'] for item in release['urls'] if item['filename'] == wheel.name and item['digests']['sha256'] == sha256), None) if not source: raise SystemExit('could not establish exact source identity for ' + wheel.name) - rows.append({'filename': wheel.name, 'sha256': sha256, 'distribution': values['Name'].lower().replace('-', '_'), 'version': values['Version'], 'license': values.get('License', 'UNKNOWN'), 'sourceUrl': source}) + rows.append({'filename': wheel.name, 'sha256': sha256, 'distribution': meta['Name'].lower().replace('-', '_'), 'version': meta['Version'], 'license': license_value, 'sourceUrl': source}) pathlib.Path('release/wheel-inputs.json').write_text(json.dumps(rows)) pathlib.Path('release/wheel-license-inventory.json').write_text(json.dumps({'schema': 1, 'wheels': rows}, indent=2)) PY - curl --fail --location --proto '=https' --tlsv1.2 -o release/python.tar.gz 'https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.11.16%2B20260901-aarch64-apple-darwin-install_only.tar.gz' - test "$(shasum -a 256 release/python.tar.gz | awk '{print $1}')" = 50424fa409e8ae84b82a3052522f64695b47dff2158b70bb7358e0ebd6c085c9 - /usr/bin/tar -tzf release/python.tar.gz | python -c "import json,sys; paths=[p.strip() for p in sys.stdin if any(x in p.upper() for x in ('LICENSE','COPYING','NOTICE'))]; print(json.dumps({'schema':1,'archiveSha256':'50424fa409e8ae84b82a3052522f64695b47dff2158b70bb7358e0ebd6c085c9','bundledLicenseEvidence':[{'path':p,'declaredLicense':'REVIEW_REQUIRED'} for p in paths]},indent=2))" > release/pbs-license-inventory.json + curl --fail --location --proto '=https' --tlsv1.2 -o release/python.tar.gz "$PYTHON_URL" + test "$(shasum -a 256 release/python.tar.gz | awk '{print $1}')" = "$PYTHON_SHA256" + /usr/bin/tar -tzf release/python.tar.gz | python -c "import json,os,sys; paths=[p.strip() for p in sys.stdin if any(x in p.upper() for x in ('LICENSE','COPYING','NOTICE'))]; print(json.dumps({'schema':1,'archiveSha256':os.environ['PYTHON_SHA256'],'bundledLicenseEvidence':[{'path':p,'declaredLicense':'REVIEW_REQUIRED'} for p in paths]},indent=2))" > release/pbs-license-inventory.json /usr/bin/tar -xpzf release/python.tar.gz -C release/runtime /bin/chmod 700 release/runtime release/runtime/python/bin/python3.11 -m venv release/runtime/venv diff --git a/apps/genbi/scripts/managed-wren-release.test.mjs b/apps/genbi/scripts/managed-wren-release.test.mjs index d6d797b30..ff3072a44 100644 --- a/apps/genbi/scripts/managed-wren-release.test.mjs +++ b/apps/genbi/scripts/managed-wren-release.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { execFileSync, spawnSync } from "node:child_process"; -import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -118,7 +118,6 @@ test("workflow is manual-only, data-binds user input and exposes no raw pre-appr test("actual workflow approval commands reject missing or stale approval and bind all three review files", async (t) => { const root = await temp(t); - const { mkdir } = await import("node:fs/promises"); await mkdir(path.join(root, "release")); const names = ["managed-wren-manifest.candidate.json", "pbs-license-inventory.json", "wheel-license-inventory.json"]; const bytes = names.map((name) => JSON.stringify({ fixture: name }) + "\n"); @@ -138,3 +137,63 @@ test("actual workflow approval commands reject missing or stale approval and bin await writeFile(path.join(root, "release", names[i]), bytes[i]); } }); + +test("actual staging commands derive source identities from the exact input file", async (t) => { + const root = await temp(t); + const inputDir = path.join(root, "apps/genbi/managed-wren"); + await mkdir(inputDir, { recursive: true }); + const changed = { python: { url: "https://example.invalid/changed.tar.gz", sha256: "e".repeat(64) }, wrenai: { version: "99.1.2" } }; + await writeFile(path.join(inputDir, "release-inputs.json"), JSON.stringify(changed)); + const workflow = await readFile(path.join(repo, ".github/workflows/managed-wren-runtime.yml"), "utf8"); + const prelude = workflow.slice(workflow.indexOf(" set -euo pipefail"), workflow.indexOf(" mkdir -p release/wheels")); + assert.ok(prelude.includes("release-inputs.json")); + const output = execFileSync("/bin/bash", ["-c", prelude + '\nnode -e \'console.log(JSON.stringify(process.argv.slice(1)))\' "$PYTHON_URL" "$PYTHON_SHA256" "$WRENAI_VERSION"'], { cwd: root, encoding: "utf8" }); + assert.deepEqual(JSON.parse(output), [changed.python.url, changed.python.sha256, changed.wrenai.version]); + assert.match(workflow, /--dest release\/wheels "wrenai==\$WRENAI_VERSION"/); + assert.match(workflow, /-o release\/python.tar.gz "\$PYTHON_URL"/); + assert.match(workflow, /= "\$PYTHON_SHA256"/); + assert.match(workflow, /'archiveSha256':os.environ\['PYTHON_SHA256'\]/); + assert.doesNotMatch(workflow, /50424fa4|wrenai==0\.13\.0|releases\/download\/20260901/); +}); + +test("actual wheel inventory parser respects header boundaries and modern, folded and repeated license fields", async (t) => { + const root = await temp(t); + const workflow = await readFile(path.join(repo, ".github/workflows/managed-wren-runtime.yml"), "utf8"); + const snippet = workflow.match(/ python - <<'PY'\n([\s\S]*?) PY/)[1].split("\n").map((line) => line.slice(10)).join("\n"); + const bootstrap = ` +import hashlib, io, json, pathlib, urllib.request, zipfile +root = pathlib.Path('release/wheels') +root.mkdir(parents=True) +headers = { + 'modern': 'License-Expression: MIT OR Apache-2.0\\nLicense: ignored', + 'folded': 'License: first line\\n second line', + 'classified': 'License: UNKNOWN\\nClassifier: Topic :: Database\\nClassifier: License :: OSI Approved :: MIT License\\nClassifier: License :: OSI Approved :: BSD License', + 'repeated': 'License: first\\nLicense: second', + 'unknown': '', +} +responses = {} +for name, license_headers in headers.items(): + filename = name + '-1.0-py3-none-any.whl' + target = root / filename + metadata = 'Metadata-Version: 2.4\\nName: ' + name + '\\nVersion: 1.0\\n' + license_headers + '\\n\\nName: spoofed\\nVersion: 9.9\\nLicense: SPOOFED\\n' + with zipfile.ZipFile(target, 'w') as z: + z.writestr(name + '-1.0.dist-info/METADATA', metadata) + sha = hashlib.sha256(target.read_bytes()).hexdigest() + responses['https://pypi.org/pypi/' + name + '/1.0/json'] = {'urls': [{'url': 'https://files.pythonhosted.org/fixture/' + filename, 'filename': filename, 'digests': {'sha256': sha}}]} +def fake_urlopen(url, timeout): + assert timeout == 30 + assert url in responses, 'unexpected source identity: ' + url + return io.BytesIO(json.dumps(responses[url]).encode()) +urllib.request.urlopen = fake_urlopen +`; + execFileSync("python3", ["-c", bootstrap + "\n" + snippet], { cwd: root }); + const rows = JSON.parse(await readFile(path.join(root, "release/wheel-inputs.json"), "utf8")); + const licenses = Object.fromEntries(rows.map((row) => [row.distribution, row.license])); + assert.equal(licenses.modern, "MIT OR Apache-2.0"); + assert.match(licenses.folded, /first line\n\s+second line/); + assert.equal(licenses.repeated, "first; second"); + assert.match(licenses.classified, /MIT License; License :: OSI Approved :: BSD License/); + assert.equal(licenses.unknown, "UNKNOWN"); + assert.equal(rows.length, 5); + assert.ok(rows.every((row) => row.version === "1.0" && !row.license.includes("SPOOFED"))); +});