From 80d53e700f3e1edc0e6a30c1170bff831b124ed7 Mon Sep 17 00:00:00 2001 From: akfoster Date: Mon, 31 Mar 2025 19:48:01 -0700 Subject: [PATCH] Add coverage tests to github workflow for PRs to `main` (#2500) * add coverage to github workflows * add changeset * continue to run on errors * address comments from ellipsis-dev * add unit tests, confirm passing * remove pyproject.toml approach to deps instalL * use github_output env var * add verbose mode for debugging * verbose was not being captured * std out, not print * remove 1000 line dump... * print every line separately to prevent clipping * build extension before testing * clean up debugging prints * break coverage.py into smaller files * break coverage.py into smaller files * Relative module names not supported * rename coverage to coverage_check to avoid naming collision * rename coverage to coverage_check to avoid naming collision * debugging missing coverage files * output error on coverage report * handling ellipsis-dev comments * see if we need vscode test deps * experimenting to try to get it running * use absolute paths * put everything in root directory * address more comments from ellipsis-dev * import run_command in extraction.py * fix verbose flag * use parent parser * ensure safe command test will allow our commands * make sure we install before coverage * update tests to ensure run --------- Co-authored-by: Dennis Bartlett --- .changeset/warm-pets-worry.md | 5 + .github/scripts/coverage_check/__init__.py | 19 + .github/scripts/coverage_check/__main__.py | 154 +++++++ .github/scripts/coverage_check/extraction.py | 265 ++++++++++++ .github/scripts/coverage_check/github_api.py | 177 ++++++++ .github/scripts/coverage_check/util.py | 245 +++++++++++ .github/scripts/coverage_check/workflow.py | 432 +++++++++++++++++++ .github/scripts/tests/coverage_check_test.py | 282 ++++++++++++ .github/workflows/test.yml | 125 +++++- .gitignore | 10 +- 10 files changed, 1710 insertions(+), 4 deletions(-) create mode 100644 .changeset/warm-pets-worry.md create mode 100644 .github/scripts/coverage_check/__init__.py create mode 100644 .github/scripts/coverage_check/__main__.py create mode 100644 .github/scripts/coverage_check/extraction.py create mode 100644 .github/scripts/coverage_check/github_api.py create mode 100644 .github/scripts/coverage_check/util.py create mode 100644 .github/scripts/coverage_check/workflow.py create mode 100644 .github/scripts/tests/coverage_check_test.py diff --git a/.changeset/warm-pets-worry.md b/.changeset/warm-pets-worry.md new file mode 100644 index 0000000000..92ea085cc8 --- /dev/null +++ b/.changeset/warm-pets-worry.md @@ -0,0 +1,5 @@ +--- +"claude-dev": minor +--- + +add coverage tests to github workflows diff --git a/.github/scripts/coverage_check/__init__.py b/.github/scripts/coverage_check/__init__.py new file mode 100644 index 0000000000..581cf61c9f --- /dev/null +++ b/.github/scripts/coverage_check/__init__.py @@ -0,0 +1,19 @@ +""" +Coverage utility package for GitHub Actions workflows. +This package handles extracting coverage percentages, comparing them, and generating PR comments. +""" + +# Import external dependencies +import requests + +# Import main function for CLI usage +from .__main__ import main + +# Import functions from extraction module +from .extraction import extract_coverage, compare_coverage, run_coverage, set_verbose + +# Import functions from github_api module +from .github_api import generate_comment, post_comment, set_github_output + +# Import functions from workflow module +from .workflow import process_coverage_workflow diff --git a/.github/scripts/coverage_check/__main__.py b/.github/scripts/coverage_check/__main__.py new file mode 100644 index 0000000000..7e9f918b55 --- /dev/null +++ b/.github/scripts/coverage_check/__main__.py @@ -0,0 +1,154 @@ +""" +Main module. +This module provides the CLI interface for the coverage utility script. +""" + +import sys +import argparse + +from .extraction import extract_coverage, compare_coverage, run_coverage, set_verbose +from .github_api import generate_comment, post_comment, set_github_output +from .workflow import process_coverage_workflow +from .util import log + +def setup_verbose_mode(args): + """ + Set up verbose mode based on command line arguments. + + Args: + args: Parsed command line arguments + """ + if getattr(args, 'verbose', False): + set_verbose(True) + log("Verbose mode enabled") + +def main(): + # Create parent parser with common arguments + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output') + + # Create main parser that inherits common arguments + parser = argparse.ArgumentParser(description='Coverage utility script for GitHub Actions workflows', parents=[parent_parser]) + subparsers = parser.add_subparsers(dest='command', help='Command to run') + + # extract-coverage command - used directly in workflow + extract_parser = subparsers.add_parser('extract-coverage', help='Extract coverage percentage from a file', parents=[parent_parser]) + extract_parser.add_argument('file_path', help='Path to the coverage report file') + extract_parser.add_argument('--type', choices=['extension', 'webview'], default='extension', + help='Type of coverage report') + extract_parser.add_argument('--github-output', action='store_true', help='Output in GitHub Actions format') + + # compare-coverage command - used by process-workflow + compare_parser = subparsers.add_parser('compare-coverage', help='Compare coverage percentages', parents=[parent_parser]) + compare_parser.add_argument('base_cov', help='Base branch coverage percentage') + compare_parser.add_argument('pr_cov', help='PR branch coverage percentage') + compare_parser.add_argument('--output-prefix', default='', help='Prefix for GitHub Actions output variables') + compare_parser.add_argument('--github-output', action='store_true', help='Output in GitHub Actions format') + + # generate-comment command - used by process-workflow + comment_parser = subparsers.add_parser('generate-comment', help='Generate PR comment with coverage comparison', parents=[parent_parser]) + comment_parser.add_argument('base_ext_cov', help='Base branch extension coverage') + comment_parser.add_argument('pr_ext_cov', help='PR branch extension coverage') + comment_parser.add_argument('ext_decreased', help='Whether extension coverage decreased (true/false)') + comment_parser.add_argument('ext_diff', help='Extension coverage difference') + comment_parser.add_argument('base_web_cov', help='Base branch webview coverage') + comment_parser.add_argument('pr_web_cov', help='PR branch webview coverage') + comment_parser.add_argument('web_decreased', help='Whether webview coverage decreased (true/false)') + comment_parser.add_argument('web_diff', help='Webview coverage difference') + + # post-comment command - used by process-workflow + post_parser = subparsers.add_parser('post-comment', help='Post a comment to a GitHub PR', parents=[parent_parser]) + post_parser.add_argument('comment_path', help='Path to the file containing the comment text') + post_parser.add_argument('pr_number', help='PR number') + post_parser.add_argument('repo', help='Repository in the format "owner/repo"') + post_parser.add_argument('--token', help='GitHub token') + + # run-coverage command - used by process-workflow + run_parser = subparsers.add_parser('run-coverage', help='Run a coverage command and extract the coverage percentage', parents=[parent_parser]) + run_parser.add_argument('coverage_cmd', help='Command to run') + run_parser.add_argument('output_file', help='File to save the output to') + run_parser.add_argument('--type', choices=['extension', 'webview'], default='extension', + help='Type of coverage report') + run_parser.add_argument('--github-output', action='store_true', help='Output in GitHub Actions format') + + # process-workflow command - used directly in workflow + workflow_parser = subparsers.add_parser('process-workflow', help='Process the entire coverage workflow', parents=[parent_parser]) + workflow_parser.add_argument('--base-branch', required=True, help='Base branch name') + workflow_parser.add_argument('--pr-number', help='PR number') + workflow_parser.add_argument('--repo', help='Repository in the format "owner/repo"') + workflow_parser.add_argument('--token', help='GitHub token') + + # set-github-output command - used by process-workflow + output_parser = subparsers.add_parser('set-github-output', help='Set GitHub Actions output variable', parents=[parent_parser]) + output_parser.add_argument('name', help='Output variable name') + output_parser.add_argument('value', help='Output variable value') + + args = parser.parse_args() + + # Set up verbose mode + setup_verbose_mode(args) + + if args.command == 'extract-coverage': + log(f"Extracting coverage from file: {args.file_path} (type: {args.type})") + coverage_pct = extract_coverage(args.file_path, args.type) + if args.github_output: + set_github_output(f"{args.type}_coverage", coverage_pct) + else: + log(f"Coverage: {coverage_pct}%") + + elif args.command == 'compare-coverage': + log(f"Comparing coverage: base={args.base_cov}%, PR={args.pr_cov}%") + decreased, diff = compare_coverage(args.base_cov, args.pr_cov) + if args.github_output: + prefix = args.output_prefix + set_github_output(f"{prefix}decreased", str(decreased).lower()) + set_github_output(f"{prefix}diff", diff) + log(f"Coverage difference: {diff}%") + log(f"Coverage decreased: {decreased}") + else: + log(f"decreased={str(decreased).lower()}") + log(f"diff={diff}") + + elif args.command == 'generate-comment': + log("Generating coverage comparison comment") + comment = generate_comment( + args.base_ext_cov, args.pr_ext_cov, args.ext_decreased, args.ext_diff, + args.base_web_cov, args.pr_web_cov, args.web_decreased, args.web_diff + ) + # Output the comment to stdout + log(comment) + + elif args.command == 'post-comment': + log(f"Posting comment from {args.comment_path} to PR #{args.pr_number} in {args.repo}") + post_comment(args.comment_path, args.pr_number, args.repo, args.token) + + elif args.command == 'run-coverage': + log(f"Running coverage command: {args.coverage_cmd}") + log(f"Output file: {args.output_file}") + log(f"Coverage type: {args.type}") + coverage_pct = run_coverage(args.coverage_cmd, args.output_file, args.type) + if args.github_output: + set_github_output(f"{args.type}_coverage", coverage_pct) + else: + log(f"Coverage: {coverage_pct}%") + + elif args.command == 'process-workflow': + log("Processing coverage workflow") + log(f"Base branch: {args.base_branch}") + if args.pr_number: + log(f"PR number: {args.pr_number}") + if args.repo: + log(f"Repository: {args.repo}") + process_coverage_workflow(args) + + elif args.command == 'set-github-output': + log(f"Setting GitHub output: {args.name}={args.value}") + set_github_output(args.name, args.value) + + else: + log("No command specified") + parser.print_help() + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/.github/scripts/coverage_check/extraction.py b/.github/scripts/coverage_check/extraction.py new file mode 100644 index 0000000000..ec9b732d11 --- /dev/null +++ b/.github/scripts/coverage_check/extraction.py @@ -0,0 +1,265 @@ +""" +Coverage extraction module. +This module handles extracting coverage percentages from coverage report files. +""" + +import os +import re +import sys +import shlex +import subprocess +import traceback +from .util import log, file_exists, get_file_size, list_directory, is_safe_command, run_command + +# Global verbose flag +verbose = False + +def set_verbose(value): + """Set the global verbose flag.""" + global verbose + verbose = value + +def print_debug_output(content, coverage_type): + """ + Print debug information about the coverage output. + + Args: + content: The content of the coverage file + coverage_type: Type of coverage report (extension or webview) + """ + if not verbose: + return + + # Extract and print only the coverage summary section + if coverage_type == "extension": + # Look for the coverage summary section + summary_match = re.search(r'=============================== Coverage summary ===============================\n(.*?)\n=+', content, re.DOTALL) + if summary_match: + sys.stdout.write("\n##[group]EXTENSION COVERAGE SUMMARY\n") + sys.stdout.write("=============================== Coverage summary ===============================\n") + sys.stdout.write(summary_match.group(1) + "\n") + sys.stdout.write("================================================================================\n") + sys.stdout.write("##[endgroup]\n") + sys.stdout.flush() + else: + sys.stdout.write("\n##[warning]No coverage summary found in extension coverage file\n") + sys.stdout.flush() + else: # webview + # Look for the coverage table - specifically the "All files" row + table_match = re.search(r'% Coverage report from v8.*?-+\|.*?\n.*?\n(All files.*?)(?:\n[^\n]*\|)', content, re.DOTALL) + if table_match: + sys.stdout.write("\n##[group]WEBVIEW COVERAGE SUMMARY\n") + sys.stdout.write("% Coverage report from v8\n") + sys.stdout.write("-------------------|---------|----------|---------|---------|-------------------\n") + sys.stdout.write("File | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s \n") + sys.stdout.write("-------------------|---------|----------|---------|---------|-------------------\n") + sys.stdout.write(table_match.group(1) + "\n") + sys.stdout.write("-------------------|---------|----------|---------|---------|-------------------\n") + sys.stdout.write("##[endgroup]\n") + sys.stdout.flush() + else: + sys.stdout.write("\n##[warning]No coverage table found in webview coverage file\n") + sys.stdout.flush() + +def extract_coverage(file_path, coverage_type="extension"): + """ + Extract coverage percentage from a coverage report file. + + Args: + file_path: Path to the coverage report file + coverage_type: Type of coverage report (extension or webview) + + Returns: + Coverage percentage as a float + """ + + # Always print file path for debugging + log(f"Checking coverage file: {file_path}") + + # Check if file exists and get its size + if not file_exists(file_path): + sys.stdout.write(f"\n##[error]File {file_path} does not exist\n") + sys.stdout.flush() + log(f"Error: File {file_path} does not exist") + + # Check if the directory exists + dir_path = os.path.dirname(file_path) + if not os.path.exists(dir_path): + sys.stdout.write(f"\n##[error]Directory {dir_path} does not exist\n") + sys.stdout.flush() + log(f"Error: Directory {dir_path} does not exist") + else: + # List directory contents for debugging + log(f"Directory {dir_path} exists, listing contents:") + try: + dir_contents = list_directory(dir_path) + for name, size in dir_contents: + log(f" {name} - {size}") + sys.stdout.write(f" {name} - {size}\n") + sys.stdout.flush() + except Exception as e: + log(f"Error listing directory: {e}") + + return 0.0 + + file_size = get_file_size(file_path) + log(f"File size: {file_size} bytes") + sys.stdout.write(f"\n##[info]Coverage file {file_path} exists, size: {file_size} bytes\n") + sys.stdout.flush() + + if file_size == 0: + sys.stdout.write(f"\n##[warning]File {file_path} is empty\n") + sys.stdout.flush() + log(f"Warning: File {file_path} is empty") + return 0.0 + + # List directory contents for debugging + dir_path = os.path.dirname(file_path) + log(f"Directory contents of {dir_path}:") + try: + dir_contents = list_directory(dir_path) + for name, size in dir_contents: + log(f" {name} - {size}") + except Exception as e: + log(f"Error listing directory: {e}") + + with open(file_path, 'r') as f: + content = f.read() + + # Print debug information if verbose + print_debug_output(content, coverage_type) + + # Extract coverage percentage based on coverage type + if coverage_type == "extension": + # Extract the percentage from the "Lines" row in the coverage summary + # Pattern: Lines : xx.xx% ( xxxxxxx/xxxxxxx ) + lines_match = re.search(r'Lines\s*:\s*(\d+\.\d+)%', content) + if lines_match: + coverage_pct = float(lines_match.group(1)) + if verbose: + sys.stdout.write(f"Pattern matched (Lines percentage): {coverage_pct}\n") + sys.stdout.flush() + return coverage_pct + else: + # No coverage data found, log full content for debugging + log("No coverage data found. Full file content:") + log("=== Full file content ===") + log(content) + log("=== End file content ===") + else: # webview + # Extract the percentage from the "% Lines" column in the "All files" row + # Pattern: All files | xx.xx | xx.xx | xx.xx | xx.xx | + all_files_match = re.search(r'All files\s+\|\s+\d+\.\d+\s+\|\s+\d+\.\d+\s+\|\s+\d+\.\d+\s+\|\s+(\d+\.\d+)', content) + if all_files_match: + coverage_pct = float(all_files_match.group(1)) + if verbose: + sys.stdout.write(f"Pattern matched (All files % Lines): {coverage_pct}\n") + sys.stdout.flush() + return coverage_pct + else: + # No coverage data found, log full content for debugging + log("No coverage data found. Full file content:") + log("=== Full file content ===") + log(content) + log("=== End file content ===") + + # If no match found, return 0.0 + return 0.0 + +def compare_coverage(base_cov, pr_cov): + """ + Compare coverage percentages between base and PR branches. + + Args: + base_cov: Base branch coverage percentage + pr_cov: PR branch coverage percentage + + Returns: + Tuple of (decreased, diff) + """ + try: + base_cov = float(base_cov) + pr_cov = float(pr_cov) + except ValueError: + sys.stdout.write(f"Error: Invalid coverage values - base: {base_cov}, PR: {pr_cov}\n") + sys.stdout.flush() + return False, 0 + + diff = pr_cov - base_cov + decreased = diff < 0 + + return decreased, abs(diff) + +def run_coverage(command, output_file, coverage_type="extension"): + """ + Run a coverage command and extract the coverage percentage. + + Args: + command: Command to run + output_file: File to save the output to + coverage_type: Type of coverage report (extension or webview) + + Returns: + Coverage percentage as a float + + Raises: + SystemExit: If the output file is not created or is empty + """ + + try: + # Run the command and capture output + if not is_safe_command(command): + error_msg = f"ERROR: Unsafe command detected: {command}" + log(error_msg) + sys.stdout.write(f"\n##[error]{error_msg}\n") + sys.stdout.flush() + sys.exit(1) + + # Run command using safe execution from util + returncode, stdout, stderr = run_command(command) + + # Log command result + log(f"Command exit code: {returncode}") + log(f"Command stdout length: {len(stdout)} bytes") + log(f"Command stderr length: {len(stderr)} bytes") + + # Save output to file + log(f"Saving command output to {output_file}") + with open(output_file, 'w') as f: + f.write(stdout) + if stderr: + f.write("\n\n=== STDERR ===\n") + f.write(stderr) + + # Verify file was created and has content + if not file_exists(output_file): + error_msg = f"ERROR: Output file {output_file} was not created" + log(error_msg) + sys.stdout.write(f"\n##[error]{error_msg}\n") + sys.stdout.flush() + sys.exit(1) # Exit with error code to fail the workflow + + file_size = get_file_size(output_file) + if file_size == 0: + error_msg = f"ERROR: Output file {output_file} is empty" + log(error_msg) + sys.stdout.write(f"\n##[error]{error_msg}\n") + sys.stdout.flush() + sys.exit(1) # Exit with error code to fail the workflow + + log(f"Output file size: {file_size} bytes") + + # Extract coverage percentage + coverage_pct = extract_coverage(output_file, coverage_type) + + log(f"{coverage_type.capitalize()} coverage: {coverage_pct}%") + return coverage_pct + + except Exception as e: + error_msg = f"Error running coverage command: {e}" + log(error_msg) + sys.stdout.write(f"\n##[error]{error_msg}\n") + sys.stdout.flush() + # Print stack trace for debugging + log(traceback.format_exc()) + sys.exit(1) # Exit with error code to fail the workflow diff --git a/.github/scripts/coverage_check/github_api.py b/.github/scripts/coverage_check/github_api.py new file mode 100644 index 0000000000..6251c70808 --- /dev/null +++ b/.github/scripts/coverage_check/github_api.py @@ -0,0 +1,177 @@ +""" +GitHub API module. +This module handles interactions with the GitHub API for posting comments to PRs. +""" + +import os +import requests +from .util import log, file_exists + +def generate_comment(base_ext_cov, pr_ext_cov, ext_decreased, ext_diff, + base_web_cov, pr_web_cov, web_decreased, web_diff): + """ + Generate a PR comment with coverage comparison. + + Args: + base_ext_cov: Base branch extension coverage + pr_ext_cov: PR branch extension coverage + ext_decreased: Whether extension coverage decreased + ext_diff: Extension coverage difference + base_web_cov: Base branch webview coverage + pr_web_cov: PR branch webview coverage + web_decreased: Whether webview coverage decreased + web_diff: Webview coverage difference + + Returns: + Comment text + """ + from datetime import datetime + + # Convert string inputs to appropriate types + try: + base_ext_cov = float(base_ext_cov) + pr_ext_cov = float(pr_ext_cov) + # Handle ext_decreased as either string or boolean + if isinstance(ext_decreased, str): + ext_decreased = ext_decreased.lower() == 'true' + else: + ext_decreased = bool(ext_decreased) + ext_diff = float(ext_diff) + base_web_cov = float(base_web_cov) + pr_web_cov = float(pr_web_cov) + # Handle web_decreased as either string or boolean + if isinstance(web_decreased, str): + web_decreased = web_decreased.lower() == 'true' + else: + web_decreased = bool(web_decreased) + web_diff = float(web_diff) + except ValueError as e: + log(f"Error converting input values: {e}") + return "" + + # Add a unique identifier to find this comment later + comment = '\n' + comment += '## Coverage Report\n\n' + + # Extension coverage + comment += '### Extension Coverage\n\n' + comment += f'Base branch: {base_ext_cov:.0f}%\n\n' + comment += f'PR branch: {pr_ext_cov:.0f}%\n\n' + + if ext_decreased: + comment += f'⚠️ **Warning: Coverage decreased by {ext_diff:.2f}%**\n\n' + comment += 'Consider adding tests to cover your changes.\n\n' + else: + comment += '✅ Coverage increased or remained the same\n\n' + + # Webview coverage + comment += '### Webview Coverage\n\n' + comment += f'Base branch: {base_web_cov:.0f}%\n\n' + comment += f'PR branch: {pr_web_cov:.0f}%\n\n' + + if web_decreased: + comment += f'⚠️ **Warning: Coverage decreased by {web_diff:.2f}%**\n\n' + comment += 'Consider adding tests to cover your changes.\n\n' + else: + comment += '✅ Coverage increased or remained the same\n\n' + + # Overall assessment + comment += '### Overall Assessment\n\n' + if ext_decreased or web_decreased: + comment += '⚠️ **Test coverage has decreased in this PR**\n\n' + comment += 'Please consider adding tests to maintain or improve coverage.\n\n' + else: + comment += '✅ **Test coverage has been maintained or improved**\n\n' + + # Add timestamp + comment += f'\n\nLast updated: {datetime.now().isoformat()}' + + return comment + +def post_comment(comment_path, pr_number, repo, token=None): + """ + Post a comment to a GitHub PR. + + Args: + comment_path: Path to the file containing the comment text + pr_number: PR number + repo: Repository in the format "owner/repo" + token: GitHub token + """ + if not file_exists(comment_path): + log(f"Error: Comment file {comment_path} does not exist") + return + + with open(comment_path, 'r') as f: + comment_body = f.read() + + if not token: + token = os.environ.get('GITHUB_TOKEN') + if not token: + log("Error: GitHub token not provided") + return + + # Find existing comment + headers = { + 'Authorization': f'token {token}', + 'Accept': 'application/vnd.github.v3+json' + } + + # Get all comments + comments_url = f'https://api.github.com/repos/{repo}/issues/{pr_number}/comments' + log(f"Getting comments from: {comments_url}") + response = requests.get(comments_url, headers=headers) + + if response.status_code != 200: + log(f"Error getting comments: {response.status_code} - {response.text}") + return + + comments = response.json() + log(f"Found {len(comments)} existing comments") + + # Find comment with our identifier + comment_id = None + for comment in comments: + if '' in comment['body']: + comment_id = comment['id'] + log(f"Found existing coverage report comment with ID: {comment_id}") + break + + if comment_id: + # Update existing comment + update_url = f'https://api.github.com/repos/{repo}/issues/comments/{comment_id}' + log(f"Updating existing comment at: {update_url}") + response = requests.patch(update_url, headers=headers, json={'body': comment_body}) + + if response.status_code == 200: + log(f"Successfully updated existing comment: {comment_id}") + else: + log(f"Error updating comment: {response.status_code} - {response.text}") + else: + # Create new comment + log(f"Creating new comment at: {comments_url}") + response = requests.post(comments_url, headers=headers, json={'body': comment_body}) + + if response.status_code == 201: + log("Successfully created new comment") + else: + log(f"Error creating comment: {response.status_code} - {response.text}") + +def set_github_output(name, value): + """ + Set GitHub Actions output variable. + + Args: + name: Output variable name + value: Output variable value + """ + # Write to the GitHub output file if available + if 'GITHUB_OUTPUT' in os.environ: + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f"{name}={value}\n") + else: + # Fallback to the deprecated method for backward compatibility + log(f"::set-output name={name}::{value}") + + # Also print for human readability + log(f"{name}: {value}") diff --git a/.github/scripts/coverage_check/util.py b/.github/scripts/coverage_check/util.py new file mode 100644 index 0000000000..b5f0b188fa --- /dev/null +++ b/.github/scripts/coverage_check/util.py @@ -0,0 +1,245 @@ +""" +Utility module. +This module provides utility functions used across the coverage check scripts. +""" + +import os +import sys +import re +import shlex +import subprocess +import traceback +from typing import List, Tuple, Dict, Any, Optional, Union + +# List of allowed commands and their arguments +ALLOWED_COMMANDS = { + 'xvfb-run': ['-a'], + 'npm': ['run', 'test:coverage', 'ci', 'install', '--no-save', '@vitest/coverage-v8', 'check-types', 'lint', 'format', 'compile'], + 'cd': ['webview-ui'], + 'python': ['-m', 'coverage_check'], + 'git': ['fetch', 'checkout', 'origin'], +} + +def is_safe_command(command: Union[str, List[str]]) -> bool: + """ + Check if a command is safe to execute. + + Args: + command: Command to check (string or list) + + Returns: + True if command is safe, False otherwise + """ + # Convert string command to list + if isinstance(command, str): + try: + cmd_parts = shlex.split(command) + except ValueError: + return False + else: + cmd_parts = command + + if not cmd_parts: + return False + + # Get base command + base_cmd = os.path.basename(cmd_parts[0]) + + # Check if command is in allowed list + if base_cmd not in ALLOWED_COMMANDS: + return False + + # For each argument, check for suspicious patterns + for arg in cmd_parts[1:]: + # Check for shell metacharacters + if re.search(r'[;&|`$]', arg): + return False + # Check for path traversal + if '..' in arg and not (base_cmd == 'npm' and arg.startswith('@')): + return False + + return True + +def log(message: str) -> None: + """ + Write a message to stdout and flush. + + Args: + message: The message to write + """ + sys.stdout.write(f"{message}\n") + sys.stdout.flush() + +def file_exists(file_path: str) -> bool: + """ + Check if a file exists. + + Args: + file_path: Path to the file + + Returns: + True if the file exists, False otherwise + """ + return os.path.exists(file_path) and os.path.isfile(file_path) + +def get_file_size(file_path: str) -> int: + """ + Get the size of a file in bytes. + + Args: + file_path: Path to the file + + Returns: + Size of the file in bytes, or 0 if the file doesn't exist + """ + if file_exists(file_path): + return os.path.getsize(file_path) + return 0 + +def list_directory(dir_path: str) -> List[Tuple[str, Union[int, str]]]: + """ + List the contents of a directory. + + Args: + dir_path: Path to the directory + + Returns: + List of (name, size) tuples for each file/directory in the directory + """ + if not os.path.exists(dir_path) or not os.path.isdir(dir_path): + return [] + + contents = [] + for item in os.listdir(dir_path): + item_path = os.path.join(dir_path, item) + if os.path.isfile(item_path): + contents.append((item, os.path.getsize(item_path))) + else: + contents.append((item, "DIR")) + + return contents + +def read_file_content(file_path: str, default: str = "") -> str: + """ + Read file content with error handling. + + Args: + file_path: Path to the file + default: Default value to return if file cannot be read + + Returns: + File content or default value + """ + if not file_exists(file_path): + log(f"File does not exist: {file_path}") + return default + + try: + with open(file_path, 'r') as f: + return f.read() + except Exception as e: + log(f"Error reading file {file_path}: {e}") + return default + +def write_file_content(file_path: str, content: str) -> bool: + """ + Write content to file with error handling. + + Args: + file_path: Path to the file + content: Content to write + + Returns: + True if successful, False otherwise + """ + try: + # Create directory if it doesn't exist + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + with open(file_path, 'w') as f: + f.write(content) + return True + except Exception as e: + log(f"Error writing to file {file_path}: {e}") + return False + +def run_command(command: Union[str, List[str]], capture_output: bool = True) -> Tuple[int, str, str]: + """ + Run a command and return the result. + + Args: + command: Command to run (string or list) + capture_output: Whether to capture stdout/stderr + + Returns: + Tuple of (returncode, stdout, stderr) + """ + if not is_safe_command(command): + error_msg = f"Unsafe command detected: {command}" + log(error_msg) + return 1, "", error_msg + + log(f"Running command: {command}") + try: + # Convert string command to list + if isinstance(command, str): + cmd_list = shlex.split(command) + else: + cmd_list = command + + result = subprocess.run( + cmd_list, + shell=False, # Never use shell=True for security + capture_output=capture_output, + text=True + ) + log(f"Command exit code: {result.returncode}") + return result.returncode, result.stdout, result.stderr + except Exception as e: + log(f"Error running command: {e}") + log(traceback.format_exc()) + return 1, "", str(e) + +def find_pattern(content: str, pattern: str, group: int = 0, + default: Optional[str] = None) -> Optional[str]: + """ + Find a pattern in content and return the specified group. + + Args: + content: Text content to search + pattern: Regex pattern to search for + group: Group number to return (default: 0 for entire match) + default: Default value to return if pattern not found + + Returns: + Matched text or default value + """ + match = re.search(pattern, content, re.DOTALL) + if match: + return match.group(group) + return default + +def get_env_var(name: str, default: Optional[str] = None) -> Optional[str]: + """ + Get environment variable with default value. + + Args: + name: Environment variable name + default: Default value if not set + + Returns: + Environment variable value or default + """ + return os.environ.get(name, default) + +def format_exception(e: Exception) -> str: + """ + Format an exception with traceback for logging. + + Args: + e: Exception to format + + Returns: + Formatted exception string + """ + return f"{type(e).__name__}: {str(e)}\n{traceback.format_exc()}" diff --git a/.github/scripts/coverage_check/workflow.py b/.github/scripts/coverage_check/workflow.py new file mode 100644 index 0000000000..8288143504 --- /dev/null +++ b/.github/scripts/coverage_check/workflow.py @@ -0,0 +1,432 @@ +""" +Workflow module. +This module handles the main workflow logic for running coverage tests and processing results. +""" + +import os +import re +import sys +import subprocess +import traceback + +from .extraction import run_coverage, compare_coverage, extract_coverage +from .github_api import generate_comment, post_comment, set_github_output +from .util import log, file_exists, get_file_size, list_directory, run_command + +def is_valid_branch_name(branch_name: str) -> bool: + """ + Validate a git branch name. + + Args: + branch_name: Branch name to validate + + Returns: + True if valid, False otherwise + """ + # Check for common branch name patterns + if not re.match(r'^[a-zA-Z0-9_\-./]+$', branch_name): + return False + + # Check for path traversal + if '..' in branch_name: + return False + + # Check for shell metacharacters + if re.search(r'[;&|`$]', branch_name): + return False + + return True + +def checkout_branch(branch_name: str) -> None: + """ + Checkout a branch for testing. + + Args: + branch_name: Branch name to checkout + + Raises: + RuntimeError: If branch checkout fails + ValueError: If branch name is invalid + """ + if not is_valid_branch_name(branch_name): + raise ValueError(f"Invalid branch name: {branch_name}") + + log(f"=== Checking out branch: {branch_name} ===") + + # Fetch the branch + returncode, stdout, stderr = run_command(['git', 'fetch', 'origin', branch_name]) + if returncode != 0: + log(f"ERROR: Failed to fetch branch {branch_name}") + log(f"Error details: {stderr}") + raise RuntimeError(f"Git fetch failed: {stderr}") + + # Checkout the branch + returncode, stdout, stderr = run_command(['git', 'checkout', branch_name]) + if returncode != 0: + log(f"ERROR: Failed to checkout branch {branch_name}") + log(f"Error details: {stderr}") + raise RuntimeError(f"Git checkout failed: {stderr}") + + log(f"Successfully checked out branch: {branch_name}") + +def extract_extension_coverage_from_file(file_path): + """Extract extension coverage from file when run_coverage returns 0.""" + if not file_exists(file_path): + log(f"File {file_path} does not exist, cannot extract extension coverage") + return 0.0 + + file_size = get_file_size(file_path) + if file_size == 0: + log(f"File {file_path} is empty, cannot extract extension coverage") + return 0.0 + + log(f"Extension coverage is 0.0, trying to read from file directly: {file_path} (size: {file_size} bytes)") + with open(file_path, 'r') as f: + content = f.read() + # Extract the percentage from the "Lines" row in the coverage summary + # Pattern: Lines : xx.xx% ( xxxxxxx/xxxxxxx ) + lines_match = re.search(r'Lines\s*:\s*(\d+\.\d+)%', content) + if lines_match: + coverage = float(lines_match.group(1)) + log(f"Found extension coverage in file: {coverage}%") + return coverage + return 0.0 + +def extract_webview_coverage_from_file(file_path): + """Extract webview coverage from file when run_coverage returns 0.""" + if not file_exists(file_path): + log(f"File {file_path} does not exist, cannot extract webview coverage") + return 0.0 + + file_size = get_file_size(file_path) + if file_size == 0: + log(f"File {file_path} is empty, cannot extract webview coverage") + return 0.0 + + log(f"Webview coverage is 0.0, trying to read from file directly: {file_path} (size: {file_size} bytes)") + with open(file_path, 'r') as f: + content = f.read() + # Extract the percentage from the "% Lines" column in the "All files" row + # Pattern: All files | xx.xx | xx.xx | xx.xx | xx.xx | + all_files_match = re.search(r'All files\s+\|\s+\d+\.\d+\s+\|\s+\d+\.\d+\s+\|\s+\d+\.\d+\s+\|\s+(\d+\.\d+)', content) + if all_files_match: + coverage = float(all_files_match.group(1)) + log(f"Found webview coverage in file: {coverage}%") + return coverage + return 0.0 + +def run_extension_coverage(branch_name=None): + """Run extension coverage tests and extract results.""" + prefix = 'base_' if branch_name else '' + file_path = f"{prefix}extension_coverage.txt" + + # Run coverage tests + ext_cov = run_coverage( + ["xvfb-run", "-a", "npm", "run", "test:coverage"], + file_path, + "extension" + ) + + # If coverage is 0.0, try to extract from file directly + if ext_cov == 0.0: + ext_cov = extract_extension_coverage_from_file(file_path) + + return ext_cov + +def run_webview_coverage(branch_name=None): + """Run webview coverage tests and extract results.""" + prefix = 'base_' if branch_name else '' + file_path = f"{prefix}webview_coverage.txt" + + # Save current directory + original_dir = os.getcwd() + + try: + # Change to webview-ui directory + os.chdir('webview-ui') + + # Install coverage dependency + returncode, stdout, stderr = run_command(["npm", "install", "--no-save", "@vitest/coverage-v8"]) + if returncode != 0: + log(f"Failed to install coverage dependency: {stderr}") + return 0.0 + + # Run coverage tests from webview-ui directory + web_cov = run_coverage( + ["npm", "run", "test:coverage"], + os.path.join('..', file_path), + "webview" + ) + finally: + # Always change back to original directory + os.chdir(original_dir) + + # If coverage is 0.0, try to extract from file directly + if web_cov == 0.0: + web_cov = extract_webview_coverage_from_file(file_path) + + return web_cov + +def run_branch_coverage(branch_name=None): + """ + Run coverage tests for a branch. + + Args: + branch_name: Name of the branch to checkout before running tests (optional) + + Returns: + Tuple of (extension_coverage, webview_coverage) + """ + # Checkout branch if specified + if branch_name: + checkout_branch(branch_name) + + # Run coverage tests + log(f"=== Running coverage tests{' for ' + branch_name if branch_name else ''} ===") + + # Run extension and webview coverage + ext_cov = run_extension_coverage(branch_name) + web_cov = run_webview_coverage(branch_name) + + return ext_cov, web_cov + +def find_potential_coverage_files(): + """Find potential coverage files in the current directory and webview-ui.""" + log("Searching for potential coverage files...") + + # Find files in current directory + current_dir_files = list_directory('.') + for name, size in current_dir_files: + if 'coverage' in name.lower() and size != "DIR": + log(f"Found potential coverage file: {name} (size: {size} bytes)") + + # Find files in webview-ui directory + if os.path.exists('webview-ui') and os.path.isdir('webview-ui'): + webview_files = list_directory('webview-ui') + for name, size in webview_files: + if 'coverage' in name.lower() and size != "DIR": + log(f"Found potential webview coverage file: webview-ui/{name} (size: {size} bytes)") + else: + log("webview-ui directory not found") + +def generate_warnings(base_ext_cov, pr_ext_cov, ext_decreased, ext_diff, + base_web_cov, pr_web_cov, web_decreased, web_diff): + """Generate warnings for coverage decreases.""" + if not (ext_decreased or web_decreased): + return [] + + warnings = [ + "Test coverage has decreased in this PR", + f"Extension coverage: {base_ext_cov}% -> {pr_ext_cov}% (Diff: {ext_diff}%)", + f"Webview coverage: {base_web_cov}% -> {pr_web_cov}% (Diff: {web_diff}%)" + ] + + # Additional warning for significant decrease (more than 1%) + if ext_decreased and ext_diff > 1.0: + warnings.append(f"Extension coverage decreased by more than 1% ({ext_diff}%). Consider adding tests to cover your changes.") + + if web_decreased and web_diff > 1.0: + warnings.append(f"Webview coverage decreased by more than 1% ({web_diff}%). Consider adding tests to cover your changes.") + + return warnings + +def output_warnings(warnings): + """Output warnings to GitHub step summary and console.""" + if not warnings: + return + + # Get the GitHub step summary file path from environment variable + github_step_summary = os.environ.get('GITHUB_STEP_SUMMARY') + + # Write to GitHub step summary if available + if github_step_summary: + with open(github_step_summary, 'a') as f: + f.write("## Coverage Warnings\n\n") + for warning in warnings: + f.write(f"⚠️ {warning}\n\n") + + # Also output to console with ::warning:: syntax for backward compatibility + for warning in warnings: + log(f"::warning::{warning}") + +def output_github_results(pr_ext_cov, pr_web_cov, base_ext_cov, base_web_cov, + ext_decreased, ext_diff, web_decreased, web_diff): + """Output results for GitHub Actions.""" + set_github_output("pr_extension_coverage", pr_ext_cov) + set_github_output("pr_webview_coverage", pr_web_cov) + set_github_output("base_extension_coverage", base_ext_cov) + set_github_output("base_webview_coverage", base_web_cov) + set_github_output("extension_decreased", str(ext_decreased).lower()) + set_github_output("extension_diff", ext_diff) + set_github_output("webview_decreased", str(web_decreased).lower()) + set_github_output("webview_diff", web_diff) + +def extract_pr_coverage_from_artifacts(): + """ + Extract PR branch coverage from artifact files. + + Returns: + Tuple of (extension_coverage, webview_coverage) + + Raises: + SystemExit: If the coverage files don't exist + """ + log("=== Extracting PR branch coverage from artifacts ===") + + # Check if the coverage files exist + ext_file_path = "extension_coverage.txt" + web_file_path = "webview-ui/webview_coverage.txt" + + # Extract extension coverage + log(f"Extracting extension coverage from {ext_file_path}") + if not file_exists(ext_file_path): + error_msg = f"ERROR: PR extension coverage file {ext_file_path} not found" + log(error_msg) + + # List directory contents for debugging + log("Current directory contents:") + try: + dir_contents = list_directory('.') + for name, size in dir_contents: + log(f" {name} - {size}\n") + except Exception as e: + log(f"Error listing directory: {e}") + + sys.exit(1) # Exit with error code to fail the workflow + + ext_cov = extract_extension_coverage_from_file(ext_file_path) + log(f"PR extension coverage from artifact: {ext_cov}%") + + # Extract webview coverage + log(f"Extracting webview coverage from {web_file_path}") + if not file_exists(web_file_path): + error_msg = f"ERROR: PR webview coverage file {web_file_path} not found" + log(error_msg) + + # Check if the webview-ui directory exists + if not os.path.exists('webview-ui'): + log("ERROR: webview-ui directory not found") + else: + # List webview-ui directory contents for debugging + log("webview-ui directory contents:") + try: + dir_contents = list_directory('webview-ui') + for name, size in dir_contents: + log(f" {name} - {size}") + except Exception as e: + log(f"Error listing directory: {e}") + + sys.exit(1) # Exit with error code to fail the workflow + + web_cov = extract_webview_coverage_from_file(web_file_path) + log(f"PR webview coverage from artifact: {web_cov}%") + + return ext_cov, web_cov + +def process_coverage_workflow(args): + """ + Process the entire coverage workflow. + + Args: + args: Command line arguments + """ + # Initialize all variables at the start + pr_ext_cov = 0.0 + pr_web_cov = 0.0 + base_ext_cov = 0.0 + base_web_cov = 0.0 + ext_decreased = False + ext_diff = 0.0 + web_decreased = False + web_diff = 0.0 + + try: + # Validate branch name + if not is_valid_branch_name(args.base_branch): + raise ValueError(f"Invalid base branch name: {args.base_branch}") + + # Check if we're running in GitHub Actions + is_github_actions = 'GITHUB_ACTIONS' in os.environ + if is_github_actions: + log("Running in GitHub Actions environment") + + # Extract PR branch coverage from artifacts (from test job) + pr_ext_cov, pr_web_cov = extract_pr_coverage_from_artifacts() + + # Verify PR coverage values + if pr_ext_cov == 0.0: + log("WARNING: PR extension coverage is 0.0, this may indicate an issue with the coverage report") + find_potential_coverage_files() + + if pr_web_cov == 0.0: + log("WARNING: PR webview coverage is 0.0, this may indicate an issue with the coverage report") + find_potential_coverage_files() + + # Run base branch coverage + log(f"=== Running base branch coverage for {args.base_branch} ===") + base_ext_cov, base_web_cov = run_branch_coverage(args.base_branch) + + # Verify base coverage values + if base_ext_cov == 0.0: + log("WARNING: Base extension coverage is 0.0, this may indicate an issue with the coverage report") + + if base_web_cov == 0.0: + log("WARNING: Base webview coverage is 0.0, this may indicate an issue with the coverage report") + + # Compare coverage + log("=== Comparing extension coverage ===") + ext_decreased, ext_diff = compare_coverage(base_ext_cov, pr_ext_cov) + + log("=== Comparing webview coverage ===") + web_decreased, web_diff = compare_coverage(base_web_cov, pr_web_cov) + + # Print summary of coverage values + log("\n=== Coverage Summary ===") + log(f"PR extension coverage: {pr_ext_cov}%") + log(f"Base extension coverage: {base_ext_cov}%") + log(f"Extension coverage change: {'+' if not ext_decreased else '-'}{ext_diff}%") + log(f"PR webview coverage: {pr_web_cov}%") + log(f"Base webview coverage: {base_web_cov}%") + log(f"Webview coverage change: {'+' if not web_decreased else '-'}{web_diff}%") + + # Generate and output warnings + warnings = generate_warnings( + base_ext_cov, pr_ext_cov, ext_decreased, ext_diff, + base_web_cov, pr_web_cov, web_decreased, web_diff + ) + output_warnings(warnings) + + # Generate comment + log("=== Generating comment ===") + comment = generate_comment( + base_ext_cov, pr_ext_cov, str(ext_decreased).lower(), ext_diff, + base_web_cov, pr_web_cov, str(web_decreased).lower(), web_diff + ) + + # Save comment to file + with open("coverage_comment.md", "w") as f: + f.write(comment) + + # Post comment if PR number is provided + if args.pr_number: + log(f"=== Posting comment to PR #{args.pr_number} ===") + post_comment("coverage_comment.md", args.pr_number, args.repo, args.token) + + # Output results for GitHub Actions + output_github_results( + pr_ext_cov, pr_web_cov, base_ext_cov, base_web_cov, + ext_decreased, ext_diff, web_decreased, web_diff + ) + + except Exception as e: + log(f"ERROR in process_coverage_workflow: {e}") + traceback.print_exc() + + # Try to output results even if there was an error + try: + output_github_results( + pr_ext_cov, pr_web_cov, base_ext_cov, base_web_cov, + ext_decreased, ext_diff, web_decreased, web_diff + ) + except Exception as e2: + log(f"ERROR outputting GitHub results: {e2}") diff --git a/.github/scripts/tests/coverage_check_test.py b/.github/scripts/tests/coverage_check_test.py new file mode 100644 index 0000000000..9325fad278 --- /dev/null +++ b/.github/scripts/tests/coverage_check_test.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +""" +Tests for coverage_check script. +""" + +import os +import sys +import unittest +import subprocess +import tempfile +from unittest.mock import patch, MagicMock, call, mock_open + +# Add parent directory to path so we can import coverage modules +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from coverage_check import extract_coverage, compare_coverage, set_verbose, generate_comment, post_comment, set_github_output +from coverage_check.util import log, file_exists, get_file_size, list_directory + + +class TestCoverage(unittest.TestCase): + # Class variables to store coverage files + temp_dir = None + extension_coverage_file = None + webview_coverage_file = None + + @classmethod + def setUpClass(cls): + """Set up test environment once for all tests.""" + # Create temporary directory for test files + cls.temp_dir = tempfile.TemporaryDirectory() + cls.extension_coverage_file = os.path.join(cls.temp_dir.name, 'extension_coverage.txt') + cls.webview_coverage_file = os.path.join(cls.temp_dir.name, 'webview_coverage.txt') + + # Run actual tests to generate coverage reports + cls.generate_coverage_reports() + + # Verify files exist and are not empty + assert os.path.exists(cls.extension_coverage_file), \ + f"Extension coverage file {cls.extension_coverage_file} does not exist" + assert os.path.getsize(cls.extension_coverage_file) > 0, \ + f"Extension coverage file {cls.extension_coverage_file} is empty" + assert os.path.exists(cls.webview_coverage_file), \ + f"Webview coverage file {cls.webview_coverage_file} does not exist" + assert os.path.getsize(cls.webview_coverage_file) > 0, \ + f"Webview coverage file {cls.webview_coverage_file} is empty" + + @classmethod + def tearDownClass(cls): + """Clean up test environment after all tests.""" + if cls.temp_dir: + cls.temp_dir.cleanup() + + @classmethod + def generate_coverage_reports(cls): + """Generate real coverage reports by running tests.""" + log("Generating coverage reports (this may take a while)...") + + # Run extension tests with coverage + try: + # Get absolute paths + root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../..')) + webview_dir = os.path.join(root_dir, 'webview-ui') + + # Use xvfb-run on Linux + if sys.platform.startswith('linux'): + cmd = f"cd {root_dir} && xvfb-run -a npm run test:coverage > {cls.extension_coverage_file} 2>&1" + else: + cmd = f"cd {root_dir} && npm run test:coverage > {cls.extension_coverage_file} 2>&1" + + log("Running extension tests...") + log(f"Command: {cmd}") + result = subprocess.run(cmd, shell=True, check=False, capture_output=True, text=True) + log(f"Extension tests exit code: {result.returncode}") + + # Run webview tests with coverage + log("Running webview tests...") + cmd = f"cd {webview_dir} && npm run test:coverage > {cls.webview_coverage_file} 2>&1" + log(f"Command: {cmd}") + result = subprocess.run(cmd, shell=True, check=False, capture_output=True, text=True) + log(f"Webview tests exit code: {result.returncode}") + + # Verify files were created + if file_exists(cls.extension_coverage_file): + ext_size = get_file_size(cls.extension_coverage_file) + log(f"Extension coverage file created: {cls.extension_coverage_file} (size: {ext_size} bytes)") + else: + log(f"WARNING: Extension coverage file was not created: {cls.extension_coverage_file}") + + if file_exists(cls.webview_coverage_file): + web_size = get_file_size(cls.webview_coverage_file) + log(f"Webview coverage file created: {cls.webview_coverage_file} (size: {web_size} bytes)") + else: + log(f"WARNING: Webview coverage file was not created: {cls.webview_coverage_file}") + + log("Coverage reports generation completed.") + except Exception as e: + log(f"Error generating coverage reports: {e}") + import traceback + log(traceback.format_exc()) + + # Create empty files if tests fail + log("Creating fallback coverage files...") + with open(cls.extension_coverage_file, 'w') as f: + f.write("No coverage data available") + with open(cls.webview_coverage_file, 'w') as f: + f.write("No coverage data available") + + def test_extract_coverage(self): + """Test extract_coverage function with both extension and webview coverage.""" + # Check if verbose mode is enabled + if '-v' in sys.argv or '--verbose' in sys.argv: + set_verbose(True) + + # Verify files exist before testing + self.assertTrue(file_exists(self.extension_coverage_file), + f"Extension coverage file does not exist: {self.extension_coverage_file}") + self.assertTrue(file_exists(self.webview_coverage_file), + f"Webview coverage file does not exist: {self.webview_coverage_file}") + + # Log file sizes + ext_size = get_file_size(self.extension_coverage_file) + web_size = get_file_size(self.webview_coverage_file) + log(f"Extension coverage file size: {ext_size} bytes") + log(f"Webview coverage file size: {web_size} bytes") + + # Test extension coverage + log("Testing extension coverage extraction...") + ext_coverage_pct = extract_coverage(self.extension_coverage_file, 'extension') + + # Check that coverage percentage is a float + self.assertIsInstance(ext_coverage_pct, float) + + # Check that coverage percentage is between 0 and 100 + self.assertGreaterEqual(ext_coverage_pct, 0) + self.assertLessEqual(ext_coverage_pct, 100) + + # Log coverage percentage for debugging + log(f"Extension coverage: {ext_coverage_pct}%") + + # Test webview coverage + log("Testing webview coverage extraction...") + web_coverage_pct = extract_coverage(self.webview_coverage_file, 'webview') + + # Convert to float if it's an integer + if isinstance(web_coverage_pct, int): + web_coverage_pct = float(web_coverage_pct) + + # Check that coverage percentage is a float + self.assertIsInstance(web_coverage_pct, float) + + # Check that coverage percentage is between 0 and 100 + self.assertGreaterEqual(web_coverage_pct, 0) + self.assertLessEqual(web_coverage_pct, 100) + + # Log coverage percentage for debugging + log(f"Webview coverage: {web_coverage_pct}%") + + def test_compare_coverage(self): + """Test compare_coverage function.""" + # Test with coverage increase + decreased, diff = compare_coverage(80, 90) + self.assertFalse(decreased) + self.assertEqual(diff, 10) + + # Test with coverage decrease + decreased, diff = compare_coverage(90, 80) + self.assertTrue(decreased) + self.assertEqual(diff, 10) + + # Test with no change + decreased, diff = compare_coverage(80, 80) + self.assertFalse(decreased) + self.assertEqual(diff, 0) + + def test_generate_comment(self): + """Test generate_comment function.""" + comment = generate_comment( + 80, 90, 'false', 10, + 70, 75, 'false', 5 + ) + + # Check that comment contains expected sections + self.assertIn('Coverage Report', comment) + self.assertIn('Extension Coverage', comment) + self.assertIn('Webview Coverage', comment) + self.assertIn('Overall Assessment', comment) + + # Check that comment contains coverage percentages + self.assertIn('Base branch: 80%', comment) + self.assertIn('PR branch: 90%', comment) + self.assertIn('Base branch: 70%', comment) + self.assertIn('PR branch: 75%', comment) + + # Check that comment contains correct assessment + self.assertIn('Coverage increased or remained the same', comment) + self.assertIn('Test coverage has been maintained or improved', comment) + + @patch('coverage_check.requests.get') + @patch('coverage_check.requests.post') + @patch('coverage_check.requests.patch') + def test_post_comment_new(self, mock_patch, mock_post, mock_get): + """Test post_comment function when creating a new comment.""" + # Create a temporary comment file + comment_file = os.path.join(self.temp_dir.name, 'comment.md') + with open(comment_file, 'w') as f: + f.write('\nTest comment') + + # Mock the API responses + mock_get.return_value = MagicMock(status_code=200, json=lambda: []) + mock_post.return_value = MagicMock(status_code=201) + + # Test post_comment function + post_comment(comment_file, '123', 'owner/repo', 'token') + + # Check that the correct API calls were made + mock_get.assert_called_once() + mock_post.assert_called_once() + mock_patch.assert_not_called() + + @patch('coverage_check.requests.get') + @patch('coverage_check.requests.post') + @patch('coverage_check.requests.patch') + def test_post_comment_update(self, mock_patch, mock_post, mock_get): + """Test post_comment function when updating an existing comment.""" + # Create a temporary comment file + comment_file = os.path.join(self.temp_dir.name, 'comment.md') + with open(comment_file, 'w') as f: + f.write('\nTest comment') + + # Mock the API responses + mock_get.return_value = MagicMock( + status_code=200, + json=lambda: [{'id': 456, 'body': '\nOld comment'}] + ) + mock_patch.return_value = MagicMock(status_code=200) + + # Test post_comment function + post_comment(comment_file, '123', 'owner/repo', 'token') + + # Check that the correct API calls were made + mock_get.assert_called_once() + mock_patch.assert_called_once() + mock_post.assert_not_called() + + def test_set_github_output(self): + """Test set_github_output function.""" + # Capture stdout + with patch('sys.stdout', new=MagicMock()) as mock_stdout: + # Mock environment without GITHUB_OUTPUT + with patch.dict('os.environ', {}, clear=True): + set_github_output('test_name', 'test_value') + + # Check that the correct output was printed to stdout + mock_stdout.assert_has_calls([ + # GitHub Actions output format (deprecated method) + call.write('::set-output name=test_name::test_value\n'), + call.flush(), + # Human readable format + call.write('test_name: test_value\n'), + call.flush() + ], any_order=False) + + # Reset mock for next test + mock_stdout.reset_mock() + + # Test with GITHUB_OUTPUT environment variable + with patch.dict('os.environ', {'GITHUB_OUTPUT': '/tmp/github_output'}), \ + patch('builtins.open', mock_open()) as mock_file: + set_github_output('test_name', 'test_value') + + # Check that file was written to + mock_file.assert_called_once_with('/tmp/github_output', 'a') + mock_file().write.assert_called_once_with('test_name=test_value\n') + + # Check that human readable output was printed + mock_stdout.assert_has_calls([ + call.write('test_name: test_value\n'), + call.flush() + ], any_order=False) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 518bc8d244..93a0cb49b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,17 @@ jobs: with: node-version: 20.15.1 + # Setup Python for coverage script + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install requests + # Cache root dependencies - only reuse if package-lock.json exactly matches - name: Cache root dependencies uses: actions/cache@v4 @@ -58,5 +69,115 @@ jobs: - name: Prettier / Format Check run: npm run format - - name: Extension Tests - run: xvfb-run -a npm run test + # Build the extension before running tests + - name: Build Extension + run: npm run compile + + # Run extension tests with coverage + - name: Extension Tests with Coverage + id: extension_coverage + continue-on-error: true + run: | + xvfb-run -a npm run test:coverage > extension_coverage.txt 2>&1 || true + PYTHONPATH=.github/scripts python -m coverage_check extract-coverage extension_coverage.txt --type=extension --github-output --verbose + + # Run webview tests with coverage + - name: Webview Tests with Coverage + id: webview_coverage + continue-on-error: true + run: | + cd webview-ui + # Ensure coverage dependency is installed + npm install --no-save @vitest/coverage-v8 + npm run test:coverage > webview_coverage.txt 2>&1 || true + cd .. + PYTHONPATH=.github/scripts python -m coverage_check extract-coverage webview-ui/webview_coverage.txt --type=webview --github-output --verbose + + # Save coverage reports as artifacts (workflow-scoped) + - name: Save Coverage Reports + uses: actions/upload-artifact@v4 + with: + name: pr-coverage-reports + path: | + extension_coverage.txt + webview-ui/webview_coverage.txt + retention-period: workflow # Artifacts are automatically deleted when the workflow completes + + coverage: + needs: test + runs-on: ubuntu-latest + # Only run on PRs to main branch + if: github.event_name == 'pull_request' && github.base_ref == 'main' + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for accurate comparison + + # Setup Python for coverage script + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install requests + + - name: Setup Node.js environment + uses: actions/setup-node@v4 + with: + node-version: 20.15.1 + + # Cache root dependencies - only reuse if package-lock.json exactly matches + - name: Cache root dependencies + uses: actions/cache@v4 + id: root-cache + with: + path: node_modules + key: ${{ runner.os }}-npm-${{ hashFiles('package-lock.json') }} + + # Cache webview-ui dependencies - only reuse if package-lock.json exactly matches + - name: Cache webview-ui dependencies + uses: actions/cache@v4 + id: webview-cache + with: + path: webview-ui/node_modules + key: ${{ runner.os }}-npm-webview-${{ hashFiles('webview-ui/package-lock.json') }} + + - name: Install root dependencies + if: steps.root-cache.outputs.cache-hit != 'true' + run: npm ci + + - name: Install webview-ui dependencies + if: steps.webview-cache.outputs.cache-hit != 'true' + run: cd webview-ui && npm ci + + # Build the extension before running tests + - name: Build Extension + run: npm run compile + + # Download coverage artifacts from test job + - name: Download Coverage Reports + uses: actions/download-artifact@v4 + with: + name: pr-coverage-reports + path: . # Download to root directory to match expected paths + + # Process coverage workflow + - name: Process coverage workflow + id: coverage + run: | + # Extract PR number from GITHUB_REF + PR_NUMBER=$(echo "$GITHUB_REF" | sed -e 's/refs\/pull\///' -e 's/\/merge//') + + # Run the coverage workflow from root directory + PYTHONPATH=.github/scripts python -m coverage_check process-workflow \ + --base-branch ${{ github.base_ref }} \ + --pr-number $PR_NUMBER \ + --repo $GITHUB_REPOSITORY \ + --token ${{ secrets.GITHUB_TOKEN }} \ + --verbose + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 5ab339dfb3..c7949d35ce 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ out -coverage dist node_modules tmp @@ -10,4 +9,11 @@ tmp pnpm-lock.yaml -.clineignore \ No newline at end of file +.clineignore +.venv +.actrc + +# Ignore coverage directories and files +coverage +# But don't ignore the coverage scripts in .github/scripts/ +!.github/scripts/coverage/