From ecd5ccc76ca2468bee63cf8108dd488100a31cae Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Mon, 4 May 2026 14:50:15 +0800 Subject: [PATCH] ci: add CI/CD workflows, dependabot, labeler + release-please for core/ layout (#2216) --- .../actions/rust/setup-builder/action.yaml | 45 ++++ .../setup-macos-aarch64-builder/action.yaml | 36 +++ .../rust/setup-macos-builder/action.yaml | 36 +++ .../rust/setup-rust-runtime/action.yaml | 27 +++ .../rust/setup-windows-builder/action.yaml | 32 +++ .github/dependabot.yaml | 16 -- .github/dependabot.yml | 81 +++++++ .github/labeler.yml | 93 ++++++++ .../workflows/ai-service-release-image.yaml | 105 --------- .../ai-service-release-nightly-image.yaml | 101 --------- .../ai-service-release-stable-image.yaml | 162 -------------- .github/workflows/ai-service-test.yaml | 57 ----- .github/workflows/core-py-ci.yml | 46 ++++ .github/workflows/create-rc-release-pr.yaml | 75 ------- .github/workflows/create-rc-release.yaml | 97 -------- .github/workflows/labeler.yaml | 12 + .github/workflows/pr-tagger.yaml | 147 ------------- .github/workflows/publish-wren-core-py.yml | 170 ++++++++++++++ .github/workflows/publish-wren-core-wasm.yml | 90 ++++++++ .github/workflows/publish-wren.yml | 80 +++++++ .../pull-request-title-validator.yaml | 16 +- .github/workflows/rc-release.yml | 144 ++++++++++++ .github/workflows/release-please.yml | 62 ++++++ .github/workflows/rust.yml | 131 +++++++++++ .github/workflows/skills-check.yml | 19 ++ .github/workflows/sync-docs.yml | 61 ++++++ .github/workflows/ui-lint.yaml | 52 ----- .../workflows/ui-release-image-stable.yaml | 150 ------------- .github/workflows/ui-release-image.yaml | 104 --------- .github/workflows/ui-test.yaml | 50 ----- .github/workflows/wasm-ci.yml | 99 +++++++++ .github/workflows/wren-ci.yml | 207 ++++++++++++++++++ .github/workflows/wren-launcher-ci.yaml | 83 ------- .release-please-manifest.json | 5 + release-please-config.json | 31 +++ 35 files changed, 1514 insertions(+), 1208 deletions(-) create mode 100644 .github/actions/rust/setup-builder/action.yaml create mode 100644 .github/actions/rust/setup-macos-aarch64-builder/action.yaml create mode 100644 .github/actions/rust/setup-macos-builder/action.yaml create mode 100644 .github/actions/rust/setup-rust-runtime/action.yaml create mode 100644 .github/actions/rust/setup-windows-builder/action.yaml delete mode 100644 .github/dependabot.yaml create mode 100644 .github/dependabot.yml create mode 100644 .github/labeler.yml delete mode 100644 .github/workflows/ai-service-release-image.yaml delete mode 100644 .github/workflows/ai-service-release-nightly-image.yaml delete mode 100644 .github/workflows/ai-service-release-stable-image.yaml delete mode 100644 .github/workflows/ai-service-test.yaml create mode 100644 .github/workflows/core-py-ci.yml delete mode 100644 .github/workflows/create-rc-release-pr.yaml delete mode 100644 .github/workflows/create-rc-release.yaml create mode 100644 .github/workflows/labeler.yaml delete mode 100644 .github/workflows/pr-tagger.yaml create mode 100644 .github/workflows/publish-wren-core-py.yml create mode 100644 .github/workflows/publish-wren-core-wasm.yml create mode 100644 .github/workflows/publish-wren.yml create mode 100644 .github/workflows/rc-release.yml create mode 100644 .github/workflows/release-please.yml create mode 100644 .github/workflows/rust.yml create mode 100644 .github/workflows/skills-check.yml create mode 100644 .github/workflows/sync-docs.yml delete mode 100644 .github/workflows/ui-lint.yaml delete mode 100644 .github/workflows/ui-release-image-stable.yaml delete mode 100644 .github/workflows/ui-release-image.yaml delete mode 100644 .github/workflows/ui-test.yaml create mode 100644 .github/workflows/wasm-ci.yml create mode 100644 .github/workflows/wren-ci.yml delete mode 100644 .github/workflows/wren-launcher-ci.yaml create mode 100644 .release-please-manifest.json create mode 100644 release-please-config.json diff --git a/.github/actions/rust/setup-builder/action.yaml b/.github/actions/rust/setup-builder/action.yaml new file mode 100644 index 000000000..39b67b691 --- /dev/null +++ b/.github/actions/rust/setup-builder/action.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Prepare Rust Builder +description: 'Prepare Rust Build Environment' +inputs: + rust-version: + description: 'version of rust to install (e.g. stable)' + required: true + default: 'stable' +runs: + using: "composite" + steps: + - name: Install Build Dependencies + shell: bash + run: | + apt-get update + - name: Setup Rust toolchain + shell: bash + # rustfmt is needed for the substrait build script + run: | + echo "Installing ${{ inputs.rust-version }}" + rustup toolchain install ${{ inputs.rust-version }} + rustup default ${{ inputs.rust-version }} + rustup component add rustfmt + - name: Configure rust runtime env + uses: ./.github/actions/rust/setup-rust-runtime + - name: Fixup git permissions + # https://github.com/actions/checkout/issues/766 + shell: bash + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.github/actions/rust/setup-macos-aarch64-builder/action.yaml b/.github/actions/rust/setup-macos-aarch64-builder/action.yaml new file mode 100644 index 000000000..d81c37527 --- /dev/null +++ b/.github/actions/rust/setup-macos-aarch64-builder/action.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Prepare Rust Builder for MacOS +description: 'Prepare Rust Build Environment for MacOS' +inputs: + rust-version: + description: 'version of rust to install (e.g. stable)' + required: true + default: 'stable' +runs: + using: "composite" + steps: + - name: Setup Rust toolchain + shell: bash + run: | + rustup update stable + rustup toolchain install stable + rustup default stable + rustup component add rustfmt + - name: Configure rust runtime env + uses: ./.github/actions/rust/setup-rust-runtime diff --git a/.github/actions/rust/setup-macos-builder/action.yaml b/.github/actions/rust/setup-macos-builder/action.yaml new file mode 100644 index 000000000..d81c37527 --- /dev/null +++ b/.github/actions/rust/setup-macos-builder/action.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Prepare Rust Builder for MacOS +description: 'Prepare Rust Build Environment for MacOS' +inputs: + rust-version: + description: 'version of rust to install (e.g. stable)' + required: true + default: 'stable' +runs: + using: "composite" + steps: + - name: Setup Rust toolchain + shell: bash + run: | + rustup update stable + rustup toolchain install stable + rustup default stable + rustup component add rustfmt + - name: Configure rust runtime env + uses: ./.github/actions/rust/setup-rust-runtime diff --git a/.github/actions/rust/setup-rust-runtime/action.yaml b/.github/actions/rust/setup-rust-runtime/action.yaml new file mode 100644 index 000000000..047a50e47 --- /dev/null +++ b/.github/actions/rust/setup-rust-runtime/action.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Setup Rust Runtime +description: 'Setup Rust Runtime Environment' +runs: + using: "composite" + steps: + - name: Configure runtime env + shell: bash + run: | + echo "RUST_BACKTRACE=1" >> $GITHUB_ENV + echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV diff --git a/.github/actions/rust/setup-windows-builder/action.yaml b/.github/actions/rust/setup-windows-builder/action.yaml new file mode 100644 index 000000000..07331ce04 --- /dev/null +++ b/.github/actions/rust/setup-windows-builder/action.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Prepare Rust Builder for Windows +description: 'Prepare Rust Build Environment for Windows' + +runs: + using: "composite" + steps: + - name: Setup Rust toolchain + shell: bash + run: | + # Avoid self update to avoid CI failures: https://github.com/apache/datafusion/issues/9653 + rustup toolchain install stable --no-self-update + rustup default stable + rustup component add rustfmt + - name: Configure rust runtime env + uses: ./.github/actions/rust/setup-rust-runtime diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml deleted file mode 100644 index f74af7409..000000000 --- a/.github/dependabot.yaml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "pip" - directory: "/wren-ai-service" - schedule: - interval: "weekly" - groups: - all: - patterns: ["*"] - commit-message: - prefix: "chore(wren-ai-service)" - labels: - - "dependencies" - - "python" - - "ci/ai-service" - - "module/ai-service" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..bfc7e6ca6 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,81 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/core/wren-core-py" + schedule: + interval: "weekly" + groups: + all: + patterns: ["*"] + commit-message: + prefix: "deps(core-py)" + labels: + - "dependencies" + - "python" + - "core" + + - package-ecosystem: "pip" + directory: "/core/wren" + schedule: + interval: "weekly" + groups: + all: + patterns: ["*"] + commit-message: + prefix: "deps(wren)" + labels: + - "dependencies" + - "python" + - "core" + + - package-ecosystem: "cargo" + directory: "/core/wren-core" + schedule: + interval: "weekly" + groups: + all: + patterns: ["*"] + commit-message: + prefix: "deps(core)" + labels: + - "dependencies" + - "rust" + - "core" + + - package-ecosystem: "cargo" + directory: "/core/wren-core-base" + schedule: + interval: "weekly" + groups: + all: + patterns: ["*"] + commit-message: + prefix: "deps(core)" + labels: + - "dependencies" + - "rust" + - "core" + + - package-ecosystem: "cargo" + directory: "/core/wren-core-wasm" + schedule: + interval: "weekly" + groups: + all: + patterns: ["*"] + commit-message: + prefix: "deps(wasm)" + labels: + - "dependencies" + - "rust" + - "core" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "deps(ci)" + labels: + - "dependencies" + - "ci" diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 000000000..ea0c8a7f5 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,93 @@ +# PR auto-labels driven by changed files. +# Used by .github/workflows/labeler.yaml (actions/labeler). + +core: + - changed-files: + - any-glob-to-any-file: + - "core/**" + +rust: + - changed-files: + - any-glob-to-any-file: + - "core/wren-core/**" + - "core/wren-core-base/**" + - "core/wren-core-wasm/**" + +python: + - changed-files: + - any-glob-to-any-file: + - "core/wren-core-py/**" + - "core/wren/**" + +wasm: + - changed-files: + - any-glob-to-any-file: + - "core/wren-core-wasm/**" + +skills: + - changed-files: + - any-glob-to-any-file: + - "skills/**" + +sdk: + - changed-files: + - any-glob-to-any-file: + - "sdks/**" + +"sdk:langchain": + - changed-files: + - any-glob-to-any-file: + - "sdks/integrations/langchain/**" + +"sdk:crewai": + - changed-files: + - any-glob-to-any-file: + - "sdks/integrations/crewai/**" + +"sdk:pydantic": + - changed-files: + - any-glob-to-any-file: + - "sdks/integrations/pydantic/**" + +"sdk:goose": + - changed-files: + - any-glob-to-any-file: + - "sdks/integrations/goose/**" + +"sdk:llamaindex": + - changed-files: + - any-glob-to-any-file: + - "sdks/integrations/llamaindex/**" + +"sdk:mastra": + - changed-files: + - any-glob-to-any-file: + - "sdks/integrations/mastra/**" + +documentation: + - changed-files: + - any-glob-to-any-file: + - "docs/**" + - "**/*.md" + +examples: + - changed-files: + - any-glob-to-any-file: + - "examples/**" + +ci: + - changed-files: + - any-glob-to-any-file: + - ".github/**" + +dependencies: + - changed-files: + - any-glob-to-any-file: + - "**/Cargo.toml" + - "**/Cargo.lock" + - "**/pyproject.toml" + - "**/poetry.lock" + - "**/uv.lock" + - "**/package.json" + - "**/yarn.lock" + - "**/pnpm-lock.yaml" diff --git a/.github/workflows/ai-service-release-image.yaml b/.github/workflows/ai-service-release-image.yaml deleted file mode 100644 index 584630feb..000000000 --- a/.github/workflows/ai-service-release-image.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: AI Service Release image - -on: - workflow_dispatch: - inputs: - tag_name: - description: Docker image tag name (Optional) - type: string - -env: - WREN_AI_SERVICE_IMAGE: ghcr.io/canner/wren-ai-service - -defaults: - run: - working-directory: wren-ai-service - -jobs: - build-image: - outputs: - tag_name: ${{ steps.tag-preparation.outputs.TAG_NAME }} - strategy: - fail-fast: false - matrix: - arch: - - runner: ubuntu-latest - platform: linux/amd64 - - runner: linux_arm64_runner - platform: linux/arm64 - runs-on: ${{ matrix.arch.runner }} - steps: - - uses: actions/checkout@v4 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Prepare tag name - id: tag-preparation - run: | - if [ -n "${{ github.event.inputs.tag_name }}" ]; then - tag_name=${{ github.event.inputs.tag_name }} - else - tag_name=commit-$(git log -1 --pretty=%h) - fi - echo "TAG_NAME=$tag_name" >> $GITHUB_OUTPUT - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Prepare platform - run: | - platform=${{ matrix.arch.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - platforms: ${{ matrix.arch.platform }} - labels: ${{ env.WREN_AI_SERVICE_IMAGE }} - context: ./wren-ai-service - file: ./wren-ai-service/docker/Dockerfile - outputs: type=image,name=${{ env.WREN_AI_SERVICE_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - merge: - runs-on: ubuntu-latest - needs: [ build-image ] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.WREN_AI_SERVICE_IMAGE }} - tags: | - ${{ needs.build-image.outputs.tag_name }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - TAGS=$(echo "${{ steps.meta.outputs.tags }}" | awk '{printf "--tag %s ", $0}') - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.WREN_AI_SERVICE_IMAGE }}@sha256:%s ' *) \ - $TAGS diff --git a/.github/workflows/ai-service-release-nightly-image.yaml b/.github/workflows/ai-service-release-nightly-image.yaml deleted file mode 100644 index 7ba0dd21a..000000000 --- a/.github/workflows/ai-service-release-nightly-image.yaml +++ /dev/null @@ -1,101 +0,0 @@ -name: AI Service Release nightly image - -on: - push: - branches: [main] - paths: - - 'wren-ai-service/**' - -env: - WREN_AI_SERVICE_IMAGE: ghcr.io/canner/wren-ai-service - -defaults: - run: - working-directory: wren-ai-service - -jobs: - build-image: - outputs: - tag_name: ${{ steps.tag-preparation.outputs.TAG_NAME }} - strategy: - fail-fast: false - matrix: - arch: - - runner: ubuntu-latest - platform: linux/amd64 - - runner: linux_arm64_runner - platform: linux/arm64 - runs-on: ${{ matrix.arch.runner }} - steps: - - uses: actions/checkout@v4 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Prepare tag name - id: tag-preparation - run: | - tag_name=main-$(git log -1 --pretty=%h) - echo "TAG_NAME=$tag_name" >> $GITHUB_OUTPUT - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Prepare platform - run: | - platform=${{ matrix.arch.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - platforms: ${{ matrix.arch.platform }} - labels: ${{ env.WREN_AI_SERVICE_IMAGE }} - context: ./wren-ai-service - file: ./wren-ai-service/docker/Dockerfile - outputs: type=image,name=${{ env.WREN_AI_SERVICE_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - merge: - runs-on: ubuntu-latest - needs: [ build-image ] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.WREN_AI_SERVICE_IMAGE }} - tags: | - type=raw,${{ needs.build-image.outputs.tag_name }} - type=raw,nightly - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - TAGS=$(echo "${{ steps.meta.outputs.tags }}" | awk '{printf "--tag %s ", $0}') - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.WREN_AI_SERVICE_IMAGE }}@sha256:%s ' *) \ - $TAGS diff --git a/.github/workflows/ai-service-release-stable-image.yaml b/.github/workflows/ai-service-release-stable-image.yaml deleted file mode 100644 index e66f2d320..000000000 --- a/.github/workflows/ai-service-release-stable-image.yaml +++ /dev/null @@ -1,162 +0,0 @@ -name: AI Service Release stable image - -on: - workflow_dispatch: - inputs: - version: - description: Give a version for this release - type: string - required: true - -env: - WREN_AI_SERVICE_IMAGE: ghcr.io/canner/wren-ai-service - -defaults: - run: - working-directory: wren-ai-service - -jobs: - upgrade-ai-service-version: - runs-on: ubuntu-latest - steps: - - name: Generate a token - id: generate-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ vars.CI_APP_ID }} - private-key: ${{ secrets.CI_APP_PRIVATE_KEY }} - - uses: actions/checkout@v4 - with: - token: ${{ steps.generate-token.outputs.token }} - fetch-depth: 0 - - name: Set up Git - run: | - git config --global user.name "wren-ai[bot]" - git config --global user.email "dev@cannerdata.com" - - name: Set up Python 3.12.0 - uses: actions/setup-python@v4 - with: - python-version: 3.12.0 - - name: Install Poetry - uses: abatilo/actions-poetry@v2 - with: - poetry-version: 1.8.3 - - name: Generate and Save Change Log - id: changelog - run: | - echo "Generating change log..." - PREVIOUS_VERSION=release/ai-service/$(poetry version -s) - echo "Previous version: $PREVIOUS_VERSION" - - # Get the change log from the previous version to the current HEAD - # If there is no change log, the command will return an error, so we use || true to ignore the error - CHANGE_LOG=$(git log --pretty=format:"%s" $PREVIOUS_VERSION..HEAD | grep wren-ai-service || true) - - # Separate the change log into categories - FEATURES=$(echo "$CHANGE_LOG" | grep "^feat" | sed 's/^/- /') - FIXES_AND_CHORES=$(echo "$CHANGE_LOG" | grep -E "^(fix|chore)" | sed 's/^/- /') - - # Create the full change log - FULL_CHANGE_LOG="\nChangelog for the version\n" - if [ -n "$FEATURES" ]; then - FULL_CHANGE_LOG+="\nFeature and Enhancement\n$FEATURES" - fi - if [ -n "$FIXES_AND_CHORES" ]; then - FULL_CHANGE_LOG+="\n\nFixes and Chores\n$FIXES_AND_CHORES" - fi - - { - echo "CHANGE_LOG<> $GITHUB_ENV - - name: Upgrade AI Service version - run: | - version=${{ github.event.inputs.version }} - poetry version --next-phase $version - git add pyproject.toml - git commit -m "Upgrade AI Service version to $version" - git push - git tag -a "release/ai-service/$version" -m "${{ env.CHANGE_LOG }}" - git push origin "release/ai-service/$version" - build-image: - needs: upgrade-ai-service-version - strategy: - fail-fast: false - matrix: - arch: - - runner: ubuntu-latest - platform: linux/amd64 - - runner: linux_arm64_runner - platform: linux/arm64 - runs-on: ${{ matrix.arch.runner }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Prepare platform - run: | - platform=${{ matrix.arch.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - platforms: ${{ matrix.arch.platform }} - labels: ${{ env.WREN_AI_SERVICE_IMAGE }} - context: ./wren-ai-service - file: ./wren-ai-service/docker/Dockerfile - outputs: type=image,name=${{ env.WREN_AI_SERVICE_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - merge: - runs-on: ubuntu-latest - needs: [ build-image ] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.WREN_AI_SERVICE_IMAGE }} - tags: | - type=raw,${{ github.event.inputs.version }} - type=raw,latest - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - TAGS=$(echo "${{ steps.meta.outputs.tags }}" | awk '{printf "--tag %s ", $0}') - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.WREN_AI_SERVICE_IMAGE }}@sha256:%s ' *) \ - $TAGS diff --git a/.github/workflows/ai-service-test.yaml b/.github/workflows/ai-service-test.yaml deleted file mode 100644 index d11a01ef9..000000000 --- a/.github/workflows/ai-service-test.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: AI Service Test - -on: - push: - branches: - - main - paths: - - "wren-ai-service/**" - pull_request: - types: [synchronize, labeled] - workflow_dispatch: - -permissions: - contents: read - -concurrency: - # avoid mis-canceling the ci runs while other labels are added to the PR, so we add the label name as the condition - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.label.name == 'ci/ai-service' && github.event.number || github.sha }} - cancel-in-progress: true - -defaults: - run: - working-directory: wren-ai-service - -jobs: - pytest: - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci/ai-service') || github.event_name == 'push' }} - runs-on: ubuntu-22.04 - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - name: Install Poetry - uses: abatilo/actions-poetry@v3 - with: - poetry-version: "1.8.3" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version-file: ./wren-ai-service/pyproject.toml - cache: "poetry" - - name: Install the project dependencies - run: poetry install --without eval - - name: Install Just - uses: extractions/setup-just@v2 - with: - just-version: "1.36.0" - - name: Initialize the config - run: | - just init --non-dev - - name: Prepare testing environment and Run tests - run: | - just test - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/.github/workflows/core-py-ci.yml b/.github/workflows/core-py-ci.yml new file mode 100644 index 000000000..c9d7de424 --- /dev/null +++ b/.github/workflows/core-py-ci.yml @@ -0,0 +1,46 @@ +name: Core Python CI +permissions: + contents: read + pull-requests: write + +on: + pull_request: + paths: + - 'core/wren-core-py/**' + - 'core/wren-core/**' + +defaults: + run: + working-directory: core/wren-core-py + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.11 + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./core/wren-core-py/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + - name: Install poetry + run: pipx install poetry + - name: Install dependencies + run: poetry install --no-root + - name: Test Rust and Python code + run: | + cargo test + poetry run maturin develop + poetry run pytest diff --git a/.github/workflows/create-rc-release-pr.yaml b/.github/workflows/create-rc-release-pr.yaml deleted file mode 100644 index b1bb8b68b..000000000 --- a/.github/workflows/create-rc-release-pr.yaml +++ /dev/null @@ -1,75 +0,0 @@ -name: Create RC Release PR - -on: - workflow_dispatch: - inputs: - release_version: - description: "Release version (e.g., 0.23.0-rc.1)" - required: true - ui_version: - description: "UI version (e.g., 0.22.0)" - required: true - ai_version: - description: "AI service version (e.g., 0.17.0)" - required: true - engine_version: - description: "Engine version (e.g., 0.14.7)" - required: true - -permissions: - contents: write - pull-requests: write - -jobs: - create-release-pr: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Setup Git Identity - run: | - git config --global user.name "wren-ai[bot]" - git config --global user.email "dev@cannerdata.com" - - - name: Update docker.go - run: | - FILE_PATH="wren-launcher/utils/docker.go" - - # Replace the WREN_PRODUCT_VERSION value with the new release version - sed -i "s/WREN_PRODUCT_VERSION\s*string = \"[^\"]*\"/WREN_PRODUCT_VERSION\tstring = \"${{ github.event.inputs.release_version }}\"/" $FILE_PATH - - # Verify the changes - grep "WREN_PRODUCT_VERSION" $FILE_PATH - - - name: Update .env.example - run: | - FILE_PATH="docker/.env.example" - - # Update all versions in the .env.example file - sed -i "s/WREN_PRODUCT_VERSION=.*/WREN_PRODUCT_VERSION=${{ github.event.inputs.release_version }}/" $FILE_PATH - sed -i "s/WREN_ENGINE_VERSION=.*/WREN_ENGINE_VERSION=${{ github.event.inputs.engine_version }}/" $FILE_PATH - sed -i "s/IBIS_SERVER_VERSION=.*/IBIS_SERVER_VERSION=${{ github.event.inputs.engine_version }}/" $FILE_PATH - sed -i "s/WREN_AI_SERVICE_VERSION=.*/WREN_AI_SERVICE_VERSION=${{ github.event.inputs.ai_version }}/" $FILE_PATH - sed -i "s/WREN_UI_VERSION=.*/WREN_UI_VERSION=${{ github.event.inputs.ui_version }}/" $FILE_PATH - - # Verify the changes - grep "VERSION" $FILE_PATH - - - name: Show git status - run: git status - - - name: Show git diff - run: | - echo "===== Git diff for changed files =====" - git diff - - - name: Create PR - uses: peter-evans/create-pull-request@v7 - with: - base: main - branch: "release/${{ github.event.inputs.release_version }}" - commit-message: "release ${{ github.event.inputs.release_version }}" - title: "Release ${{ github.event.inputs.release_version }}" - body: "Release ${{ github.event.inputs.release_version }}" - draft: false diff --git a/.github/workflows/create-rc-release.yaml b/.github/workflows/create-rc-release.yaml deleted file mode 100644 index 64ab80762..000000000 --- a/.github/workflows/create-rc-release.yaml +++ /dev/null @@ -1,97 +0,0 @@ -name: Create RC Release - -on: - workflow_dispatch: - inputs: - release_version: - description: "Release version" - required: true - issue_comment: - types: [created] - -jobs: - release: - runs-on: macos-latest - if: ${{ github.event_name == 'issue_comment' && contains(github.event.comment.body, '/release-rc') && startsWith(github.event.issue.title, 'Release') }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: main - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: "1.25" - - - name: Add rocket emoji to comment - run: | - curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -d '{"body": "🚀 Starting the release process!"}' \ - "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" - - - name: Parse release version from PR title - id: parse_release_version - env: - GITHUB_ISSUE_TITLE: ${{ github.event.issue.title }} - run: | - release_version=$(echo "$GITHUB_ISSUE_TITLE" | sed 's/ /\n/g' | tail -n 1) - echo "Release version: $release_version" - echo "release_version=$release_version" >> $GITHUB_OUTPUT - - - name: Build for macOS - working-directory: ./wren-launcher - run: | - mkdir -p dist - env GOARCH=amd64 GOOS=darwin CGO_ENABLED=1 go build -o dist/wren-launcher-darwin main.go - cd dist && chmod +x wren-launcher-darwin && tar zcvf wren-launcher-darwin.tar.gz wren-launcher-darwin - - - name: Build for macOS(arm64) - working-directory: ./wren-launcher - run: | - mkdir -p dist - env GOARCH=arm64 GOOS=darwin CGO_ENABLED=1 go build -o dist/wren-launcher-darwin-arm64 main.go - cd dist && chmod +x wren-launcher-darwin-arm64 && tar zcvf wren-launcher-darwin-arm64.tar.gz wren-launcher-darwin-arm64 - - - name: Build for Linux - working-directory: ./wren-launcher - run: | - mkdir -p dist - env GOARCH=amd64 GOOS=linux CGO_ENABLED=0 go build -o dist/wren-launcher-linux main.go - cd dist && chmod +x wren-launcher-linux && tar zcvf wren-launcher-linux.tar.gz wren-launcher-linux - - - name: Build for Linux(arm64) - working-directory: ./wren-launcher - run: | - mkdir -p dist - env GOARCH=arm64 GOOS=linux CGO_ENABLED=0 go build -o dist/wren-launcher-linux-arm64 main.go - cd dist && chmod +x wren-launcher-linux-arm64 && tar zcvf wren-launcher-linux-arm64.tar.gz wren-launcher-linux-arm64 - - - name: Build for Windows - working-directory: ./wren-launcher - run: | - mkdir -p dist - env GOARCH=amd64 GOOS=windows CGO_ENABLED=0 go build -o dist/wren-launcher-windows.exe main.go - cd dist && zip wren-launcher-windows.zip wren-launcher-windows.exe - - - name: Create GitHub Release - uses: softprops/action-gh-release@v2 - with: - files: | - ./wren-launcher/dist/wren-launcher-darwin.tar.gz - ./wren-launcher/dist/wren-launcher-linux.tar.gz - ./wren-launcher/dist/wren-launcher-windows.zip - ./wren-launcher/dist/wren-launcher-darwin-arm64.tar.gz - ./wren-launcher/dist/wren-launcher-linux-arm64.tar.gz - tag_name: ${{ steps.parse_release_version.outputs.release_version }} - prerelease: true - generate_release_notes: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Comment with release link - run: | - release_url="https://github.com/${{ github.repository }}/releases/tag/${{ steps.parse_release_version.outputs.release_version }}" - curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -d "{\"body\": \"🚀 A new release has been created! [View Release](${release_url})\"}" \ - "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml new file mode 100644 index 000000000..3690824c1 --- /dev/null +++ b/.github/workflows/labeler.yaml @@ -0,0 +1,12 @@ +name: 'Pull Request Labeler' +on: +- pull_request_target + +jobs: + labeler: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v5 diff --git a/.github/workflows/pr-tagger.yaml b/.github/workflows/pr-tagger.yaml deleted file mode 100644 index 7990e05df..000000000 --- a/.github/workflows/pr-tagger.yaml +++ /dev/null @@ -1,147 +0,0 @@ -name: PR Tagger - -on: - pull_request: - types: [opened, synchronize, reopened] - paths: - - "wren-ui/src/**" - - "deployment/**" - - "wren-launcher/**" - - "wren-ai-service/**" - - "docker/**" - -jobs: - tag-pr: - runs-on: ubuntu-latest - permissions: - pull-requests: write - - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v46 - - - name: Tag PR based on changed files and title - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const changedFiles = '${{ steps.changed-files.outputs.all_changed_files }}'.split(' '); - const labels = new Set(); - - // Define label colors - const labelColors = { - 'wren-ui': 'f29513', // Orange - 'launcher': 'f29513', // Orange (same as wren-ui) - 'wren-ai-service': 'f29513', // Orange (same as wren-ui) - 'deployment': '0075ca', // Blue - 'ai-env-changed': 'd73a4a', // Red - 'ui-env-changed': 'd73a4a' // Red - }; - - - // Check for files in specific directories - for (const file of changedFiles) { - if (file.startsWith('wren-ui/src/')) { - labels.add('wren-ui'); - } else if (file.startsWith('deployment/')) { - labels.add('deployment'); - } else if (file.startsWith('wren-launcher/')) { - labels.add('launcher'); - } else if (file.startsWith('wren-ai-service/')) { - labels.add('wren-ai-service'); - } - if (file.startsWith('docker/') && file.includes('.example')) { - labels.add('ai-env-changed'); - } - if (file.startsWith('wren-ui/src/apollo/server/config.ts')) { - labels.add('ui-env-changed'); - } - } - - // Extract milestone from PR title - const prTitle = context.payload.pull_request.title; - const milestoneMatch = prTitle.match(/\(milestone:\s*([^)]+)\)/i); - - if (milestoneMatch && milestoneMatch[1]) { - const milestone = milestoneMatch[1].trim(); - if (milestone) { - labels.add(milestone); - console.log(`Found milestone in PR title: ${milestone}`); - } - } - - // Append env changed label to PR title - if (labels.has('ai-env-changed')) { - // check if pr title contains "ai-env-changed" - if (!prTitle.includes('ai-env-changed')) { - const newTitle = `${prTitle} (ai-env-changed)`; - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - title: newTitle - }); - } - } - - if (labels.has('ui-env-changed')) { - // check if pr title contains "ui-env-changed" - if (!prTitle.includes('ui-env-changed')) { - const newTitle = `${prTitle} (ui-env-changed)`; - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - title: newTitle - }); - } - } - - if (labels.size > 0) { - // Create or update labels with colors - for (const label of labels) { - try { - // Try to create the label with color - if (labelColors[label]) { - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: label, - color: labelColors[label] - }); - } - } catch (error) { - // Label already exists, update its color if needed - if (labelColors[label]) { - try { - await github.rest.issues.updateLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: label, - color: labelColors[label] - }); - } catch (updateError) { - console.log(`Could not update color for label ${label}: ${updateError.message}`); - } - } - } - } - - // Add labels to PR - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - labels: Array.from(labels) - }); - - console.log(`Added labels: ${Array.from(labels).join(', ')}`); - } else { - console.log('No matching directories or milestone found'); - } diff --git a/.github/workflows/publish-wren-core-py.yml b/.github/workflows/publish-wren-core-py.yml new file mode 100644 index 000000000..4222d0a0e --- /dev/null +++ b/.github/workflows/publish-wren-core-py.yml @@ -0,0 +1,170 @@ +name: Publish wren-core-py to PyPI + +on: + workflow_call: + inputs: + version: + description: "Version number (e.g. 0.2.0)" + required: true + type: string + tag_name: + description: "Git tag to checkout (e.g. wren-core-py-v0.2.0)" + required: true + type: string + pypi_target: + description: "Publish target (pypi or testpypi)" + required: false + type: string + default: "pypi" + +permissions: + contents: read + id-token: write + +jobs: + build-wheels: + name: Build wheel — ${{ matrix.os }} (${{ matrix.target }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + target: x86_64-unknown-linux-gnu + - os: macos-15-intel + target: x86_64-apple-darwin + - os: macos-15 + target: aarch64-apple-darwin + - os: windows-latest + target: x86_64-pc-windows-msvc + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag_name }} + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Set version + env: + VERSION: ${{ inputs.version }} + shell: python + run: | + import re, os, sys + version = os.environ["VERSION"] + if re.fullmatch(r"\d+\.\d+\.\d+", version): + cargo_version = version + elif re.fullmatch(r"\d+\.\d+\.\d+rc\d+", version): + cargo_version = re.sub(r"(\d+\.\d+\.\d+)rc(\d+)", r"\1-rc.\2", version) + else: + print(f"::error::Unsupported version format: {version!r}. Expected X.Y.Z or X.Y.ZrcN.") + sys.exit(1) + for path, ver in [ + ("core/wren-core-py/Cargo.toml", cargo_version), + ("core/wren-core-py/pyproject.toml", version), + ]: + text = open(path).read() + text, n = re.subn(r'^(version\s*=\s*)".*?"', rf'\1"{ver}"', text, count=1, flags=re.MULTILINE) + if n != 1: + print(f"::error::Failed to update version in {path}") + sys.exit(1) + open(path, "w").write(text) + lock = open("core/wren-core-py/Cargo.lock").read() + lock, n = re.subn( + r'(name = "wren-core-py"\nversion = )".*?"', + rf'\1"{cargo_version}"', + lock, count=1, + ) + if n != 1: + print("::error::Failed to update wren-core-py version in Cargo.lock") + sys.exit(1) + open("core/wren-core-py/Cargo.lock", "w").write(lock) + - name: Build wheel + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist + manylinux: auto + working-directory: core/wren-core-py + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.target }} + path: core/wren-core-py/dist/*.whl + + build-sdist: + name: Build sdist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag_name }} + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Set version + env: + VERSION: ${{ inputs.version }} + shell: python + run: | + import re, os, sys + version = os.environ["VERSION"] + if re.fullmatch(r"\d+\.\d+\.\d+", version): + cargo_version = version + elif re.fullmatch(r"\d+\.\d+\.\d+rc\d+", version): + cargo_version = re.sub(r"(\d+\.\d+\.\d+)rc(\d+)", r"\1-rc.\2", version) + else: + print(f"::error::Unsupported version format: {version!r}. Expected X.Y.Z or X.Y.ZrcN.") + sys.exit(1) + for path, ver in [ + ("core/wren-core-py/Cargo.toml", cargo_version), + ("core/wren-core-py/pyproject.toml", version), + ]: + text = open(path).read() + text, n = re.subn(r'^(version\s*=\s*)".*?"', rf'\1"{ver}"', text, count=1, flags=re.MULTILINE) + if n != 1: + print(f"::error::Failed to update version in {path}") + sys.exit(1) + open(path, "w").write(text) + lock = open("core/wren-core-py/Cargo.lock").read() + lock, n = re.subn( + r'(name = "wren-core-py"\nversion = )".*?"', + rf'\1"{cargo_version}"', + lock, count=1, + ) + if n != 1: + print("::error::Failed to update wren-core-py version in Cargo.lock") + sys.exit(1) + open("core/wren-core-py/Cargo.lock", "w").write(lock) + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + working-directory: core/wren-core-py + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: sdist + path: core/wren-core-py/dist/*.tar.gz + + publish: + name: Publish to ${{ inputs.pypi_target }} + needs: [build-wheels, build-sdist] + runs-on: ubuntu-latest + environment: + name: ${{ inputs.pypi_target }} + url: ${{ inputs.pypi_target == 'pypi' && 'https://pypi.org/project/wren-core-py/' || 'https://test.pypi.org/project/wren-core-py/' }} + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - name: List artifacts + run: ls -lhR dist/ + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: ${{ inputs.pypi_target == 'testpypi' && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }} + packages-dir: dist/ + attestations: false diff --git a/.github/workflows/publish-wren-core-wasm.yml b/.github/workflows/publish-wren-core-wasm.yml new file mode 100644 index 000000000..78f5bbe60 --- /dev/null +++ b/.github/workflows/publish-wren-core-wasm.yml @@ -0,0 +1,90 @@ +name: Publish wren-core-wasm to npm + +on: + workflow_call: + inputs: + version: + description: "Version number (e.g. 0.2.0)" + required: true + type: string + tag_name: + description: "Git tag to checkout (e.g. wren-core-wasm-v0.2.0)" + required: true + type: string + npm_tag: + description: "npm dist-tag (latest or rc)" + required: false + type: string + default: "latest" + secrets: + NPM_TOKEN: + required: true + +permissions: + contents: read + +jobs: + build-and-publish: + name: Build WASM + publish to npm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag_name }} + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + core/wren-core-wasm/target/ + key: wasm-cargo-${{ hashFiles('core/wren-core-wasm/Cargo.toml', 'core/wren-core-wasm/Cargo.lock') }} + restore-keys: wasm-cargo- + + - name: Install wasm-pack + uses: taiki-e/install-action@v2 + with: + tool: wasm-pack@0.14.0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + registry-url: https://registry.npmjs.org + + - name: Install npm dependencies + working-directory: core/wren-core-wasm + run: npm install + + - name: Set version in package.json + working-directory: core/wren-core-wasm + env: + VERSION: ${{ inputs.version }} + run: npm version "$VERSION" --no-git-tag-version + + - name: Build WASM (wasm-pack) + working-directory: core/wren-core-wasm + run: wasm-pack build --target web --release + + - name: Build dist (TypeScript + copy) + working-directory: core/wren-core-wasm + run: npm run build:dist + + - name: Run integration tests + working-directory: core/wren-core-wasm + run: npm test + + - name: Publish to npm + working-directory: core/wren-core-wasm + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + NPM_TAG: ${{ inputs.npm_tag }} + run: npm publish --tag "$NPM_TAG" --access public diff --git a/.github/workflows/publish-wren.yml b/.github/workflows/publish-wren.yml new file mode 100644 index 000000000..aa59ba213 --- /dev/null +++ b/.github/workflows/publish-wren.yml @@ -0,0 +1,80 @@ +name: Publish wren-engine to PyPI + +on: + workflow_call: + inputs: + version: + description: "Version number (e.g. 0.3.0)" + required: true + type: string + tag_name: + description: "Git tag to checkout (e.g. wren-v0.3.0)" + required: true + type: string + pypi_target: + description: "Publish target (pypi or testpypi)" + required: false + type: string + default: "pypi" + +permissions: + contents: read + id-token: write + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag_name }} + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Set version + env: + VERSION: ${{ inputs.version }} + shell: python + run: | + import re, os + version = os.environ["VERSION"] + for path, pattern in [ + ("core/wren/pyproject.toml", r'^(version\s*=\s*)".*?"'), + ("core/wren/src/wren/__init__.py", r'^(__version__\s*=\s*)".*?"'), + ]: + text = open(path).read() + text = re.sub(pattern, rf'\1"{version}"', text, count=1, flags=re.MULTILINE) + open(path, "w").write(text) + - name: Install build tool + run: pip install build + - name: Build sdist and wheel + run: python -m build + working-directory: core/wren + - name: Upload distributions + uses: actions/upload-artifact@v4 + with: + name: dist + path: core/wren/dist/ + + publish: + name: Publish to ${{ inputs.pypi_target }} + needs: build + runs-on: ubuntu-latest + environment: + name: ${{ inputs.pypi_target }} + url: ${{ inputs.pypi_target == 'pypi' && 'https://pypi.org/project/wren-engine/' || 'https://test.pypi.org/project/wren-engine/' }} + steps: + - name: Download distributions + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - name: List artifacts + run: ls -lhR dist/ + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: ${{ inputs.pypi_target == 'testpypi' && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }} + packages-dir: dist/ + attestations: false diff --git a/.github/workflows/pull-request-title-validator.yaml b/.github/workflows/pull-request-title-validator.yaml index 6719b2fa8..a9a996d23 100644 --- a/.github/workflows/pull-request-title-validator.yaml +++ b/.github/workflows/pull-request-title-validator.yaml @@ -1,12 +1,11 @@ -name: Pull Request Title Validator +name: Pull request title validator on: pull_request: - paths: - - "wren-ai-service/**" - - ".github/workflows/pull-request-title-validator.yaml" - - ".github/workflows/ai-service-*.yaml" - types: [opened, edited, synchronize] + types: + - opened + - edited + - synchronize permissions: pull-requests: read @@ -17,7 +16,6 @@ jobs: runs-on: ubuntu-latest steps: - name: validate pull request title - uses: kontrolplane/pull-request-title-validator@v1.3.1 + uses: kontrolplane/pull-request-title-validator@v1.3.2 with: - types: "fix,feat,chore" - scopes: "wren-ai-service" \ No newline at end of file + types: "feat,fix,perf,deps,revert,docs,chore,refactor,test,build,ci" diff --git a/.github/workflows/rc-release.yml b/.github/workflows/rc-release.yml new file mode 100644 index 000000000..6f64a18e7 --- /dev/null +++ b/.github/workflows/rc-release.yml @@ -0,0 +1,144 @@ +name: RC Release + +on: + workflow_dispatch: + inputs: + component: + description: "Component to release" + required: true + type: choice + options: + - wren-core-py + - wren + - wren-core-wasm + rc_version: + description: "RC version override (e.g. 0.25.0-rc.2). Leave empty to auto-increment." + required: false + type: string + +permissions: + contents: read + +concurrency: + group: rc-release-${{ inputs.component }} + +jobs: + create-rc: + runs-on: ubuntu-latest + permissions: + contents: write + outputs: + version: ${{ steps.rc.outputs.version }} + tag_name: ${{ steps.rc.outputs.tag_name }} + pypi_version: ${{ steps.rc.outputs.pypi_version }} + steps: + - uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + - name: Ensure running on main + run: | + if [ "$GITHUB_REF" != "refs/heads/main" ]; then + echo "::error::RC releases must be dispatched from the main branch" + exit 1 + fi + - name: Determine RC version + id: rc + env: + RC_VERSION_INPUT: ${{ inputs.rc_version }} + COMPONENT: ${{ inputs.component }} + run: | + # Read current version from manifest + BASE_VERSION=$(python3 -c " + import json + with open('.release-please-manifest.json') as f: + manifest = json.load(f) + print(manifest['${COMPONENT}']) + ") + + if [ -n "$RC_VERSION_INPUT" ]; then + # Validate: must start with BASE_VERSION-rc. and end with a positive integer + EXPECTED_PREFIX="${BASE_VERSION}-rc." + SUFFIX="${RC_VERSION_INPUT#"$EXPECTED_PREFIX"}" + if [ "$SUFFIX" = "$RC_VERSION_INPUT" ] || ! [[ "$SUFFIX" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Invalid rc_version: $RC_VERSION_INPUT (expected: ${BASE_VERSION}-rc.N)" + exit 1 + fi + VERSION="$RC_VERSION_INPUT" + else + # Find the latest RC tag for this component and bump + LATEST_RC=$(git tag -l "${COMPONENT}-v${BASE_VERSION}-rc.*" --sort=-v:refname | head -n1) + if [ -z "$LATEST_RC" ]; then + RC_NUM=1 + else + # Extract RC number from tag like "wren-core-py-v0.5.0-rc.3" + RC_NUM=$(echo "$LATEST_RC" | grep -oP 'rc\.\K[0-9]+') + RC_NUM=$((RC_NUM + 1)) + fi + VERSION="${BASE_VERSION}-rc.${RC_NUM}" + fi + + TAG_NAME="${COMPONENT}-v${VERSION}" + + # PEP 440 format for PyPI: 0.25.0-rc.1 → 0.25.0rc1 + PYPI_VERSION=$(echo "$VERSION" | sed 's/-rc\.\([0-9]*\)/rc\1/') + + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "tag_name=$TAG_NAME" >> "$GITHUB_OUTPUT" + echo "pypi_version=$PYPI_VERSION" >> "$GITHUB_OUTPUT" + echo "::notice::Creating RC release: $TAG_NAME (PyPI: $PYPI_VERSION)" + + - name: Create tag + env: + TAG_NAME: ${{ steps.rc.outputs.tag_name }} + run: | + git tag "$TAG_NAME" + git push origin "$TAG_NAME" + + publish-wren-core-py: + needs: create-rc + if: inputs.component == 'wren-core-py' + uses: ./.github/workflows/publish-wren-core-py.yml + with: + version: ${{ needs.create-rc.outputs.pypi_version }} + tag_name: ${{ needs.create-rc.outputs.tag_name }} + permissions: + contents: read + id-token: write + + publish-wren: + needs: create-rc + if: inputs.component == 'wren' + uses: ./.github/workflows/publish-wren.yml + with: + version: ${{ needs.create-rc.outputs.pypi_version }} + tag_name: ${{ needs.create-rc.outputs.tag_name }} + permissions: + contents: read + id-token: write + + publish-wren-core-wasm: + needs: create-rc + if: inputs.component == 'wren-core-wasm' + uses: ./.github/workflows/publish-wren-core-wasm.yml + with: + version: ${{ needs.create-rc.outputs.version }} + tag_name: ${{ needs.create-rc.outputs.tag_name }} + npm_tag: rc + secrets: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + + create-release: + needs: [create-rc, publish-wren-core-py, publish-wren, publish-wren-core-wasm] + if: always() && needs.create-rc.result == 'success' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Create GitHub pre-release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ needs.create-rc.outputs.tag_name }} + name: "${{ inputs.component }} ${{ needs.create-rc.outputs.version }}" + prerelease: true + generate_release_notes: true diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 000000000..e7292dcde --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,62 @@ +name: Release Please + +on: + push: + branches: + - main + +permissions: + contents: write + pull-requests: write + +jobs: + release-please: + runs-on: ubuntu-latest + outputs: + wren-core-py--release_created: ${{ steps.release.outputs['wren-core-py--release_created'] }} + wren-core-py--tag_name: ${{ steps.release.outputs['wren-core-py--tag_name'] }} + wren-core-py--version: ${{ steps.release.outputs['wren-core-py--version'] }} + wren--release_created: ${{ steps.release.outputs['wren--release_created'] }} + wren--tag_name: ${{ steps.release.outputs['wren--tag_name'] }} + wren--version: ${{ steps.release.outputs['wren--version'] }} + wren-core-wasm--release_created: ${{ steps.release.outputs['wren-core-wasm--release_created'] }} + wren-core-wasm--tag_name: ${{ steps.release.outputs['wren-core-wasm--tag_name'] }} + wren-core-wasm--version: ${{ steps.release.outputs['wren-core-wasm--version'] }} + steps: + - uses: googleapis/release-please-action@v4 + id: release + with: + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + + publish-wren-core-py: + needs: release-please + if: needs.release-please.outputs['wren-core-py--release_created'] == 'true' + uses: ./.github/workflows/publish-wren-core-py.yml + with: + version: ${{ needs.release-please.outputs['wren-core-py--version'] }} + tag_name: ${{ needs.release-please.outputs['wren-core-py--tag_name'] }} + permissions: + contents: read + id-token: write + + publish-wren: + needs: release-please + if: needs.release-please.outputs['wren--release_created'] == 'true' + uses: ./.github/workflows/publish-wren.yml + with: + version: ${{ needs.release-please.outputs['wren--version'] }} + tag_name: ${{ needs.release-please.outputs['wren--tag_name'] }} + permissions: + contents: read + id-token: write + + publish-wren-core-wasm: + needs: release-please + if: needs.release-please.outputs['wren-core-wasm--release_created'] == 'true' + uses: ./.github/workflows/publish-wren-core-wasm.yml + with: + version: ${{ needs.release-please.outputs['wren-core-wasm--version'] }} + tag_name: ${{ needs.release-please.outputs['wren-core-wasm--tag_name'] }} + secrets: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 000000000..2038e3d8d --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,131 @@ +name: Rust +permissions: + contents: read + pull-requests: write + +on: + push: + branches: + - main + paths: + - 'core/wren-core/**' + pull_request: + paths: + - 'core/wren-core/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +defaults: + run: + working-directory: core/wren-core + +jobs: + # Check crate compiles + linux-build-lib: + name: cargo check + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/rust/setup-builder + with: + rust-version: stable + + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./core/wren-core/target/ + # this key equals the ones on `linux-build-lib` for re-use + key: cargo-cache-benchmark-${{ hashFiles('core/wren-core/Cargo.toml') }} + + - name: Check datafusion-common without default features + run: cargo check --all-targets + + # Run tests + linux-test: + name: cargo test (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/rust/setup-builder + with: + rust-version: stable + - name: Run tests (excluding doctests) + run: RUST_MIN_STACK=8388608 cargo test --lib --tests --bins + - name: Verify Working Directory Clean + run: git diff --exit-code + + # Run tests + linux-test-arm64: + name: cargo test (arm64) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + needs: [ linux-build-lib ] + runs-on: ubuntu-24.04-arm + container: + image: arm64v8/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/rust/setup-builder + with: + rust-version: stable + - name: Run tests (excluding doctests) + run: RUST_MIN_STACK=8388608 cargo test --lib --tests --bins + - name: Verify Working Directory Clean + run: git diff --exit-code + + clippy: + name: clippy + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/rust/setup-builder + with: + rust-version: stable + - name: Install Clippy + run: rustup component add clippy + - name: Run clippy + run: cargo clippy --all-targets --all-features -- -D warnings + + cargo-toml-formatting-checks: + name: check Cargo.toml formatting + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/rust/setup-builder + with: + rust-version: stable + - name: Install taplo + run: cargo +stable install taplo-cli --version ^0.9 --locked + # if you encounter an error, try running 'taplo format' to fix the formatting automatically. + - name: Check Cargo.toml formatting + run: taplo format --check diff --git a/.github/workflows/skills-check.yml b/.github/workflows/skills-check.yml new file mode 100644 index 000000000..cb9bbb795 --- /dev/null +++ b/.github/workflows/skills-check.yml @@ -0,0 +1,19 @@ +name: Skills Version Check + +on: + pull_request: + paths: + - "skills/**" + +permissions: + contents: read + +jobs: + version-parity: + name: Check skills/versions.json parity + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Verify versions.json matches SKILL.md frontmatter + run: bash skills/check-versions.sh diff --git a/.github/workflows/sync-docs.yml b/.github/workflows/sync-docs.yml new file mode 100644 index 000000000..da86d09a7 --- /dev/null +++ b/.github/workflows/sync-docs.yml @@ -0,0 +1,61 @@ +name: Sync Docs to Website + +on: + push: + branches: [main] + paths: + - 'docs/core/get_started/**' + - 'docs/core/concept/**' + - 'docs/core/guide/**' + - 'docs/core/reference/**' + +permissions: + contents: read + +jobs: + sync-docs: + runs-on: ubuntu-latest + steps: + - name: Checkout wren-engine + uses: actions/checkout@v4 + + - name: Checkout doc website + uses: actions/checkout@v4 + with: + repository: ${{ vars.DOCS_REPO }} + token: ${{ secrets.CROSS_REPO_TOKEN }} + path: _docs-site + ref: ${{ vars.DOCS_REPO_BRANCH }} + + - name: Sync doc directories + run: | + TARGET="_docs-site/docs/oss/engine" + + for dir in get_started concept guide reference; do + rm -rf "${TARGET}/${dir}" + cp -r "docs/core/${dir}" "${TARGET}/${dir}" + done + + - name: Check for changes + id: diff + working-directory: _docs-site + run: | + git diff --quiet && echo "changed=false" >> "$GITHUB_OUTPUT" || echo "changed=true" >> "$GITHUB_OUTPUT" + + - name: Create PR + if: steps.diff.outputs.changed == 'true' + working-directory: _docs-site + env: + GH_TOKEN: ${{ secrets.CROSS_REPO_TOKEN }} + run: | + BRANCH="sync/engine-docs-${GITHUB_SHA::8}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b "${BRANCH}" + git add -A + git commit -m "docs: sync from wren-engine@${GITHUB_SHA::8}" + git push origin "${BRANCH}" + gh pr create \ + --title "docs: sync Wren Engine docs from wren-engine" \ + --body "Auto-synced from [wren-engine@\`${GITHUB_SHA::8}\`](https://github.com/Canner/wren-engine/commit/${GITHUB_SHA})." \ + --base "${{ vars.DOCS_REPO_BRANCH }}" diff --git a/.github/workflows/ui-lint.yaml b/.github/workflows/ui-lint.yaml deleted file mode 100644 index 7d335b86e..000000000 --- a/.github/workflows/ui-lint.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# create the github action to run the yarn lint when PR created or pushed to main branch -# - -name: Wren-UI Lint - -on: - pull_request: - types: [ labeled, synchronize ] - -permissions: - contents: read - -concurrency: - # avoid mis-canceling the ci runs while other labels are added to the PR, so we add the label name as the condition - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.label.name == 'ci/ui' && github.event.number || github.sha }} - cancel-in-progress: true - -defaults: - run: - working-directory: wren-ui - -jobs: - eslint: - # run this job only if the PR is labeled with "ci/ui" - if: ${{ github.event.label.name == 'ci/ui' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Node.js - uses: actions/setup-node@v2 - with: - node-version: '18' - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 - id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) - with: - path: | # should cache node_modules as well - ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - if: ${{ steps.yarn-cache.outputs.cache-hit != 'true' }} - name: List the state of node modules - continue-on-error: true - run: yarn list - - name: Install Node.js dependencies - run: yarn install - - name: Run lint - run: yarn lint - \ No newline at end of file diff --git a/.github/workflows/ui-release-image-stable.yaml b/.github/workflows/ui-release-image-stable.yaml deleted file mode 100644 index b25bbce3a..000000000 --- a/.github/workflows/ui-release-image-stable.yaml +++ /dev/null @@ -1,150 +0,0 @@ -name: Wren-UI Release stable image - -on: - workflow_dispatch: - inputs: - version: - description: Give a version for this release - type: string - required: true - -env: - WREN_UI_IMAGE: ghcr.io/canner/wren-ui - -defaults: - run: - working-directory: wren-ui - -jobs: - build-image: - strategy: - fail-fast: false - matrix: - arch: - - runner: ubuntu-latest - platform: linux/amd64 - - runner: linux_arm64_runner - platform: linux/arm64 - runs-on: ${{ matrix.arch.runner }} - steps: - - uses: actions/checkout@v4 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Prepare platform - run: | - platform=${{ matrix.arch.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - platforms: ${{ matrix.arch.platform }} - labels: ${{ env.WREN_UI_IMAGE }} - context: ./wren-ui - outputs: type=image,name=${{ env.WREN_UI_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - merge: - runs-on: ubuntu-latest - needs: [build-image] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.WREN_UI_IMAGE }} - tags: | - ${{ github.event.inputs.version }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Create manifest list and push - working-directory: /tmp/digests - # tag with input version and latest - run: | - TAGS=$(echo "${{ steps.meta.outputs.tags }}" | awk '{printf "--tag %s ", $0}') - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.WREN_UI_IMAGE }}@sha256:%s ' *) \ - $TAGS - tag-ui-version: - runs-on: ubuntu-latest - needs: [merge] - steps: - - name: Generate a token - id: generate-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ vars.CI_APP_ID }} - private-key: ${{ secrets.CI_APP_PRIVATE_KEY }} - - uses: actions/checkout@v4 - with: - token: ${{ steps.generate-token.outputs.token }} - fetch-depth: 0 - - name: Set up Git - run: | - git config --global user.name "wren-ai[bot]" - git config --global user.email "dev@cannerdata.com" - - name: Generate and Save Change Log - id: changelog - run: | - echo "Generating change log..." - PREVIOUS_VERSION=release/ui/$(jq -r '.version' package.json) - echo "Previous version: $PREVIOUS_VERSION" - echo "================ change log ================" - CHANGE_LOG=$(git log --pretty=format:"%s" $PREVIOUS_VERSION..HEAD | grep wren-ui) - echo "================ change log ================" - - # Separate the change log into categories - FEATURES=$(echo "$CHANGE_LOG" | grep "^feat" | sed 's/^/- /') - FIXES_AND_CHORES=$(echo "$CHANGE_LOG" | grep -E "^(fix|chore)" | sed 's/^/- /') - - # Create the full change log - FULL_CHANGE_LOG="\nChangelog for the version\n" - if [ -n "$FEATURES" ]; then - FULL_CHANGE_LOG+="\nFeature and Enhancement\n$FEATURES" - fi - if [ -n "$FIXES_AND_CHORES" ]; then - FULL_CHANGE_LOG+="\n\nFixes and Chores\n$FIXES_AND_CHORES" - fi - - { - echo "CHANGE_LOG<> $GITHUB_ENV - - name: Update Wren-UI version - run: | - version=${{ github.event.inputs.version }} - sed -i 's/"version": "[^"]*"/"version": "'"$version"'"/' package.json - git add package.json - git commit -m "update wren-ui version to $version" - git push - git tag -a "release/ui/$version" -m "${{ env.CHANGE_LOG }}" - git push origin "release/ui/$version" diff --git a/.github/workflows/ui-release-image.yaml b/.github/workflows/ui-release-image.yaml deleted file mode 100644 index ea98f8f8c..000000000 --- a/.github/workflows/ui-release-image.yaml +++ /dev/null @@ -1,104 +0,0 @@ -name: Wren-UI Release image - -on: - workflow_dispatch: - inputs: - docker_image_tag_name: - description: Docker image tag name (optional, default branch name/tag name replaced with hyphens) - type: string - -env: - WREN_UI_IMAGE: ghcr.io/canner/wren-ui - -defaults: - run: - working-directory: wren-ui - -jobs: - build-image: - outputs: - tag_name: ${{ steps.tag-preparation.outputs.TAG_NAME }} - strategy: - fail-fast: false - matrix: - arch: - - runner: ubuntu-latest - platform: linux/amd64 - - runner: linux_arm64_runner - platform: linux/arm64 - runs-on: ${{ matrix.arch.runner }} - steps: - - uses: actions/checkout@v4 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Prepare tag name - id: tag-preparation - run: | - if [ -n "${{ github.event.inputs.docker_image_tag_name }}" ]; then - tag_name=${{ github.event.inputs.docker_image_tag_name }} - else - tag_name=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')-$(git log -1 --pretty=%h) - fi - echo "TAG_NAME=$tag_name" >> $GITHUB_OUTPUT - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Prepare platform - run: | - platform=${{ matrix.arch.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - platforms: ${{ matrix.arch.platform }} - labels: ${{ env.WREN_UI_IMAGE }} - context: ./wren-ui - outputs: type=image,name=${{ env.WREN_UI_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - merge: - runs-on: ubuntu-latest - needs: [ build-image ] - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.WREN_UI_IMAGE }} - tags: | - ${{ needs.build-image.outputs.tag_name }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - TAGS=$(echo "${{ steps.meta.outputs.tags }}" | awk '{printf "--tag %s ", $0}') - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.WREN_UI_IMAGE }}@sha256:%s ' *) \ - $TAGS diff --git a/.github/workflows/ui-test.yaml b/.github/workflows/ui-test.yaml deleted file mode 100644 index 5e6320941..000000000 --- a/.github/workflows/ui-test.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# create the github action to run the UI tests when PR created - -name: Wren-UI Test - -on: - pull_request: - types: [ labeled, synchronize ] - -permissions: - contents: read - -concurrency: - # avoid mis-canceling the ci runs while other labels are added to the PR, so we add the label name as the condition - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.label.name == 'ci/ui' && github.event.number || github.sha }} - cancel-in-progress: true - -defaults: - run: - working-directory: wren-ui - -jobs: - unit-test: - # run this job only if the PR is labeled with "ci/ui" - if: ${{ github.event.label.name == 'ci/ui'}} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Node.js - uses: actions/setup-node@v2 - with: - node-version: '18' - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 - id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) - with: - path: | # should cache node_modules as well - ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - if: ${{ steps.yarn-cache.cache-hit != 'true' }} - name: List the state of node modules - continue-on-error: true - run: yarn list - - name: Install dependencies - run: yarn - - name: Run tests - run: yarn test \ No newline at end of file diff --git a/.github/workflows/wasm-ci.yml b/.github/workflows/wasm-ci.yml new file mode 100644 index 000000000..0bef5e4fd --- /dev/null +++ b/.github/workflows/wasm-ci.yml @@ -0,0 +1,99 @@ +name: WASM SDK +permissions: + contents: read + +on: + push: + branches: + - main + paths: + - 'core/wren-core-wasm/**' + - 'core/wren-core/core/**' + - 'core/wren-core-base/**' + pull_request: + paths: + - 'core/wren-core-wasm/**' + - 'core/wren-core/core/**' + - 'core/wren-core-base/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + build: + name: Build WASM + TypeScript SDK + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + core/wren-core-wasm/target/ + key: wasm-cargo-${{ hashFiles('core/wren-core-wasm/Cargo.toml', 'core/wren-core-wasm/Cargo.lock') }} + restore-keys: wasm-cargo- + + - name: Install wasm-pack + uses: taiki-e/install-action@v2 + with: + tool: wasm-pack@0.14.0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install npm dependencies + working-directory: core/wren-core-wasm + run: npm install + + - name: Build WASM (wasm-pack) + working-directory: core/wren-core-wasm + run: wasm-pack build --target web --release + + - name: Build dist (TypeScript + copy) + working-directory: core/wren-core-wasm + run: npm run build:dist + + - name: TypeScript typecheck + working-directory: core/wren-core-wasm + run: npm run typecheck + + - name: WASM binary size check + working-directory: core/wren-core-wasm + run: | + raw_size=$(stat -c%s dist/wren_core_wasm_bg.wasm) + gzip_size=$(gzip -c dist/wren_core_wasm_bg.wasm | wc -c) + raw_mb=$(echo "scale=1; $raw_size / 1048576" | bc) + gzip_mb=$(echo "scale=1; $gzip_size / 1048576" | bc) + echo "WASM binary: ${raw_mb} MB raw, ${gzip_mb} MB gzip" + + # Fail if gzip > 15 MB + max_gzip=$((15 * 1048576)) + if [ "$gzip_size" -gt "$max_gzip" ]; then + echo "::error::WASM binary gzip size (${gzip_mb} MB) exceeds 15 MB limit" + exit 1 + fi + + - name: Run integration tests + working-directory: core/wren-core-wasm + run: npm test + + - name: Upload dist artifact + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + uses: actions/upload-artifact@v4 + with: + name: wasm-sdk-dist + path: core/wren-core-wasm/dist/ + retention-days: 30 diff --git a/.github/workflows/wren-ci.yml b/.github/workflows/wren-ci.yml new file mode 100644 index 000000000..5697077a2 --- /dev/null +++ b/.github/workflows/wren-ci.yml @@ -0,0 +1,207 @@ +name: Wren SDK CI +permissions: + contents: read + pull-requests: write + +on: + pull_request: + paths: + - 'core/wren/**' + - 'core/wren-core/**' + - 'core/wren-core-py/**' + - 'core/wren-core-base/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number }} + cancel-in-progress: true + +jobs: + lint: + name: lint + runs-on: ubuntu-latest + defaults: + run: + working-directory: core/wren + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v4 + - name: Lint + run: | + uvx ruff format --check src/ + uvx ruff check src/ + + test-unit: + name: unit tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: core/wren + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install uv + uses: astral-sh/setup-uv@v4 + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./core/wren-core-py/target/ + key: ${{ runner.os }}-cargo-wren-${{ hashFiles('**/Cargo.lock') }} + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Build wren-core-py wheel + working-directory: core/wren-core-py + run: | + pipx install poetry + poetry install --no-root + poetry run maturin build + - name: Install dependencies + run: | + uv lock --find-links ../wren-core-py/target/wheels/ --upgrade-package wren-core-py + uv sync --extra dev --find-links ../wren-core-py/target/wheels/ + - name: Run unit tests + run: uv run pytest tests/unit/ -v -m unit + + test-connector: + name: ${{ matrix.datasource }} tests + runs-on: ubuntu-latest + strategy: + matrix: + include: + - datasource: postgres + extra: postgres + test_file: tests/connectors/test_postgres.py + marker: postgres + - datasource: mysql + extra: mysql + test_file: tests/connectors/test_mysql.py + marker: mysql + defaults: + run: + working-directory: core/wren + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install uv + uses: astral-sh/setup-uv@v4 + - name: Install system dependencies for mysqlclient + if: matrix.datasource == 'mysql' + run: | + sudo apt-get update + sudo apt-get install -y default-libmysqlclient-dev pkg-config + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./core/wren-core-py/target/ + key: ${{ runner.os }}-cargo-wren-${{ hashFiles('**/Cargo.lock') }} + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Build wren-core-py wheel + working-directory: core/wren-core-py + run: | + pipx install poetry + poetry install --no-root + poetry run maturin build + - name: Install dependencies + run: | + uv lock --find-links ../wren-core-py/target/wheels/ --upgrade-package wren-core-py + uv sync --extra ${{ matrix.extra }} --extra dev --find-links ../wren-core-py/target/wheels/ + - name: Run ${{ matrix.datasource }} tests + run: uv run pytest ${{ matrix.test_file }} -v -m ${{ matrix.marker }} + + test-ui: + name: ui tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: core/wren + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install uv + uses: astral-sh/setup-uv@v4 + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./core/wren-core-py/target/ + key: ${{ runner.os }}-cargo-wren-${{ hashFiles('**/Cargo.lock') }} + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Build wren-core-py wheel + working-directory: core/wren-core-py + run: | + pipx install poetry + poetry install --no-root + poetry run maturin build + - name: Install dependencies + run: | + uv lock --find-links ../wren-core-py/target/wheels/ --upgrade-package wren-core-py + uv sync --extra dev --extra ui --find-links ../wren-core-py/target/wheels/ + - name: Run UI tests + run: uv run pytest tests/test_profile_web.py tests/test_field_registry.py -v + + test-memory: + name: memory tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: core/wren + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install uv + uses: astral-sh/setup-uv@v4 + - name: Cache Cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ./core/wren-core-py/target/ + key: ${{ runner.os }}-cargo-wren-${{ hashFiles('**/Cargo.lock') }} + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Build wren-core-py wheel + working-directory: core/wren-core-py + run: | + pipx install poetry + poetry install --no-root + poetry run maturin build + - name: Install dependencies + run: | + uv lock --find-links ../wren-core-py/target/wheels/ --upgrade-package wren-core-py + uv sync --extra dev --extra memory --find-links ../wren-core-py/target/wheels/ + - name: Cache sentence-transformers model + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: hf-paraphrase-MiniLM-L3-v2 + - name: Run memory tests + env: + WREN_EMBEDDING_MODEL: paraphrase-MiniLM-L3-v2 + run: uv run pytest tests/unit/test_memory.py -v diff --git a/.github/workflows/wren-launcher-ci.yaml b/.github/workflows/wren-launcher-ci.yaml deleted file mode 100644 index 8baf1ce99..000000000 --- a/.github/workflows/wren-launcher-ci.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: Wren Launcher CI - -on: - pull_request: - types: [synchronize, labeled] - paths: - - 'wren-launcher/**' - - '.github/workflows/wren-launcher-ci.yaml' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.number || github.sha }} - cancel-in-progress: true - -defaults: - run: - working-directory: wren-launcher - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 - with: - go-version: '1.24' - cache-dependency-path: wren-launcher/go.sum - - name: golangci-lint - uses: golangci/golangci-lint-action@v8 - with: - version: v2.3.1 - working-directory: wren-launcher - - fmt-and-test: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.24' - cache-dependency-path: wren-launcher/go.sum - - name: Install goimports - run: go install golang.org/x/tools/cmd/goimports@latest - - name: Download dependencies - run: go mod download - - name: Run format check - run: | - make fmt - # Check if there are any formatting changes - if [ -n "$(git diff --name-only)" ]; then - echo "Code is not formatted properly. Please run 'make fmt' and commit the changes." - git diff - exit 1 - fi - - name: Run go vet - run: make vet - - name: Run tests - run: go test ./commands/dbt - - security-scan: - runs-on: ubuntu-latest - needs: fmt-and-test - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.24' - cache-dependency-path: wren-launcher/go.sum - - name: Run Gosec Security Scanner - run: | - go install github.com/securego/gosec/v2/cmd/gosec@latest - gosec ./... - - name: Run Go mod audit - run: | - go mod verify - go list -json -deps ./... | jq -r '.Module | select(.Version) | "\(.Path) \(.Version)"' | sort -u diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 000000000..0618875b1 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,5 @@ +{ + "core/wren-core-py": "0.4.0", + "core/wren": "0.3.0", + "core/wren-core-wasm": "0.1.0" +} diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 000000000..cb002c0b9 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "separate-pull-requests": true, + "include-component-in-tag": true, + "packages": { + "core/wren-core-py": { + "component": "wren-core-py", + "release-type": "python", + "bump-minor-pre-major": true, + "extra-files": [ + "Cargo.toml" + ] + }, + "core/wren": { + "component": "wren", + "release-type": "python", + "bump-minor-pre-major": true, + "extra-files": [ + "src/wren/__init__.py" + ] + }, + "core/wren-core-wasm": { + "component": "wren-core-wasm", + "release-type": "node", + "bump-minor-pre-major": true, + "extra-files": [ + "Cargo.toml" + ] + } + } +}