test(agent-eval): gate Pro onboarding claims on replayable evidence (… #12171
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generator tests # TODO needs to be duplicated for RoR Pro | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: >- | |
| ${{ | |
| github.event.pull_request.number && format('{0}-pr-{1}', github.workflow, github.event.pull_request.number) || | |
| format('{0}-{1}', github.workflow, github.sha) | |
| }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - 'release/**' | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| merge_group: | |
| workflow_dispatch: | |
| inputs: | |
| force_full_hosted: | |
| description: 'Force full hosted CI (bypass optimized detect-changes)' | |
| required: false | |
| type: boolean | |
| default: false | |
| pull_request_base_ref: | |
| description: 'Pull request base branch for label/command dispatches' | |
| required: false | |
| type: string | |
| pull_request_base_sha: | |
| description: 'Pull request base SHA for label/command dispatches' | |
| required: false | |
| type: string | |
| jobs: | |
| detect-changes: | |
| # Branch deletion push events have no checkoutable release ref. | |
| if: github.event_name != 'push' || github.event.deleted != true | |
| permissions: | |
| contents: read | |
| actions: read | |
| issues: read | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| docs_only: ${{ steps.detect.outputs.docs_only }} | |
| non_runtime_only: ${{ steps.detect.outputs.non_runtime_only }} | |
| run_lint: ${{ steps.detect.outputs.run_lint }} | |
| run_js_tests: ${{ steps.detect.outputs.run_js_tests }} | |
| run_ruby_tests: ${{ steps.detect.outputs.run_ruby_tests }} | |
| run_dummy_tests: ${{ steps.detect.outputs.run_dummy_tests }} | |
| run_generators: ${{ steps.detect.outputs.run_generators }} | |
| should_run_hosted_ci: ${{ steps.hosted-ci.outputs.should_run_hosted_ci }} | |
| should_force_full_hosted_ci: ${{ steps.hosted-ci.outputs.should_force_full_hosted_ci }} | |
| should_use_full_matrix: ${{ steps.hosted-ci.outputs.should_use_full_matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Full history is only needed for workflow_dispatch base-SHA reruns. | |
| fetch-depth: ${{ github.event_name == 'workflow_dispatch' && '0' || '50' }} | |
| persist-credentials: false | |
| - name: Select hosted CI mode | |
| id: hosted-ci | |
| uses: ./.github/actions/hosted-ci-selectors | |
| with: | |
| force-full-hosted: ${{ inputs.force_full_hosted || false }} | |
| base-ref: ${{ inputs.pull_request_base_ref || '' }} | |
| - name: Detect relevant changes | |
| id: detect | |
| env: | |
| DISPATCH_BASE_REF: ${{ inputs.pull_request_base_ref || '' }} | |
| DISPATCH_BASE_SHA: ${{ inputs.pull_request_base_sha || '' }} | |
| run: | | |
| # Force-full hosted CI is the explicit escape hatch that bypasses | |
| # optimized change selection. Hosted readiness alone stays path-based. | |
| if [ "${{ steps.hosted-ci.outputs.should_force_full_hosted_ci }}" = "true" ]; then | |
| { | |
| echo "run_lint=true" | |
| echo "run_js_tests=true" | |
| echo "run_ruby_tests=true" | |
| echo "run_dummy_tests=true" | |
| echo "run_generators=true" | |
| echo "docs_only=false" | |
| echo "non_runtime_only=false" | |
| } >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| if [ -n "$DISPATCH_BASE_SHA" ]; then | |
| export GITHUB_BASE_REF="$DISPATCH_BASE_REF" | |
| BASE_REF="$DISPATCH_BASE_SHA" | |
| else | |
| BASE_REF="${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || 'origin/main' }}" | |
| fi | |
| script/ci-changes-detector "$BASE_REF" | |
| shell: bash | |
| - name: Guard docs-only main pushes | |
| if: | | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main') || | |
| (github.event_name == 'merge_group' && | |
| github.event.merge_group.base_ref == 'refs/heads/main') | |
| uses: ./.github/actions/ensure-main-docs-safety | |
| with: | |
| docs-only: ${{ steps.detect.outputs.docs_only }} | |
| previous-sha: ${{ github.event.before || github.event.merge_group.base_sha }} | |
| setup-matrix: | |
| needs: detect-changes | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 1 | |
| - name: Read runtime versions | |
| id: tool-versions | |
| uses: ./.github/actions/read-tool-versions | |
| - id: set-matrix | |
| env: | |
| SHOULD_USE_FULL_MATRIX: ${{ needs.detect-changes.outputs.should_use_full_matrix }} | |
| LATEST_RUBY_VERSION: ${{ steps.tool-versions.outputs.ruby-minor-version }} | |
| MINIMUM_RUBY_VERSION: ${{ steps.tool-versions.outputs.minimum-ruby-minor-version }} | |
| run: | | |
| latest_ruby_version="$LATEST_RUBY_VERSION" | |
| minimum_ruby_version="$MINIMUM_RUBY_VERSION" | |
| # Full matrices run only on main, merge queue, release targets, or | |
| # explicit force-full hosted CI. Normal hosted PR CI stays optimized. | |
| if [[ "$SHOULD_USE_FULL_MATRIX" == "true" ]]; then | |
| # Full matrix: test both latest and minimum supported versions | |
| matrix="$(jq -nc \ | |
| --arg latest_ruby_version "$latest_ruby_version" \ | |
| --arg minimum_ruby_version "$minimum_ruby_version" \ | |
| '{ | |
| "include": [ | |
| {"ruby-version": $latest_ruby_version, "dependency-level": "latest"}, | |
| {"ruby-version": $minimum_ruby_version, "dependency-level": "minimum"} | |
| ] | |
| }')" | |
| else | |
| # PR matrix: test only latest versions for fast feedback | |
| matrix="$(jq -nc \ | |
| --arg latest_ruby_version "$latest_ruby_version" \ | |
| '{"include": [{"ruby-version": $latest_ruby_version, "dependency-level": "latest"}]}')" | |
| fi | |
| echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" | |
| examples: | |
| needs: [detect-changes, setup-matrix] | |
| # Run hosted generator tests only when hosted CI is allowed and generator paths changed. | |
| if: | | |
| needs.detect-changes.outputs.should_run_hosted_ci == 'true' && | |
| needs.detect-changes.outputs.run_generators == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.setup-matrix.outputs.matrix) }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Read runtime versions | |
| id: tool-versions | |
| uses: ./.github/actions/read-tool-versions | |
| - name: Setup Ruby | |
| uses: ./.github/actions/setup-ruby | |
| with: | |
| ruby-version: ${{ matrix.ruby-version }} | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| # All example lanes test on the minimum Shakapacker Node floor from | |
| # .minimum.tool-versions; example tests exercise Ruby/gem compatibility, | |
| # not Node version compatibility. | |
| node-version: ${{ steps.tool-versions.outputs.minimum-node-version }} | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| - name: Get pnpm store directory | |
| shell: bash | |
| run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_ENV" | |
| - name: Setup pnpm cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.STORE_PATH }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| - name: Print system information | |
| run: | | |
| echo "Linux release: "; cat /etc/issue | |
| echo "Current user: "; whoami | |
| echo "Current directory: "; pwd | |
| echo "Ruby version: "; ruby -v | |
| echo "Node version: "; node -v | |
| echo "pnpm version: "; pnpm --version | |
| echo "Bundler version: "; bundle --version | |
| - name: Run conversion script for older Node compatibility | |
| if: matrix.dependency-level == 'minimum' | |
| run: script/convert | |
| - id: get-sha | |
| run: echo "sha=\"$(git rev-parse HEAD)\"" >> "$GITHUB_OUTPUT" | |
| - name: Install Node modules with pnpm | |
| run: pnpm install ${{ matrix.dependency-level == 'latest' && '--frozen-lockfile' || '--no-frozen-lockfile' }} | |
| - name: Build workspace packages | |
| run: pnpm run build | |
| - name: Install Ruby Gems for package | |
| uses: ./.github/actions/setup-bundle | |
| with: | |
| working-directory: react_on_rails | |
| ruby-version: ${{ matrix.ruby-version }} | |
| # Minimum-dependency legs intentionally install with frozen=false after script/convert, | |
| # matching the historical bundle install behavior for converted dependency files. | |
| frozen: ${{ matrix.dependency-level == 'minimum' && 'false' || 'true' }} | |
| install-libyaml: 'false' | |
| - name: Ensure minimum required Chrome version | |
| run: | | |
| echo -e "Already installed $(google-chrome --version)\n" | |
| MINIMUM_REQUIRED_CHROME_VERSION=75 | |
| INSTALLED_CHROME_MAJOR_VERSION="$(google-chrome --version | tr ' .' '\t' | cut -f3)" | |
| if [[ $INSTALLED_CHROME_MAJOR_VERSION -lt $MINIMUM_REQUIRED_CHROME_VERSION ]]; then | |
| wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add - | |
| sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' | |
| sudo apt-get update | |
| sudo apt-get install google-chrome-stable | |
| echo -e "\nInstalled $(google-chrome --version)" | |
| fi | |
| - name: Increase the amount of inotify watchers | |
| run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p | |
| - name: Set packer version environment variable | |
| run: | | |
| echo "CI_DEPENDENCY_LEVEL=${{ matrix.dependency-level }}" >> "$GITHUB_ENV" | |
| - name: Verify rake tasks exist | |
| run: | | |
| cd react_on_rails | |
| echo "Available shakapacker_examples tasks:" | |
| bundle exec rake -T | grep shakapacker_examples || true | |
| # Verify the specific task we need exists | |
| TASK_NAME="run_rspec:shakapacker_examples_${{ matrix.dependency-level == 'latest' && 'latest' || 'pinned' }}" | |
| if ! bundle exec rake -T | grep -q "$TASK_NAME"; then | |
| echo "ERROR: Required rake task '$TASK_NAME' not found!" | |
| exit 1 | |
| fi | |
| echo "✓ Found required task: $TASK_NAME" | |
| - name: Main CI - Latest version examples | |
| if: matrix.dependency-level == 'latest' | |
| run: cd react_on_rails && bundle exec rake run_rspec:shakapacker_examples_latest | |
| - name: "Main CI - Pinned version examples (React 16, 17, 18)" | |
| if: matrix.dependency-level == 'minimum' | |
| run: cd react_on_rails && bundle exec rake run_rspec:shakapacker_examples_pinned | |
| - name: Store test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: main-rspec-${{ github.run_id }}-${{ github.job }}-ruby${{ matrix.ruby-version }}-${{ matrix.dependency-level }} | |
| path: ~/rspec |