feat: Add AI/LLM-powered pipeline analysis #1123
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| schedule: | |
| - cron: "0 5 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| debug_enabled: | |
| type: boolean | |
| description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)" | |
| required: false | |
| default: false | |
| target_ref: | |
| type: string | |
| description: "Target ref to run the tests against" | |
| required: false | |
| pull_request_target: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| paths: | |
| - "**.go" | |
| - ".github/workflows/**" | |
| - "test/testdata/**" | |
| jobs: | |
| e2e-tests: | |
| # Run on schedule, unconditional workflow_dispatch, | |
| # or pull_request_target if the actor has write/admin permissions. | |
| if: > | |
| github.event_name == 'schedule' || | |
| github.event_name == 'workflow_dispatch' || | |
| github.event_name == 'pull_request_target' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ matrix.provider }}-${{ github.event.pull_request.number || github.ref_name }} | |
| cancel-in-progress: true | |
| name: e2e tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| provider: [providers, gitea_others] | |
| env: | |
| TARGET_TEAM_SLUGS: "pipeline-as-code,pipeline-as-code-contributors" | |
| KO_DOCKER_REPO: localhost:5000 | |
| CONTROLLER_DOMAIN_URL: controller.paac-127-0-0-1.nip.io | |
| TEST_GITHUB_REPO_OWNER_GITHUBAPP: openshift-pipelines/pipelines-as-code-e2e-tests | |
| KUBECONFIG: /home/runner/.kube/config.kind | |
| TEST_BITBUCKET_CLOUD_API_URL: https://api.bitbucket.org/2.0 | |
| TEST_BITBUCKET_CLOUD_E2E_REPOSITORY: cboudjna/pac-e2e-tests | |
| TEST_BITBUCKET_CLOUD_USER: cboudjna | |
| TEST_EL_URL: http://controller.paac-127-0-0-1.nip.io | |
| TEST_GITEA_API_URL: http://localhost:3000 | |
| TEST_GITEA_USERNAME: pac | |
| TEST_GITEA_PASSWORD: pac | |
| TEST_GITEA_REPO_OWNER: pac/pac | |
| TEST_GITHUB_API_URL: api.github.com | |
| TEST_GITHUB_REPO_OWNER_WEBHOOK: openshift-pipelines/pipelines-as-code-e2e-tests-webhook | |
| TEST_GITHUB_PRIVATE_TASK_URL: https://github.com/openshift-pipelines/pipelines-as-code-e2e-tests-private/blob/main/remote_task.yaml | |
| TEST_GITHUB_PRIVATE_TASK_NAME: task-remote | |
| TEST_GITHUB_SECOND_API_URL: ghe.pipelinesascode.com | |
| TEST_GITHUB_SECOND_EL_URL: http://ghe.paac-127-0-0-1.nip.io | |
| TEST_GITHUB_SECOND_REPO_OWNER_GITHUBAPP: pipelines-as-code/e2e | |
| TEST_GITHUB_SECOND_REPO_INSTALLATION_ID: 1 | |
| TEST_GITLAB_API_URL: https://gitlab.com | |
| TEST_GITLAB_PROJECT_ID: ${{ vars.TEST_GITLAB_PROJECT_ID }} | |
| TEST_BITBUCKET_SERVER_USER: pipelines | |
| TEST_BITBUCKET_SERVER_E2E_REPOSITORY: PAC/pac-e2e-tests | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ inputs.target_ref || github.event.pull_request.head.sha || github.sha }} | |
| # Step to check PR author's org membership and repo permissions. | |
| # This step will fail the job if checks do not pass, skipping subsequent steps. | |
| - name: Check user permissions on PRs | |
| if: github.event_name == 'pull_request_target' | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| if (!context || !context.payload || !context.payload.pull_request) { | |
| core.setFailed('Invalid GitHub context: missing required pull_request information'); | |
| return; | |
| } | |
| async function run() { | |
| const actor = context.payload.pull_request.user.login; | |
| const repoOwner = context.repo.owner; | |
| const repoName = context.repo.repo; | |
| const targetOrg = context.repo.owner; | |
| core.info(`🔍 Starting permission check for user: @${actor}`); | |
| core.info(`📋 Repository: ${repoOwner}/${repoName}`); | |
| core.info(`🏢 Target organization: ${targetOrg}`); | |
| // Condition 1: Check if the user is a trusted bot. | |
| const trustedBots = ["dependabot[bot]", "renovate[bot]"]; | |
| core.info(`🤖 Checking if @${actor} is a trusted bot...`); | |
| core.info(` Trusted bots list: ${trustedBots.join(', ')}`); | |
| if (trustedBots.includes(actor)) { | |
| core.info(`✅ Condition met: User @${actor} is a trusted bot. Proceeding.`); | |
| return; // Success | |
| } | |
| core.info(` ❌ User @${actor} is not a trusted bot.`); | |
| // Condition 2: Check for public membership in the target organization. | |
| core.info(`\n👥 Condition 2: Checking organization and team membership...`); | |
| core.info( | |
| `User @${actor} is not a trusted bot. Checking for membership in '${targetOrg}'...`, | |
| ); | |
| try { | |
| // Optional: check membership in one or more org teams (set TARGET_TEAM_SLUGS as comma-separated slugs in workflow env) | |
| const teamSlugsEnv = process.env.TARGET_TEAM_SLUGS || ""; | |
| const teamSlugs = teamSlugsEnv | |
| .split(",") | |
| .map((s) => s.trim()) | |
| .filter(Boolean); | |
| core.info(`🔧 TARGET_TEAM_SLUGS environment variable: "${teamSlugsEnv}"`); | |
| core.info(`📝 Parsed team slugs: [${teamSlugs.join(', ')}]`); | |
| if (teamSlugs.length > 0) { | |
| core.info(`🔍 Checking team membership for ${teamSlugs.length} team(s)...`); | |
| for (const team_slug of teamSlugs) { | |
| core.info(` Checking team: ${team_slug}...`); | |
| try { | |
| const membership = await github.rest.teams.getMembershipForUserInOrg({ | |
| org: targetOrg, | |
| team_slug, | |
| username: actor, | |
| }); | |
| core.info(` API response for team '${team_slug}': ${JSON.stringify(membership.data)}`); | |
| if ( | |
| membership && | |
| membership.data && | |
| membership.data.state === "active" | |
| ) { | |
| core.info( | |
| `✅ Condition met: User @${actor} is a member of team '${team_slug}' in '${targetOrg}'. Proceeding.`, | |
| ); | |
| return; // Success | |
| } else { | |
| core.info(` ⚠️ Team membership found but state is not 'active': ${membership.data.state}`); | |
| } | |
| } catch (err) { | |
| // Not a member of this team or team doesn't exist — continue to next | |
| core.info( | |
| ` ❌ User @${actor} is not a member of team '${team_slug}' (or team not found). Error: ${err.message}`, | |
| ); | |
| } | |
| } | |
| // If we tried team checks and none matched, continue to next org membership checks | |
| core.info( | |
| `ⓘ User @${actor} is not a member of any configured teams in '${targetOrg}'. Falling back to org membership checks.`, | |
| ); | |
| } else { | |
| core.info(`ℹ️ No teams configured in TARGET_TEAM_SLUGS. Skipping team membership checks.`); | |
| } | |
| core.info(`🏢 Checking organization membership for @${actor} in '${targetOrg}'...`); | |
| try { | |
| core.info(` Attempting checkMembershipForUser API call...`); | |
| await github.rest.orgs.checkMembershipForUser({ | |
| org: targetOrg, | |
| username: actor, | |
| }); | |
| core.info( | |
| `✅ Condition met: User @${actor} is a member of '${targetOrg}'. Proceeding.`, | |
| ); | |
| return; // Success | |
| } catch (err) { | |
| // Try public membership as fallback | |
| core.info(` ❌ Private membership check failed: ${err.message}`); | |
| core.info(` Attempting checkPublicMembershipForUser API call...`); | |
| try { | |
| await github.rest.orgs.checkPublicMembershipForUser({ | |
| org: targetOrg, | |
| username: actor, | |
| }); | |
| core.info( | |
| `✅ Condition met: User @${actor} is a public member of '${targetOrg}'. Proceeding.`, | |
| ); | |
| return; // Success | |
| } catch (publicErr) { | |
| // Neither private nor public member - will be caught by outer catch | |
| core.info(` ❌ Public membership check failed: ${publicErr.message}`); | |
| throw publicErr; | |
| } | |
| } | |
| } catch (error) { | |
| // This is not a failure, just one unmet condition. Log and continue. | |
| core.info( | |
| `ⓘ User @${actor} is not a public member of '${targetOrg}'. Checking repository permissions as a fallback.`, | |
| ); | |
| } | |
| // Condition 3: Check for write/admin permission on the repository. | |
| core.info(`\n🔐 Condition 3: Checking repository collaborator permissions...`); | |
| try { | |
| core.info(` Attempting getCollaboratorPermissionLevel API call...`); | |
| const response = await github.rest.repos.getCollaboratorPermissionLevel({ | |
| owner: repoOwner, | |
| repo: repoName, | |
| username: actor, | |
| }); | |
| const permission = response.data.permission; | |
| core.info(` User @${actor} has '${permission}' permission on ${repoOwner}/${repoName}`); | |
| if (permission === "admin" || permission === "write") { | |
| core.info( | |
| `✅ Condition met: User @${actor} has '${permission}' repository permission. Proceeding.`, | |
| ); | |
| return; // Success | |
| } else { | |
| // If we reach here, no conditions were met. This is the final failure. | |
| core.info(` ❌ Permission '${permission}' is insufficient (requires 'write' or 'admin')`); | |
| core.setFailed( | |
| `❌ Permission check failed. User @${actor} did not meet any required conditions (trusted bot, org member, or repo write access).`, | |
| ); | |
| return; | |
| } | |
| } catch (error) { | |
| // This error means they are not even a collaborator. | |
| core.info(` ❌ Collaborator permission check failed: ${error.message}`); | |
| core.setFailed( | |
| `❌ Permission check failed. User @${actor} is not a collaborator on this repository and did not meet other conditions.`, | |
| ); | |
| return; | |
| } | |
| } | |
| run().catch(err => { | |
| core.error(`💥 Unexpected error during permission check: ${err.message}`); | |
| core.error(` Stack trace: ${err.stack}`); | |
| core.setFailed(`Unexpected error during permission check: ${err.message}`); | |
| }); | |
| - uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: "go.mod" | |
| - uses: ko-build/[email protected] | |
| - name: Install gosmee | |
| uses: jaxxstorm/[email protected] | |
| with: | |
| repo: chmouel/gosmee | |
| - name: Install Snazy | |
| uses: jaxxstorm/[email protected] | |
| with: | |
| repo: chmouel/snazy | |
| - name: Run gosmee | |
| run: | | |
| nohup gosmee client --saveDir /tmp/gosmee-replay ${{ secrets.PYSMEE_URL }} "http://${CONTROLLER_DOMAIN_URL}" & | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} | |
| with: | |
| detached: true | |
| limit-access-to-actor: true | |
| - name: Start installing cluster | |
| run: | | |
| export PAC_DIR=${PWD} | |
| export TEST_GITEA_SMEEURL="${{ secrets.TEST_GITEA_SMEEURL }}" | |
| bash -x ./hack/dev/kind/install.sh | |
| - name: Create PAC github-app-secret | |
| env: | |
| PAC_GITHUB_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }} | |
| PAC_GITHUB_APPLICATION_ID: ${{ vars.APPLICATION_ID }} | |
| PAC_WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} | |
| run: | | |
| ./hack/gh-workflow-ci.sh create_pac_github_app_secret | |
| - name: Create second Github APP Controller on GHE | |
| env: | |
| TEST_GITHUB_SECOND_SMEE_URL: ${{ secrets.TEST_GITHUB_SECOND_SMEE_URL }} | |
| TEST_GITHUB_SECOND_PRIVATE_KEY: ${{ secrets.TEST_GITHUB_SECOND_PRIVATE_KEY }} | |
| TEST_GITHUB_SECOND_WEBHOOK_SECRET: ${{ secrets.TEST_GITHUB_SECOND_WEBHOOK_SECRET }} | |
| TEST_GITHUB_SECOND_APPLICATION_ID: ${{ vars.TEST_GITHUB_SECOND_APPLICATION_ID }} | |
| run: | | |
| ./hack/gh-workflow-ci.sh create_second_github_app_controller_on_ghe | |
| # Adjusted step-level conditions based on the new job-level logic | |
| - name: Run E2E Tests | |
| # This step runs for schedule, PR target (if job started), or workflow_dispatch (if job started) | |
| # Remove the old label check which is no longer relevant for triggering. | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' }} | |
| env: | |
| TEST_PROVIDER: ${{ matrix.provider }} | |
| TEST_BITBUCKET_CLOUD_TOKEN: ${{ secrets.BITBUCKET_CLOUD_TOKEN }} | |
| TEST_EL_WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} | |
| TEST_GITEA_SMEEURL: ${{ secrets.TEST_GITEA_SMEEURL }} | |
| TEST_GITHUB_REPO_INSTALLATION_ID: ${{ vars.INSTALLATION_ID }} | |
| TEST_GITHUB_TOKEN: ${{ secrets.GH_APPS_TOKEN }} | |
| TEST_GITHUB_SECOND_TOKEN: ${{ secrets.TEST_GITHUB_SECOND_TOKEN }} | |
| TEST_GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | |
| TEST_BITBUCKET_SERVER_TOKEN: ${{ secrets.BITBUCKET_SERVER_TOKEN }} | |
| TEST_BITBUCKET_SERVER_API_URL: ${{ secrets.BITBUCKET_SERVER_API_URL }} | |
| TEST_BITBUCKET_SERVER_WEBHOOK_SECRET: ${{ secrets.BITBUCKET_SERVER_WEBHOOK_SECRET }} | |
| run: | | |
| ./hack/gh-workflow-ci.sh run_e2e_tests | |
| - name: Run E2E Tests on nightly | |
| # This step still runs specifically for schedule or workflow_dispatch | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| env: | |
| NIGHTLY_E2E_TEST: "true" | |
| TEST_PROVIDER: ${{ matrix.provider }} | |
| TEST_BITBUCKET_CLOUD_TOKEN: ${{ secrets.BITBUCKET_CLOUD_TOKEN }} | |
| TEST_EL_WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} | |
| TEST_GITEA_SMEEURL: ${{ secrets.TEST_GITEA_SMEEURL }} | |
| TEST_GITHUB_REPO_INSTALLATION_ID: ${{ vars.INSTALLATION_ID }} | |
| TEST_GITHUB_TOKEN: ${{ secrets.GH_APPS_TOKEN }} | |
| TEST_GITHUB_SECOND_TOKEN: ${{ secrets.TEST_GITHUB_SECOND_TOKEN }} | |
| TEST_GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} | |
| TEST_BITBUCKET_SERVER_TOKEN: ${{ secrets.BITBUCKET_SERVER_TOKEN }} | |
| TEST_BITBUCKET_SERVER_API_URL: ${{ secrets.BITBUCKET_SERVER_API_URL }} | |
| TEST_BITBUCKET_SERVER_WEBHOOK_SECRET: ${{ secrets.BITBUCKET_SERVER_WEBHOOK_SECRET }} | |
| run: | | |
| ./hack/gh-workflow-ci.sh run_e2e_tests | |
| - name: Collect logs | |
| if: ${{ always() }} | |
| env: | |
| TEST_GITEA_SMEEURL: ${{ secrets.TEST_GITEA_SMEEURL }} | |
| TEST_GITHUB_SECOND_SMEE_URL: ${{ secrets.TEST_GITHUB_SECOND_SMEE_URL }} | |
| run: | | |
| ./hack/gh-workflow-ci.sh collect_logs | |
| - name: Show controllers/watcher logs with Snazy | |
| if: ${{ always() }} | |
| run: | | |
| ./hack/gh-workflow-ci.sh output_logs | |
| - name: Upload artifacts | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: logs-e2e-tests-${{ matrix.provider }} | |
| path: /tmp/logs | |
| - name: Report Status | |
| if: ${{ always() && github.ref_name == 'main' && github.event_name == 'schedule' }} | |
| uses: ravsamhq/notify-slack-action@v2 | |
| with: | |
| status: ${{ job.status }} | |
| notify_when: "failure" | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} |