Release #62
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Release is called by duckdb's InvokeCI -> NotifyExternalRepositories job | |
| name: Release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| duckdb-python-sha: | |
| type: string | |
| description: The commit to build against (defaults to latest commit of current ref) | |
| required: false | |
| duckdb-sha: | |
| type: string | |
| description: The DuckDB submodule commit or ref to build against | |
| required: true | |
| stable-version: | |
| type: string | |
| description: Release a stable version (vX.Y.Z-((rc|post)N)) | |
| required: false | |
| pypi-index: | |
| type: choice | |
| description: Which PyPI to use | |
| required: true | |
| options: | |
| - test | |
| - prod | |
| store-s3: | |
| type: boolean | |
| description: Also store test packages in S3 (always true for prod) | |
| default: false | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| build_sdist: | |
| name: Build an sdist and determine versions | |
| uses: ./.github/workflows/packaging_sdist.yml | |
| with: | |
| testsuite: all | |
| duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} | |
| duckdb-sha: ${{ inputs.duckdb-sha }} | |
| set-version: ${{ inputs.stable-version }} | |
| workflow_state: | |
| name: Set state for the release workflow | |
| needs: build_sdist | |
| outputs: | |
| pypi_state: ${{ steps.index_check.outputs.pypi_state }} | |
| ci_env: ${{ steps.ci_env_check.outputs.ci_env }} | |
| s3_url: ${{ steps.s3_check.outputs.s3_url }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: index_check | |
| name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI | |
| run: | | |
| set -eu | |
| # Check PyPI whether the release we're building is already present | |
| pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org | |
| pkg_version=${{ needs.build_sdist.outputs.package-version }} | |
| url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json | |
| http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? ) | |
| if [[ $http_status == "200" ]]; then | |
| echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}" | |
| pypi_state=VERSION_FOUND | |
| elif [[ $http_status == 000* ]]; then | |
| echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}" | |
| pypi_state=UNKNOWN | |
| else | |
| echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})" | |
| pypi_state=VERSION_NOT_FOUND | |
| fi | |
| echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT | |
| - id: ci_env_check | |
| name: Determine CI environment | |
| run: | | |
| set -eu | |
| if [[ test == "${{ inputs.pypi-index }}" ]]; then | |
| ci_env=pypi-test | |
| elif [[ prod == "${{ inputs.pypi-index }}" ]]; then | |
| ci_env=pypi-prod${{ inputs.stable-version == '' && '-nightly' || '' }} | |
| else | |
| echo "::error::Invalid value for inputs.pypi-index: ${{ inputs.pypi-index }}" | |
| exit 1 | |
| fi | |
| echo "ci_env=${ci_env}" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Using CI environment ${ci_env}" | |
| - id: s3_check | |
| name: Generate S3 upload URL | |
| if: github.repository_owner == 'duckdb' | |
| run: | | |
| set -eu | |
| should_store=${{ (inputs.pypi-index == 'prod' || inputs.store-s3) && '1' || '0' }} | |
| if [[ $should_store == 0 ]]; then | |
| echo "::notice::S3 upload disabled in inputs, not generating S3 URL" | |
| exit 0 | |
| fi | |
| if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then | |
| echo "::warning::S3 upload disabled because package version already uploaded to PyPI" | |
| exit 0 | |
| fi | |
| sha=${{ github.sha }} | |
| dsha=${{ inputs.duckdb-sha }} | |
| version=${{ needs.build_sdist.outputs.package-version }} | |
| s3_url="s3://duckdb-staging/python/${version}/${sha:0:10}-duckdb-${dsha:0:10}/" | |
| echo "::notice::Generated S3 URL: ${s3_url}" | |
| echo "s3_url=${s3_url}" >> $GITHUB_OUTPUT | |
| build_wheels: | |
| name: Build and test releases | |
| needs: workflow_state | |
| if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }} | |
| uses: ./.github/workflows/packaging_wheels.yml | |
| with: | |
| minimal: false | |
| testsuite: all | |
| duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }} | |
| duckdb-sha: ${{ inputs.duckdb-sha }} | |
| set-version: ${{ inputs.stable-version }} | |
| upload_s3: | |
| name: Upload Artifacts to S3 | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, build_wheels, workflow_state] | |
| if: ${{ needs.workflow_state.outputs.s3_url }} | |
| steps: | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: artifacts/ | |
| merge-multiple: true | |
| - name: Authenticate with AWS | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: 'us-east-2' | |
| aws-access-key-id: ${{ secrets.S3_DUCKDB_STAGING_ID }} | |
| aws-secret-access-key: ${{ secrets.S3_DUCKDB_STAGING_KEY }} | |
| - name: Upload Artifacts | |
| run: | | |
| aws s3 cp artifacts ${{ needs.workflow_state.outputs.s3_url }} --recursive | |
| publish_pypi: | |
| name: Publish Artifacts to PyPI | |
| runs-on: ubuntu-latest | |
| if: ${{ !always() }} | |
| needs: [workflow_state, build_sdist, build_wheels] | |
| environment: | |
| name: ${{ needs.workflow_state.outputs.ci_env }} | |
| permissions: | |
| # this is needed for the OIDC flow that is used with trusted publishing on PyPI | |
| id-token: write | |
| steps: | |
| - if: ${{ vars.PYPI_HOST == '' }} | |
| run: | | |
| echo "Error: PYPI_HOST is not set in CI environment '${{ needs.workflow_state.outputs.ci_env }}'" | |
| exit 1 | |
| - name: Fetch artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: '{sdist,wheel}*' | |
| path: packages/ | |
| merge-multiple: true | |
| - name: Upload artifacts to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| repository-url: 'https://${{ vars.PYPI_HOST }}/legacy/' | |
| packages-dir: packages | |
| verbose: 'true' | |
| cleanup_nightlies: | |
| name: Remove Nightlies from PyPI | |
| needs: [workflow_state, publish_pypi] | |
| if: ${{ inputs.stable-version == '' }} | |
| uses: ./.github/workflows/cleanup_pypi.yml | |
| with: | |
| environment: ${{ needs.workflow_state.outputs.ci_env }} | |
| secrets: | |
| # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206 | |
| PYPI_CLEANUP_OTP: ${{secrets.PYPI_CLEANUP_OTP}} | |
| PYPI_CLEANUP_PASSWORD: ${{secrets.PYPI_CLEANUP_PASSWORD}} | |
| summary: | |
| name: Release summary | |
| runs-on: ubuntu-latest | |
| needs: [build_sdist, workflow_state, build_wheels, upload_s3, publish_pypi, cleanup_nightlies] | |
| if: always() | |
| steps: | |
| - run: | | |
| sha=${{ github.sha }} | |
| dsha=${{ inputs.duckdb-sha }} | |
| pversion=${{ needs.build_sdist.outputs.package-version }} | |
| long_pversion="${pversion} (${sha:0:10})" | |
| pypi_host=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org | |
| pypi_duckdb_url=https://${pypi_host}/project/duckdb/${pversion}/ | |
| was_released=${{ needs.publish_pypi.result == 'success' && '1' || '0' }} | |
| if [[ $was_released == 1 ]]; then | |
| echo "## Version ${long_pversion} successfully released" >> $GITHUB_STEP_SUMMARY | |
| echo "* Package URL: [${pypi_duckdb_url}](${pypi_duckdb_url})" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "## Version ${long_pversion} was not released" >> $GITHUB_STEP_SUMMARY | |
| echo "* Package index state before release: ${{ needs.workflow_state.outputs.pypi_state }}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "* Package index: ${pypi_host}" >> $GITHUB_STEP_SUMMARY | |
| echo "* Vendored DuckDB Version: ${{ needs.build_sdist.outputs.duckdb-version }} (${dsha:0:10})" >> $GITHUB_STEP_SUMMARY | |
| echo "* S3 upload status: ${{ needs.upload_s3.result == 'success' && needs.workflow_state.outputs.s3_url || needs.upload_s3.result }}" >> $GITHUB_STEP_SUMMARY | |
| echo "* CI Environment: ${{ needs.workflow_state.outputs.ci_env }}" >> $GITHUB_STEP_SUMMARY |