diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 4ab223060..93856ab2c 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -300,3 +300,109 @@ jobs: else echo "No changes to commit" fi + +# ------------------------------------------------------------------------------------------------- +# Gubbins deployment +# ------------------------------------------------------------------------------------------------- + + build-gubbins-wheels: + name: Build wheels + runs-on: "ubuntu-latest" + needs: deploy-pypi + defaults: + run: + # We need extglob for REFERENCE_BRANCH substitution + shell: bash -l -O extglob {0} + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Installing dependencies + run: | + python -m pip install \ + build \ + python-dateutil \ + pytz \ + readme_renderer[md] \ + requests \ + setuptools_scm + - name: Build distributions + run: | + # build the gubbins package + for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do + # Skip the testing package + if [[ "${pkg_dir}" =~ .*testing.* ]]; + then + echo "Do not build ${pkg_dir}"; + continue; + fi + echo "Building $pkg_dir" + python -m build --outdir $PWD/dist $pkg_dir + done + - name: 'Upload Artifact' + uses: actions/upload-artifact@v4 + with: + name: gubbins-whl + path: dist/*.whl + retention-days: 5 + + # Build to docker image with the code in it + docker-gubbins: + needs: + - deploy-pypi + - build-gubbins-wheels + timeout-minutes: 30 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Download diracx wheels + uses: actions/download-artifact@v4 + with: + name: diracx-whl + - name: Download gubbins wheels + uses: actions/download-artifact@v4 + with: + name: gubbins-whl + - name: "Find wheels" + id: find_wheel + run: | + # We need to copy them there to be able to access them in the RUN --mount + cp diracx*.whl gubbins*.whl extensions/containers/client/ + cp diracx*.whl gubbins*.whl extensions/containers/services/ + for wheel_fn in *.whl; do + pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1) + echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT + done + + - name: Build and push client (dev) + uses: docker/build-push-action@v6 + with: + context: extensions/containers/client + push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }} + tags: ghcr.io/diracgrid/gubbins/client:dev + platforms: linux/amd64,linux/arm64 + build-args: | + EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration + - name: Build and push service (dev) + uses: docker/build-push-action@v6 + with: + context: extensions/containers/services + push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }} + tags: ghcr.io/diracgrid/gubbins/services:dev + platforms: linux/amd64,linux/arm64 + build-args: | + EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration + EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1f03728c4..f7b168929 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -99,15 +99,19 @@ jobs: with: cache: false environments: ${{ matrix.extension == 'diracx' && 'default' || 'default-gubbins' }} - - name: Build gubbins wheels - if: ${{ matrix.extension == 'gubbins' }} + - name: Build diracx wheels run: | for pkg_dir in $PWD/diracx-*; do echo "Building $pkg_dir" - pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir + pixi exec python-build --outdir $PWD/containers/services/ $pkg_dir done # Also build the diracx metapackage - pixi exec python-build --outdir $PWD/extensions/containers/services/ . + pixi exec python-build --outdir $PWD/containers/services/ . + - name: Build gubbins wheels + if: ${{ matrix.extension == 'gubbins' }} + run: | + cp $PWD/containers/services/*.whl $PWD/extensions/containers/services/ + cp $PWD/containers/services/*.tar.gz $PWD/extensions/containers/services/ # And build the gubbins package for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do # Skip the testing package @@ -120,25 +124,30 @@ jobs: pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir done - name: Set up Docker Buildx - if: ${{ matrix.extension == 'gubbins' }} uses: docker/setup-buildx-action@v3 + - name: Build container for diracx + if: ${{ matrix.extension == 'diracx' }} + uses: docker/build-push-action@v6 + with: + context: containers/services + tags: diracx/services:dev + load: true + build-args: | + EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration - name: Build container for gubbins if: ${{ matrix.extension == 'gubbins' }} uses: docker/build-push-action@v6 with: context: extensions/containers/services tags: gubbins/services:dev - outputs: type=docker,dest=/tmp/gubbins_services_image.tar + load: true build-args: | + EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl - - name: Load image - if: ${{ matrix.extension == 'gubbins' }} - run: | - docker load --input /tmp/gubbins_services_image.tar - docker image ls -a + - name: Start demo run: | - git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts + git clone --branch main_FEAT_remove-loaded-images https://github.com/aldbr/diracx-charts.git ../diracx-charts declare -a demo_args=() demo_args+=("--enable-open-telemetry") @@ -146,6 +155,7 @@ jobs: demo_args+=("--exit-when-done") demo_args+=("--set-value" "developer.autoReload=false") demo_args+=("--ci-values" "../diracx-charts/demo/ci_values.yaml") + demo_args+=("--prune-loaded-images") declare -a demo_source_dirs=("$PWD") @@ -162,7 +172,10 @@ jobs: demo_args+=("--ci-values" "./extensions/gubbins_values.yaml") demo_args+=("--load-docker-image" "gubbins/services:dev") demo_source_dirs+=("/tmp/gubbins/") - elif [ ${{ matrix.extension }} != 'diracx' ]; then + elif [ ${{ matrix.extension }} == 'diracx' ]; then + demo_args+=("--ci-values" "./diracx_values.yaml") + demo_args+=("--load-docker-image" "diracx/services:dev") + else echo "Unknown extension: ${{ matrix.extension }}" exit 1 fi diff --git a/diracx_values.yaml b/diracx_values.yaml new file mode 100644 index 000000000..4569ddce3 --- /dev/null +++ b/diracx_values.yaml @@ -0,0 +1,11 @@ +# values specific to testing gubbins +global: + # Needed to be able to force pre-load the image in kind + # see https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster + # and https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/ + imagePullPolicy: IfNotPresent + images: + services: diracx/services + +developer: + enabled: true diff --git a/extensions/gubbins_values.yaml b/extensions/gubbins_values.yaml index 4bc01de1a..58b3a4bed 100644 --- a/extensions/gubbins_values.yaml +++ b/extensions/gubbins_values.yaml @@ -5,7 +5,7 @@ global: # and https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/ imagePullPolicy: IfNotPresent images: - services: gubbins/services + services: ghcr.io/diracgrid/gubbins/services tag: dev developer: