Support python3.11

Author: fhoering

.github/workflows/main.yml

@@ -53,22 +53,26 @@ jobs:
   standalone_spark3_with_S3:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        version:  [ {python: 3.9, spark: 3.3.1}, {python: 3.11, spark: 3.4.2} ]
+
     steps:
     - uses: actions/checkout@v2
 
     - name: Build spark-docker
-      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=3.3.1 --build-arg PYTHON_VERSION=3.9.15
+      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=${{ matrix.version.spark }} --build-arg PYTHON_VERSION=${{ matrix.version.python }}
 
     - name: Build the docker-compose stack
       run: |
-        export PYTHON_VERSION=3.9
+        export PYTHON_VERSION=${{ matrix.version.python }}
         docker compose -f ./examples/spark-with-S3/docker-compose.yml up -d
       
     - name: Check running containers
       run: docker ps -a
 
     - name: Run spark Job
-      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.9 3.3.1
+      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python${{ matrix.version.python }} ${{ matrix.version.spark }}
 
   # hadoop_hdfs:
   #   runs-on: ubuntu-latest

examples/spark-with-S3/Dockerfile

@@ -4,19 +4,20 @@ FROM debian:bookworm
 
 RUN apt-get update
 ARG SPARK_INPUT_VERSION=3.3.1
-ARG PYTHON_VERSION=3.9.15
+ARG PYTHON_VERSION=3.9
 
 # Python
 RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
   libreadline-dev libsqlite3-dev wget curl unzip llvm libncurses5-dev libncursesw5-dev \
-  xz-utils tk-dev libffi-dev liblzma-dev
-RUN curl https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz --output /usr/python.tgz
-RUN tar xf /usr/python.tgz -C /usr && rm /usr/python.tgz
-RUN cd /usr/Python-${PYTHON_VERSION} \ 
- && ./configure --enable-shared \
- && make -j8 \
- && make altinstall \
- && cd -
+  xz-utils tk-dev libffi-dev liblzma-dev git
+RUN curl https://pyenv.run | bash
+
+ENV PATH=/root/.pyenv/bin:$PATH
+
+RUN eval "$(pyenv init -)" 
+
+RUN pyenv install ${PYTHON_VERSION} 
+RUN pyenv global ${PYTHON_VERSION}
 
 # Java
 RUN apt-get install -y openjdk-17-jre \

examples/spark-with-S3/scripts/run_spark_example.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+eval "$(pyenv init -)"
 rm -rf /tmp/pyspark_env
 $1 -m venv /tmp/pyspark_env
 . /tmp/pyspark_env/bin/activate