Support python3.11

.github/workflows/main.yml

@@ -33,7 +33,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: [3.8, 3.9, 3.11]
 
     steps:
     - uses: actions/checkout@v2
@@ -53,22 +53,26 @@ jobs:
   standalone_spark3_with_S3:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        version:  [ {python: 3.9, spark: 3.3.1}, {python: 3.11, spark: 3.4.2} ]
+
     steps:
     - uses: actions/checkout@v2
 
     - name: Build spark-docker
-      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=3.3.1 --build-arg PYTHON_VERSION=3.9.15
+      run: docker build -t spark-docker ./examples/spark-with-S3 --build-arg SPARK_INPUT_VERSION=${{ matrix.version.spark }} --build-arg PYTHON_VERSION=${{ matrix.version.python }}
 
     - name: Build the docker-compose stack
       run: |
-        export PYTHON_VERSION=3.9
+        export PYTHON_VERSION=${{ matrix.version.python }}
         docker compose -f ./examples/spark-with-S3/docker-compose.yml up -d
 
     - name: Check running containers
       run: docker ps -a
 
     - name: Run spark Job
-      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python3.9 3.3.1
+      run: docker exec spark-master ./examples/spark-with-S3/scripts/run_spark_example.sh python${{ matrix.version.python }} ${{ matrix.version.spark }}
 
   # hadoop_hdfs:
   #   runs-on: ubuntu-latest

examples/spark-with-S3/Dockerfile

@@ -4,19 +4,21 @@ FROM debian:bookworm
 
 RUN apt-get update
 ARG SPARK_INPUT_VERSION=3.3.1
-ARG PYTHON_VERSION=3.9.15
+ARG PYTHON_VERSION=3.9
 
 # Python
 RUN apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
   libreadline-dev libsqlite3-dev wget curl unzip llvm libncurses5-dev libncursesw5-dev \
-  xz-utils tk-dev libffi-dev liblzma-dev
-RUN curl https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz --output /usr/python.tgz
-RUN tar xf /usr/python.tgz -C /usr && rm /usr/python.tgz
-RUN cd /usr/Python-${PYTHON_VERSION} \ 
- && ./configure --enable-shared \
- && make -j8 \
- && make altinstall \
- && cd -
+  xz-utils tk-dev libffi-dev liblzma-dev git
+RUN curl https://pyenv.run | bash
+
+ENV PYENV_ROOT="/root/.pyenv"
+ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH"
+
+RUN eval "$(pyenv init -)" 
+RUN pyenv install ${PYTHON_VERSION} 
+RUN pyenv global ${PYTHON_VERSION}
+RUN pyenv rehash
 
 # Java
 RUN apt-get install -y openjdk-17-jre \

setup.py

@@ -28,7 +28,7 @@ def _read_reqs(relpath):
     "Intended Audience :: Science/Research",
     "Operating System :: POSIX :: Linux",
     "Environment :: Console",
-    "License :: OSI Approved :: Apache Software License",
+    "License :: OSI Approved :: Apache License, Version 2.0 (Apache-2.0)",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
@@ -43,7 +43,9 @@ def _read_reqs(relpath):
     packages=setuptools.find_packages(),
     version=versioneer.get_version(),
     install_requires=REQUIREMENTS,
-    tests_require=["pytest"],
+    extras_require={
+        "tests": ["pytest"]
+    },
     python_requires=">=3.8",
     maintainer="Criteo",
     maintainer_email="[email protected]",

tests-requirements.txt

@@ -1,5 +1,5 @@
 pypandoc<1.8
-pyspark
+pyspark<4.0.0
 pytest
 pyflakes==2.4.0
 pylama