diff --git a/.github/actions/package_lambda_solarwinds_apm_aarch64/action.yaml b/.github/actions/package_lambda_solarwinds_apm_aarch64/action.yaml index 83a3f73c4..727c03e1f 100644 --- a/.github/actions/package_lambda_solarwinds_apm_aarch64/action.yaml +++ b/.github/actions/package_lambda_solarwinds_apm_aarch64/action.yaml @@ -13,6 +13,6 @@ runs: image: quay.io/pypa/manylinux_2_28_aarch64:latest entrypoint: 'make' args: - - 'aws-lambda' + - 'aws-lambda-custom' env: PLATFORM: aarch64 \ No newline at end of file diff --git a/.github/actions/package_lambda_solarwinds_apm_x86_64/action.yaml b/.github/actions/package_lambda_solarwinds_apm_x86_64/action.yaml index 73d1e911a..00d1ac7c9 100644 --- a/.github/actions/package_lambda_solarwinds_apm_x86_64/action.yaml +++ b/.github/actions/package_lambda_solarwinds_apm_x86_64/action.yaml @@ -13,6 +13,6 @@ runs: image: quay.io/pypa/manylinux_2_28_x86_64:latest entrypoint: 'make' args: - - 'aws-lambda' + - 'aws-lambda-custom' env: PLATFORM: x86_64 \ No newline at end of file diff --git a/.github/workflows/build_publish_lambda_layer.yaml b/.github/workflows/build_publish_lambda_layer.yaml index 5243e3a7c..9559942cd 100644 --- a/.github/workflows/build_publish_lambda_layer.yaml +++ b/.github/workflows/build_publish_lambda_layer.yaml @@ -24,9 +24,15 @@ jobs: strategy: matrix: python-minor: ["8", "9", "10", "11"] - apm-env: ["lambda"] + apm-env: ["lambda-gh"] steps: - uses: actions/checkout@v4 + - name: Checkout custom instrumentor + uses: actions/checkout@v4 + with: + repository: tammy-baylis-swi/opentelemetry-python-contrib + ref: custom-sw-build + path: contrib-custom - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/build_publish_lambda_layer_aarch64.yaml b/.github/workflows/build_publish_lambda_layer_aarch64.yaml index 4e1fd2eba..7eeadba60 100644 --- a/.github/workflows/build_publish_lambda_layer_aarch64.yaml +++ b/.github/workflows/build_publish_lambda_layer_aarch64.yaml @@ -56,6 +56,11 @@ jobs: SW_APM_VERSION: ${{ steps.save-apm-python-version.outputs.SW_APM_VERSION }} steps: - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: tammy-baylis-swi/opentelemetry-python-contrib + ref: build-lambda-handler-exceptions + path: contrib-custom - uses: ./.github/actions/package_lambda_solarwinds_apm_aarch64 - name: Save APM Python Version for naming id: save-apm-python-version diff --git a/.github/workflows/build_publish_lambda_layer_x86_64.yaml b/.github/workflows/build_publish_lambda_layer_x86_64.yaml index 67164ff99..0d3265ceb 100644 --- a/.github/workflows/build_publish_lambda_layer_x86_64.yaml +++ b/.github/workflows/build_publish_lambda_layer_x86_64.yaml @@ -21,6 +21,11 @@ jobs: SW_APM_VERSION: ${{ steps.save-apm-python-version.outputs.SW_APM_VERSION }} steps: - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: tammy-baylis-swi/opentelemetry-python-contrib + ref: build-lambda-handler-exceptions + path: contrib-custom - uses: ./.github/actions/package_lambda_solarwinds_apm_x86_64 - name: Save APM Python Version for naming id: save-apm-python-version diff --git a/.gitignore b/.gitignore index 294f4e2ec..7e1fcf3c2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ __pycache__/ build/ develop-eggs/ dist/ +dist_orm/ downloads/ eggs/ .eggs/ @@ -28,6 +29,7 @@ share/python-wheels/ *.egg MANIFEST log.txt +tmp-lambda/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/Makefile b/Makefile index 95cae169e..02a8281f7 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,89 @@ aws-lambda: check-zip wrapper install-lambda-modules check-lambda-modules @rm -rf ${target_dir} ./build @echo -e "\nDone." +target_dir := "./tmp-lambda" +install-lambda-modules-custom: + @if [ -f ./dist/solarwinds_apm_lambda_${platform}.zip ]; then \ + echo -e "Deleting old solarwinds_apm_lambda_${platform}.zip"; \ + rm ./dist/solarwinds_apm_lambda_${platform}.zip; \ + fi + rm -rf ${target_dir} + @echo -e "Creating target directory ${target_dir} for AWS Lambda layer artifacts." + mkdir -p ${target_dir}/python + @echo -e "Install upstream dependencies to include in layer" + @/opt/python/cp38-cp38/bin/pip3.8 install -t ${target_dir}/python -r lambda/requirements-custom.txt + @echo -e "Install other version-specific .so files for deps" + @set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311; do /opt/python/$${PYBIN}/bin/pip install -t ${target_dir}/$${PYBIN} -r lambda/requirements-so.txt; done + @set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311; do cp ${target_dir}/$${PYBIN}/charset_normalizer/*.so ${target_dir}/python/charset_normalizer/ && cp ${target_dir}/$${PYBIN}/grpc/_cython/*.so ${target_dir}/python/grpc/_cython/ && cp ${target_dir}/$${PYBIN}/wrapt/*.so ${target_dir}/python/wrapt/; done + @set -e; for PYBIN in cp39-cp39 cp310-cp310 cp311-cp311; do rm -rf ${target_dir}/$${PYBIN} ; done + @echo -e "Install upstream dependencies without deps to include in layer" + @/opt/python/cp38-cp38/bin/pip3.8 install -t ${target_dir}/nodeps -r lambda/requirements-nodeps.txt --no-deps + @echo "Contents of contrib-custom:" + @ls -al contrib-custom/ + @echo -e "Install aws-lambda instrumentor from local to include in layer" + @/opt/python/cp38-cp38/bin/pip3.8 install -t ${target_dir}/nodeps -Ie contrib-custom/instrumentation/opentelemetry-instrumentation-aws-lambda + @echo -e "Directly copying aws-lambda src files" + @mkdir ${target_dir}/nodeps/opentelemetry/instrumentation/aws_lambda + @cp -r contrib-custom/instrumentation/opentelemetry-instrumentation-aws-lambda/src/opentelemetry/instrumentation/aws_lambda/* ${target_dir}/nodeps/opentelemetry/instrumentation/aws_lambda + @echo -e "Install solarwinds_apm to be packed up in zip archive to target directory." + @/opt/python/cp38-cp38/bin/pip3.8 install . -t ${target_dir}/nodeps --no-deps + @echo -e "Removing non-lambda C-extension library files generated by pip install under target directory." + @rm -rf ${target_dir}/nodeps/solarwinds_apm/extension/*.so* + @echo -e "Building AWS Lambda version of C-extensions for all supported Python versions in target directory." + @set -e; for PYBIN in cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311; do /opt/python/$${PYBIN}/bin/python setup.py build_ext -b ${target_dir}/nodeps; done + @echo -e "Copying AWS Lambda specific Oboe library liboboe-1.0-lambda-${platform}.so into target directory." + @cp solarwinds_apm/extension/liboboe-1.0-lambda-${platform}.so ${target_dir}/nodeps/solarwinds_apm/extension/liboboe.so + @echo -e "Moving no-deps dependencies, needed for full opentelemetry/instrumentation path" + @cp -r ${target_dir}/nodeps/* ${target_dir}/python + @rm -rf ${target_dir}/nodeps + @echo -e "Copying OpenTelemetry lambda wrapper and entry script into target directory." + @cp lambda/otel_wrapper.py ${target_dir}/python/otel_wrapper.py + @mkdir ${target_dir}/solarwinds-apm/ + @cp lambda/solarwinds-apm/wrapper ${target_dir}/solarwinds-apm/wrapper + @chmod 755 ${target_dir}/solarwinds-apm/wrapper + @echo -e "Removing unnecessary boto, six, setuptools, urllib3 installations" + @rm -rf ${target_dir}/python/boto* + @rm -rf ${target_dir}/python/six* + @rm -rf ${target_dir}/python/setuptools* + @rm -rf ${target_dir}/python/urllib3* + @find ${target_dir}/python -type d -name '__pycache__' | xargs rm -rf + +check-lambda-modules-custom: + ./lambda/check_lambda_modules.sh ${target_dir} + +# Build APM Python AWS lambda layer as zip artifact +# with extension compatible with current environment +# (x86_64 OR aarch64) using custom contrib modules +target_dir := "./tmp-lambda" +aws-lambda-custom: export AWS_LAMBDA_FUNCTION_NAME = set-for-build +aws-lambda-custom: export LAMBDA_TASK_ROOT = set-for-build +aws-lambda-custom: check-zip wrapper install-lambda-modules-custom check-lambda-modules-custom + @if [[ ! -d dist ]]; then mkdir dist; fi + @pushd ${target_dir} && zip -r ../dist/solarwinds_apm_lambda_${platform}.zip . && popd + @rm -rf ${target_dir} ./build + @echo -e "\nDone." + +# Build Python ORM AWS lambda layer as zip artifact +# with py38 extensions compatible with current environment +# (x86_64 OR aarch64) +target_dir := "./tmp-lambda" +aws-lambda-orm: check-zip + @if [ -f ./dist/orm_lambda_${platform}.zip ]; then \ + echo -e "Deleting old orm_lambda_${platform}.zip"; \ + rm ./dist/orm_lambda_${platform}.zip; \ + fi + rm -rf ${target_dir} + @echo -e "Creating target directory ${target_dir} for AWS Lambda layer artifacts." + mkdir -p ${target_dir}/python + @echo -e "Install ORM modules to include in layer" + @/opt/python/cp38-cp38/bin/pip3.8 install -t ${target_dir}/python -r lambda/requirements-orm.txt + @find ${target_dir}/python -type d -name '__pycache__' | xargs rm -rf + @echo -e "Preparing ORM layer archive" + @if [[ ! -d dist_orm ]]; then mkdir dist_orm; fi + @pushd ${target_dir} && zip -r ../dist_orm/orm_lambda_${platform}.zip . && popd + @rm -rf ${target_dir} ./build + @echo -e "\nDone." + #----------------------------------------------------------------------------------------------------------------------# # recipes for local development #----------------------------------------------------------------------------------------------------------------------# @@ -314,6 +397,7 @@ clean: @find . -type d -name '*.ropeproject' | xargs rm -rf @rm -rf build/ @rm -rf dist/ + @rm -rf dist_orm/ @rm -rf *.egg* @rm -f MANIFEST @rm -rf docs/build/ diff --git a/docker-compose.yml b/docker-compose.yml index 4ea9ebcdb..bd2c4e5cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ version: '2.4' services: x86_64: image: quay.io/pypa/manylinux_2_28_x86_64 + user: root stdin_open: true tty: true network_mode: host @@ -19,6 +20,7 @@ services: volumes: - ./:/code/solarwinds_apm - ../solarwinds-apm-liboboe/:/code/solarwinds-apm-liboboe + - ../opentelemetry-python-contrib/:/code/solarwinds_apm/contrib-custom/ working_dir: /code/solarwinds_apm entrypoint: ["/bin/bash", "-c"] command: @@ -27,10 +29,13 @@ services: # twine to upload to TestPyPi # tox for automated tests python3.8 -m pip install boto3 twine tox + # update wrapper permissions for local testing in docker + chmod 755 /code/solarwinds_apm/lambda/solarwinds-apm/wrapper /bin/bash aarch64: image: quay.io/pypa/manylinux_2_28_aarch64 + user: root stdin_open: true tty: true network_mode: host @@ -39,6 +44,7 @@ services: volumes: - ./:/code/solarwinds_apm - ../solarwinds-apm-liboboe/:/code/solarwinds-apm-liboboe + - ../opentelemetry-python-contrib/:/code/solarwinds_apm/contrib-custom/ working_dir: /code/solarwinds_apm entrypoint: ["/bin/bash", "-c"] command: @@ -47,4 +53,6 @@ services: # twine to upload to TestPyPi # tox for automated tests python3.8 -m pip install boto3 twine tox + # update wrapper permissions for local testing in docker + chmod 755 /code/solarwinds_apm/lambda/solarwinds-apm/wrapper /bin/bash diff --git a/lambda/requirements-custom.txt b/lambda/requirements-custom.txt new file mode 100644 index 000000000..7509330da --- /dev/null +++ b/lambda/requirements-custom.txt @@ -0,0 +1,7 @@ +opentelemetry-api==1.25.0 +opentelemetry-sdk==1.25.0 +opentelemetry-instrumentation==0.46b0 +opentelemetry-propagator-aws-xray==1.0.1 +opentelemetry-exporter-otlp==1.25.0 +opentelemetry-exporter-otlp-proto-grpc==1.25.0 +opentelemetry-exporter-otlp-proto-http==1.25.0 \ No newline at end of file diff --git a/lambda/requirements-nodeps.txt b/lambda/requirements-nodeps.txt index 1659a541e..b6c9449ec 100644 --- a/lambda/requirements-nodeps.txt +++ b/lambda/requirements-nodeps.txt @@ -27,4 +27,4 @@ opentelemetry-instrumentation-sqlalchemy==0.46b0 opentelemetry-instrumentation-sqlite3==0.46b0 opentelemetry-instrumentation-starlette==0.46b0 opentelemetry-instrumentation-tornado==0.46b0 -opentelemetry-instrumentation-wsgi==0.46b0 \ No newline at end of file +opentelemetry-instrumentation-wsgi==0.46b0 diff --git a/lambda/requirements-orm.txt b/lambda/requirements-orm.txt new file mode 100644 index 000000000..69ecfd41c --- /dev/null +++ b/lambda/requirements-orm.txt @@ -0,0 +1,2 @@ +SQLAlchemy==2.0.28 +psycopg2_binary==2.9.9 \ No newline at end of file diff --git a/lambda/tests/requirements.txt b/lambda/tests/requirements.txt index 8c3ea8a5d..424b908e7 100644 --- a/lambda/tests/requirements.txt +++ b/lambda/tests/requirements.txt @@ -1,2 +1,4 @@ -opentelemetry-instrumentation-aws-lambda == 0.46b0 opentelemetry-propagator-aws-xray == 1.0.1 +# This is a gross workaround to test on `custom` branch +# when pyproject.toml doesn't install upstream deps +opentelemetry-instrumentation-logging==0.46b0 diff --git a/pyproject.toml b/pyproject.toml index 8a633b02f..e388a59b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,10 +41,10 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - 'opentelemetry-api == 1.25.0', - 'opentelemetry-sdk == 1.25.0', - 'opentelemetry-instrumentation == 0.46b0', - 'opentelemetry-instrumentation-logging == 0.46b0', + # 'opentelemetry-api == 1.25.0', + # 'opentelemetry-sdk == 1.25.0', + # 'opentelemetry-instrumentation == 0.46b0', + # 'opentelemetry-instrumentation-logging == 0.46b0', ] [project.urls] diff --git a/tox.ini b/tox.ini index 07f646a37..cdac2dfa4 100644 --- a/tox.ini +++ b/tox.ini @@ -33,12 +33,17 @@ setenv = [testenv:py3{8,9,10,11}-lambda] changedir = lambda/tests +; Currently installs custom checkout for custom build tests commands_pre = + py3{8,9,10,11}-lambda: pip install -Ie /code/solarwinds_apm/contrib-custom/instrumentation/opentelemetry-instrumentation-aws-lambda py3{8,9,10,11}-lambda: pip install -r requirements.txt [testenv:py3{8,9,10,11}-lambda-gh] changedir = lambda/tests +; Currently installs custom checkout for custom build tests +; assuming GH runner filepath commands_pre = + py3{8,9,10,11}-lambda: pip install -Ie /home/runner/work/apm-python/apm-python/contrib-custom/instrumentation/opentelemetry-instrumentation-aws-lambda py3{8,9,10,11}-lambda: pip install -r requirements.txt [testenv:lint]