ci: Update to the CI/CD pipeline via github workflow to help cut turn-around time (#2189)

Update to the CI/CD pipeline via github workflow to help cut turn-around time.

* added github workflow
* changed the number of pytest-xdist workers from "auto" to "8" (based on local tests and discussion with Tim, choosing auto sometimes takes longer to run than choosing a smaller number. I suspect this is partly because for small or short tests the overhead needed to setup a worker exceeds the time savings of having extra workers).
* modified numerous tests to explicitly include a project path to avoid an attempt to find the project by making an external call via the pydata-google-auth workflow (which opens an input and waits for response from the user that never comes).

Author: chalmerlowe

.github/workflows/unittest.yml

@@ -0,0 +1,89 @@
+on:
+  pull_request:
+    branches:
+      - main
+name: unittest
+jobs:
+  unit:
+    # Use `ubuntu-latest` runner.
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ['3.9', '3.11', '3.12', '3.13']
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install nox
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install nox
+    - name: Run unit tests
+      env:
+        COVERAGE_FILE: .coverage-${{ matrix.python }}
+      run: |
+        nox -s unit-${{ matrix.python }}
+    - name: Upload coverage results
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-artifact-${{ matrix.python }}
+        path: .coverage-${{ matrix.python }}
+        include-hidden-files: true
+
+  unit_noextras:
+    # Use `ubuntu-latest` runner.
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: ['3.9', '3.13']
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install nox
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install nox
+    - name: Run unit_noextras tests
+      env:
+        COVERAGE_FILE: .coverage-unit-noextras-${{ matrix.python }}
+      run: |
+        nox -s unit_noextras-${{ matrix.python }}
+    - name: Upload coverage results
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-artifact-unit-noextras-${{ matrix.python }}
+        path: .coverage-unit-noextras-${{ matrix.python }}
+        include-hidden-files: true
+
+  cover:
+    runs-on: ubuntu-latest
+    needs:
+        - unit
+        - unit_noextras
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.9"
+    - name: Install coverage
+      run: |
+        python -m pip install --upgrade setuptools pip wheel
+        python -m pip install coverage
+    - name: Download coverage results
+      uses: actions/download-artifact@v4
+      with:
+        path: .coverage-results/
+    - name: Report coverage results
+      run: |
+        find .coverage-results -type f -name '*.zip' -exec unzip {} \;
+        coverage combine .coverage-results/**/.coverage*
+        coverage report --show-missing --fail-under=100

noxfile.py

@@ -128,7 +128,7 @@ def default(session, install_extras=True):
     # Run py.test against the unit tests.
     session.run(
         "py.test",
-        "-n=auto",
+        "-n=8",
         "--quiet",
         "-W default::PendingDeprecationWarning",
         "--cov=google/cloud/bigquery",

tests/unit/test_magics.py

@@ -480,6 +480,7 @@ def test_bigquery_magic_without_optional_arguments(monkeypatch):
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
+    magics.context.project = "unit-test-project"
     query_job_mock = mock.create_autospec(
         google.cloud.bigquery.job.QueryJob, instance=True
     )
@@ -831,6 +832,7 @@ def test_bigquery_magic_w_max_results_query_job_results_fails(monkeypatch):
     assert close_transports.called
 
 
+@pytest.mark.usefixtures("ipython_interactive")
 def test_bigquery_magic_w_table_id_invalid(monkeypatch):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
@@ -861,6 +863,7 @@ def test_bigquery_magic_w_table_id_invalid(monkeypatch):
     assert "Traceback (most recent call last)" not in output
 
 
+@pytest.mark.usefixtures("ipython_interactive")
 def test_bigquery_magic_w_missing_query(monkeypatch):
     ip = IPython.get_ipython()
     monkeypatch.setattr(bigquery, "bigquery_magics", None)
@@ -1354,6 +1357,8 @@ def test_bigquery_magic_w_progress_bar_type_w_context_setter(monkeypatch):
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
+    magics.context.project = "unit-test-project"
+
     query_job_mock = mock.create_autospec(
         google.cloud.bigquery.job.QueryJob, instance=True
     )
@@ -1383,6 +1388,8 @@ def test_bigquery_magic_with_progress_bar_type(monkeypatch):
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
+    magics.context.project = "unit-test-project"
+
     with run_query_patch as run_query_mock:
         ip.run_cell_magic(
             "bigquery", "--progress_bar_type=tqdm_gui", "SELECT 17 as num"
@@ -1565,6 +1572,8 @@ def test_bigquery_magic_with_string_params(ipython_ns_cleanup, monkeypatch):
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
+    magics.context.project = "unit-test-project"
+
     query_job_mock = mock.create_autospec(
         google.cloud.bigquery.job.QueryJob, instance=True
     )
@@ -1605,6 +1614,8 @@ def test_bigquery_magic_with_dict_params(ipython_ns_cleanup, monkeypatch):
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
+    magics.context.project = "unit-test-project"
+
     query_job_mock = mock.create_autospec(
         google.cloud.bigquery.job.QueryJob, instance=True
     )
@@ -1689,6 +1700,7 @@ def test_bigquery_magic_with_option_value_incorrect(monkeypatch):
     magics.context.credentials = mock.create_autospec(
         google.auth.credentials.Credentials, instance=True
     )
+    magics.context.project = "unit-test-project"
 
     sql = "SELECT @foo AS foo"
 
@@ -1719,6 +1731,8 @@ def test_bigquery_magic_with_dict_params_negative_value(
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
+    magics.context.project = "unit-test-project"
+
     query_job_mock = mock.create_autospec(
         google.cloud.bigquery.job.QueryJob, instance=True
     )
@@ -1760,6 +1774,8 @@ def test_bigquery_magic_with_dict_params_array_value(ipython_ns_cleanup, monkeyp
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
+    magics.context.project = "unit-test-project"
+
     query_job_mock = mock.create_autospec(
         google.cloud.bigquery.job.QueryJob, instance=True
     )
@@ -1801,6 +1817,8 @@ def test_bigquery_magic_with_dict_params_tuple_value(ipython_ns_cleanup, monkeyp
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
+    magics.context.project = "unit-test-project"
+
     query_job_mock = mock.create_autospec(
         google.cloud.bigquery.job.QueryJob, instance=True
     )
@@ -1852,6 +1870,7 @@ def test_bigquery_magic_valid_query_in_existing_variable(
     magics.context.credentials = mock.create_autospec(
         google.auth.credentials.Credentials, instance=True
     )
+    magics.context.project = "unit-test-project"
 
     ipython_ns_cleanup.append((ip, "custom_query"))
     ipython_ns_cleanup.append((ip, "query_results_df"))
@@ -1892,6 +1911,7 @@ def test_bigquery_magic_nonexisting_query_variable(monkeypatch):
     magics.context.credentials = mock.create_autospec(
         google.auth.credentials.Credentials, instance=True
     )
+    magics.context.project = "unit-test-project"
 
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
@@ -1917,7 +1937,7 @@ def test_bigquery_magic_empty_query_variable_name(monkeypatch):
     magics.context.credentials = mock.create_autospec(
         google.auth.credentials.Credentials, instance=True
     )
-
+    magics.context.project = "unit-test-project"
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
     )
@@ -1940,6 +1960,7 @@ def test_bigquery_magic_query_variable_non_string(ipython_ns_cleanup, monkeypatc
     magics.context.credentials = mock.create_autospec(
         google.auth.credentials.Credentials, instance=True
     )
+    magics.context.project = "unit-test-project"
 
     run_query_patch = mock.patch(
         "google.cloud.bigquery.magics.magics._run_query", autospec=True
@@ -1968,9 +1989,14 @@ def test_bigquery_magic_query_variable_not_identifier(monkeypatch):
         google.auth.credentials.Credentials, instance=True
     )
 
+    magics.context.project = "unit-test-project"
     cell_body = "$123foo"  # 123foo is not valid Python identifier
 
-    with io.capture_output() as captured_io:
+    run_query_patch = mock.patch(
+        "google.cloud.bigquery.magics.magics._run_query", autospec=True
+    )
+
+    with run_query_patch, io.capture_output() as captured_io:
         ip.run_cell_magic("bigquery", "", cell_body)
 
     # If "$" prefixes a string that is not a Python identifier, we do not treat such