Initial Testing Suite

.github/actions/load-data/action.yml

@@ -13,17 +13,18 @@ runs:
         rclone_config: ${{ inputs.rclone-config }}
 
     - name: Get dataset version hash
+      id: hash
       shell: bash
       run: |
-        HASH=$(rclone lsl remote:"SampleData" --drive-shared-with-me)
+        HASH=$(rclone lsjson remote:"SampleData" --drive-shared-with-me --recursive | sed 's/,$//' | sort | sha256sum | cut -d' ' -f1)
         echo "DATASET_HASH=$HASH" >> $GITHUB_OUTPUT
 
     - name: Cache datasets
       uses: actions/cache@v4
       id: cache-datasets
       with:
         path: ./testing_data
-        key: ephys-datasets-${{ steps.ephys.outputs.DATASET_HASH }}
+        key: ${{ steps.hash.outputs.DATASET_HASH }}
         enableCrossOsArchive: true
 
     - if: ${{ steps.cache-datasets.outputs.cache-hit != 'true' }}

.github/workflows/all_os_versions.txt

@@ -0,0 +1 @@
+["ubuntu-latest", "macos-latest", "windows-2022"]

.github/workflows/all_python_versions.txt

@@ -0,0 +1 @@
+["3.10", "3.11", "3.12", "3.13"]

.github/workflows/pr-tests.yml

@@ -10,4 +10,43 @@ on:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: true
+
+jobs:
+  load_python_and_os_versions:
+    runs-on: ubuntu-latest
+    outputs:
+      ALL_PYTHON_VERSIONS: ${{ steps.load_python_versions.outputs.python_versions }}
+      ALL_OS_VERSIONS: ${{ steps.load_os_versions.outputs.os_versions }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: load_python_versions
+        run: echo "python_versions=$(cat ./.github/workflows/all_python_versions.txt)" >> "$GITHUB_OUTPUT"
+      - id: load_os_versions
+        run: echo "os_versions=$(cat ./.github/workflows/all_os_versions.txt)" >> "$GITHUB_OUTPUT"
+      - name: Debugging
+        run: |
+          echo "Loaded Python versions: ${{ steps.load_python_versions.outputs.python_versions }}"
+          echo "Loaded OS versions: ${{ steps.load_os_versions.outputs.os_versions }}"
+
+  run-tests:
+    needs: [load_python_and_os_versions]
+    uses: ./.github/workflows/run-tests.yml
+    secrets:
+      RCLONE_CONFIG: ${{ secrets.RCLONE_CONFIG }}
+    with:  #  Ternary operator: condition && value_if_true || value_if_false
+      python-versions: ${{ github.event.pull_request.draft == true && '["3.10"]' || needs.load_python_and_os_versions.outputs.ALL_PYTHON_VERSIONS }}
+      os-versions: ${{ github.event.pull_request.draft == true && '["ubuntu-latest"]' || needs.load_python_and_os_versions.outputs.ALL_OS_VERSIONS }}
+
+  check-final-status:
+    name: All tests passing
+    if: always()
+    needs:
+      - run-tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Decide whether all jobs succeeded or at least one failed
+        uses: re-actors/alls-green@release/v1
+        with:
+          allowed-skips: run-tests
+          jobs: ${{ toJSON(needs) }}

.github/workflows/run-tests.yml

@@ -11,4 +11,50 @@ on:
         description: 'List of OS versions to use in matrix, as JSON string'
         required: true
         type: string
-  workflow_dispatch:
+    secrets:
+      RCLONE_CONFIG:
+        required: true
+  workflow_dispatch:
+    inputs:
+      python-versions:
+        description: 'List of Python versions to use in matrix, as JSON string'
+        required: true
+        type: string
+        default: '["3.10", "3.11", "3.12", "3.13"]'
+      os-versions:
+        description: 'List of OS versions to use in matrix, as JSON string'
+        required: true
+        type: string
+        default: '["ubuntu-latest", "windows-2022", "macos-latest"]'
+
+jobs:
+  run:
+    name: ${{ matrix.os }} Python ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ${{ fromJson(inputs.python-versions) }}
+        os: ${{ fromJson(inputs.os-versions) }}
+    steps:
+      - uses: actions/checkout@v5
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pip
+        run: python -m pip install -U pip  # Official recommended way
+
+      - name: Install GuPPy with testing requirements
+        run: |
+          python -m pip install "."
+          python -m pip install --group test
+
+      - name: Prepare data for tests
+        uses: ./.github/actions/load-data
+        with:
+          rclone-config: ${{ secrets.RCLONE_CONFIG }}
+
+      - name: Run tests
+        run: pytest tests -vv -rsx # -n auto --dist loadscope # TODO: re-enable parallel execution when logging issues with Windows are resolved

.gitignore

@@ -6,3 +6,6 @@ z-score_methods.tgn
 GuPPy/runFiberPhotometryAnalysis.ipynb
 .vscode/
 *.egg-info/
+.clinerules/
+
+testing_data/

pyproject.toml

@@ -56,6 +56,13 @@ dependencies = [
     "tables",
 ]
 
+[dependency-groups]
+test = [
+    "pytest",
+    "pytest-cov",
+    "pytest-xdist"  # Runs tests on parallel
+]
+
 [project.scripts]
 guppy = "guppy.main:main"
 

src/guppy/preprocess.py

@@ -17,7 +17,10 @@
 from matplotlib.widgets import MultiCursor
 from pathlib import Path
 from .combineDataFn import processTimestampsForCombiningData
-plt.switch_backend('TKAgg')
+
+# Only set matplotlib backend if not in CI environment
+if not os.getenv('CI'):
+	plt.switch_backend('TKAgg')
 
 def takeOnlyDirs(paths):
 	removePaths = []

src/guppy/saveStoresList.py

@@ -133,6 +133,16 @@ def saveStorenames(inputParameters, data, event_name, flag, filepath):
     # getting input parameters
     inputParameters = inputParameters
 
+    # Headless path: if storenames_map provided, write storesList.csv without building the Panel UI
+    storenames_map = inputParameters.get("storenames_map")
+    if isinstance(storenames_map, dict) and len(storenames_map) > 0:
+        op = make_dir(filepath)
+        arr = np.asarray([list(storenames_map.keys()), list(storenames_map.values())], dtype=str)
+        np.savetxt(os.path.join(op, 'storesList.csv'), arr, delimiter=",", fmt='%s')
+        insertLog(f"Storeslist file saved at {op}", logging.INFO)
+        insertLog('Storeslist : \n'+str(arr), logging.INFO)
+        return
+
     # reading storenames from the data fetched using 'readtsq' function
     if isinstance(data, pd.DataFrame):
         data['name'] = np.asarray(data['name'], dtype=str)
@@ -583,7 +593,7 @@ def check_channels(state):
     return unique_state.shape[0], unique_state
 
 # function to decide NPM timestamps unit (seconds, ms or us)
-def decide_ts_unit_for_npm(df):
+def decide_ts_unit_for_npm(df, timestamp_column_name=None, time_unit=None, headless=False):
     col_names = np.array(list(df.columns))
     col_names_ts = ['']
     for name in col_names:
@@ -592,6 +602,18 @@ def decide_ts_unit_for_npm(df):
 
     ts_unit = 'seconds'
     if len(col_names_ts)>2:
+        # Headless path: auto-select column/unit without any UI
+        if headless:
+            if timestamp_column_name is not None:
+                assert timestamp_column_name in col_names_ts, f"Provided timestamp_column_name '{timestamp_column_name}' not found in columns {col_names_ts[1:]}"
+                chosen = timestamp_column_name
+            else:
+                chosen = col_names_ts[1]
+            df.insert(1, 'Timestamp', df[chosen])
+            df = df.drop(col_names_ts[1:], axis=1)
+            valid_units = {'seconds', 'milliseconds', 'microseconds'}
+            ts_unit = time_unit if (isinstance(time_unit, str) and time_unit in valid_units) else 'seconds'
+            return df, ts_unit
         #def comboBoxSelected(event):
         #    print(event.widget.get())
 
@@ -741,10 +763,19 @@ def read_doric(filepath):
 # and recognize type of 'csv' files either from
 # Neurophotometrics, Doric systems or custom made 'csv' files
 # and read data accordingly
-def import_np_doric_csv(filepath, isosbestic_control, num_ch):
+def import_np_doric_csv(filepath, isosbestic_control, num_ch, inputParameters=None):
 
     insertLog("If it exists, importing either NPM or Doric or csv file based on the structure of file",
               logging.DEBUG)
+    # Headless configuration (used to avoid any UI prompts when running tests)
+    headless = bool(os.environ.get('GUPPY_BASE_DIR'))
+    npm_timestamp_column_name = None
+    npm_time_unit = None
+    npm_split_events = None
+    if isinstance(inputParameters, dict):
+        npm_timestamp_column_name = inputParameters.get('npm_timestamp_column_name')
+        npm_time_unit = inputParameters.get('npm_time_unit', 'seconds')
+        npm_split_events = inputParameters.get('npm_split_events', True)
     path = sorted(glob.glob(os.path.join(filepath, '*.csv'))) + \
            sorted(glob.glob(os.path.join(filepath, '*.doric')))
     path_chev = glob.glob(os.path.join(filepath, '*chev*'))
@@ -879,16 +910,19 @@ def import_np_doric_csv(filepath, isosbestic_control, num_ch):
             elif flag=='event_np':
                 type_val = np.array(df.iloc[:,1])
                 type_val_unique = np.unique(type_val)
-                window = tk.Tk()
-                if len(type_val_unique)>1:
-                    response = messagebox.askyesno('Multiple event TTLs', 'Based on the TTL file,\
+                if headless:
+                    response = 1 if bool(npm_split_events) else 0
+                else:
+                    window = tk.Tk()
+                    if len(type_val_unique)>1:
+                        response = messagebox.askyesno('Multiple event TTLs', 'Based on the TTL file,\
                                                                             it looks like TTLs \
                                                                             belongs to multipe behavior type. \
                                                                             Do you want to create multiple files for each \
                                                                             behavior type ?')
-                else:
-                    response = 0
-                window.destroy()
+                    else:
+                        response = 0
+                    window.destroy()
                 if response==1:
                     timestamps = np.array(df.iloc[:,0])
                     for j in range(len(type_val_unique)):
@@ -907,7 +941,12 @@ def import_np_doric_csv(filepath, isosbestic_control, num_ch):
                     event_from_filename.append('event'+str(0))
             else:
                 file = f'file{str(i)}_'
-                df, ts_unit = decide_ts_unit_for_npm(df)
+                df, ts_unit = decide_ts_unit_for_npm(
+                    df,
+                    timestamp_column_name=npm_timestamp_column_name,
+                    time_unit=npm_time_unit,
+                    headless=headless
+                )
                 df, indices_dict, num_channels = decide_indices(file, df, flag)
                 keys = list(indices_dict.keys())
                 for k in range(len(keys)):
@@ -1004,7 +1043,7 @@ def execute(inputParameters):
         for i in folderNames:
             filepath = os.path.join(inputParameters['abspath'], i)
             data = readtsq(filepath)
-            event_name, flag = import_np_doric_csv(filepath, isosbestic_control, num_ch)
+            event_name, flag = import_np_doric_csv(filepath, isosbestic_control, num_ch, inputParameters=inputParameters)
             saveStorenames(inputParameters, data, event_name, flag, filepath)
         insertLog('#'*400, logging.INFO)
     except Exception as e:

src/guppy/savingInputParameters.py

@@ -24,23 +24,31 @@ def savingInputParameters():
     else:
         pass
 
-    # Create the main window
-    folder_selection = tk.Tk()
-    folder_selection.title("Select the folder path where your data is located")
-    folder_selection.geometry("700x200")
-    def select_folder():
+    # Determine base folder path (headless-friendly via env var)
+    base_dir_env = os.environ.get('GUPPY_BASE_DIR')
+    is_headless = base_dir_env and os.path.isdir(base_dir_env)
+    if is_headless:
         global folder_path
-        folder_path = filedialog.askdirectory(title="Select the folder path where your data is located")
-        if folder_path:
-            print(f"Folder path set to {folder_path}")
-            folder_selection.destroy()
-        else:
-            folder_path = os.path.expanduser('~')
-            print(f"Folder path set to {folder_path}")
-
-    select_button = ttk.Button(folder_selection, text="Select a Folder", command=select_folder)
-    select_button.pack(pady=5)
-    folder_selection.mainloop()
+        folder_path = base_dir_env
+        print(f"Folder path set to {folder_path} (from GUPPY_BASE_DIR)")
+    else:
+        # Create the main window
+        folder_selection = tk.Tk()
+        folder_selection.title("Select the folder path where your data is located")
+        folder_selection.geometry("700x200")
+        def select_folder():
+            global folder_path
+            folder_path = filedialog.askdirectory(title="Select the folder path where your data is located")
+            if folder_path:
+                print(f"Folder path set to {folder_path}")
+                folder_selection.destroy()
+            else:
+                folder_path = os.path.expanduser('~')
+                print(f"Folder path set to {folder_path}")
+
+        select_button = ttk.Button(folder_selection, text="Select a Folder", command=select_folder)
+        select_button.pack(pady=5)
+        folder_selection.mainloop()
 
     current_dir = os.getcwd()
 
@@ -525,4 +533,13 @@ def onclickpsth(event=None):
     template.main.append(group)
     template.main.append(visualize)
 
+    # Expose minimal hooks and widgets to enable programmatic testing
+    template._hooks = {
+        "onclickProcess": onclickProcess,
+        "getInputParameters": getInputParameters,
+    }
+    template._widgets = {
+        "files_1": files_1,
+    }
+
     return template

src/guppy/testing/__init__.py

@@ -0,0 +1,8 @@
+from .api import step1, step2, step3, step4
+
+__all__ = [
+    "step1",
+    "step2",
+    "step3",
+    "step4",
+]