Skip to content

Commit f28119f

Browse files
authored
Change some tests to run with restapi (#1644)
* Change some tests to run with restapi Signed-off-by: Chaurasiya, Payal <[email protected]> * Add grpc and rest checks Signed-off-by: Chaurasiya, Payal <[email protected]> * small fix Signed-off-by: Chaurasiya, Payal <[email protected]> * small fix Signed-off-by: Chaurasiya, Payal <[email protected]> * add some fix Signed-off-by: Chaurasiya, Payal <[email protected]> * rebase fix Signed-off-by: Chaurasiya, Payal <[email protected]> * small fix Signed-off-by: Chaurasiya, Payal <[email protected]> * commen resiliency Signed-off-by: Chaurasiya, Payal <[email protected]> * Change rest_api to rest protocol Signed-off-by: Chaurasiya, Payal <[email protected]> * Change rest_api to rest protocol Signed-off-by: Chaurasiya, Payal <[email protected]> * Add one test in analytics Signed-off-by: payalcha <[email protected]> * Add one docker test Signed-off-by: payalcha <[email protected]> * Add one windows test Signed-off-by: payalcha <[email protected]> * small fix Signed-off-by: payalcha <[email protected]> * Small check Signed-off-by: payalcha <[email protected]> * Small check Signed-off-by: payalcha <[email protected]> --------- Signed-off-by: Chaurasiya, Payal <[email protected]> Signed-off-by: payalcha <[email protected]>
1 parent 3a72c98 commit f28119f

19 files changed

+213
-123
lines changed

.github/workflows/pq_pipeline.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ jobs:
131131
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
132132
(github.event_name == 'workflow_dispatch')
133133
name: TaskRunner Dockerized E2E
134-
needs: task_runner_straggler_e2e
134+
needs: task_runner_resiliency_e2e
135135
uses: ./.github/workflows/task_runner_dockerized_ws_e2e.yml
136136
with:
137137
commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}
@@ -173,7 +173,7 @@ jobs:
173173
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
174174
(github.event_name == 'workflow_dispatch')
175175
name: TaskRunner Verifiable Dataset E2E
176-
needs: task_runner_e2e
176+
needs: task_runner_connectivity_e2e
177177
uses: ./.github/workflows/tr_verifiable_dataset_e2e.yml
178178
with:
179179
commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}

.github/workflows/task_runner_connectivity_e2e.yml

Lines changed: 30 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -52,37 +52,36 @@ jobs:
5252
with:
5353
test_type: "TLS_Connectivity_gRPC"
5454

55-
# Uncomment once Rest API PR is merged
56-
# test_rest_connectivity:
57-
# name: Task Runner Rest connectivity (no-op, 3.11, rest)
58-
# if: |
59-
# (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
60-
# (github.event_name == 'workflow_dispatch') ||
61-
# (github.event.pull_request.draft == false)
62-
# runs-on: ubuntu-22.04
63-
# timeout-minutes: 30
64-
# env:
65-
# MODEL_NAME: 'no-op'
66-
# PYTHON_VERSION: '3.11'
67-
# steps:
68-
# - name: Checkout OpenFL repository
69-
# id: checkout_openfl
70-
# uses: actions/checkout@v4
71-
# with:
72-
# ref: ${{ env.COMMIT_ID }}
55+
test_rest_connectivity:
56+
name: Task Runner Rest connectivity (no-op, 3.11)
57+
if: |
58+
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
59+
(github.event_name == 'workflow_dispatch') ||
60+
(github.event.pull_request.draft == false)
61+
runs-on: ubuntu-22.04
62+
timeout-minutes: 30
63+
env:
64+
MODEL_NAME: 'no-op'
65+
PYTHON_VERSION: '3.11'
66+
steps:
67+
- name: Checkout OpenFL repository
68+
id: checkout_openfl
69+
uses: actions/checkout@v4
70+
with:
71+
ref: ${{ env.COMMIT_ID }}
7372

74-
# - name: Pre test run
75-
# uses: ./.github/actions/tr_pre_test_run
76-
# if: ${{ always() }}
73+
- name: Pre test run
74+
uses: ./.github/actions/tr_pre_test_run
75+
if: ${{ always() }}
7776

78-
# - name: Run Task Runner rest connectivity test
79-
# id: run_tests
80-
# run: |
81-
# python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -k test_federation_connectivity --model_name ${{ env.MODEL_NAME }} --tr_rest_api
82-
# echo "Task runner end to end test run completed"
77+
- name: Run Task Runner rest connectivity test
78+
id: run_tests
79+
run: |
80+
python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -k test_federation_connectivity --model_name ${{ env.MODEL_NAME }} --tr_rest_protocol
81+
echo "Task runner end to end test run completed"
8382
84-
# - name: Post test run
85-
# uses: ./.github/actions/tr_post_test_run
86-
# if: ${{ always() }}
87-
# with:
88-
# test_type: "TLS_Connectivity_REST"
83+
- name: Post test run
84+
uses: ./.github/actions/tr_post_test_run
85+
if: ${{ always() }}
86+
with:
87+
test_type: "TLS_Connectivity_REST"

.github/workflows/task_runner_dockerized_ws_e2e.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ jobs:
180180
test_type: "DWS_Without_Client_Auth"
181181

182182
test_memory_logs:
183-
name: With Memory Logs
183+
name: With Memory Logs REST
184184
needs: input_selection
185185
if: needs.input_selection.outputs.selected_jobs == 'all' || needs.input_selection.outputs.selected_jobs == 'test_memory_logs'
186186
runs-on: ubuntu-22.04
@@ -212,11 +212,11 @@ jobs:
212212
python -m pytest -s tests/end_to_end/test_suites/memory_logs_tests.py \
213213
-k test_log_memory_usage_dockerized_ws --model_name ${{ env.MODEL_NAME }} \
214214
--num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} \
215-
--log_memory_usage
215+
--log_memory_usage --tr_rest_protocol
216216
echo "Task runner memory logs test run completed"
217217
218218
- name: Post test run
219219
uses: ./.github/actions/tr_post_test_run
220220
if: ${{ always() }}
221221
with:
222-
test_type: "DWS_With_Memory_Logs"
222+
test_type: "DWS_With_Memory_Logs_REST"

.github/workflows/task_runner_fed_analytics_e2e.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ env:
3737

3838
jobs:
3939
test_fed_analytics_histogram:
40-
name: With TLS (federated_analytics/histogram, 3.11) # DO NOT change this name.
40+
name: With REST (federated_analytics/histogram, 3.11)
4141
runs-on: ubuntu-22.04
4242
timeout-minutes: 30
4343
if: |
@@ -62,7 +62,7 @@ jobs:
6262
- name: Run Federated Analytics Histogram
6363
id: run_tests
6464
run: |
65-
python -m pytest -s tests/end_to_end/test_suites/tr_fed_analytics_tests.py \
65+
python -m pytest -s tests/end_to_end/test_suites/tr_fed_analytics_tests.py --tr_rest_protocol \
6666
-m task_runner_fed_analytics --model_name ${{ env.MODEL_NAME }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
6767
echo "Federated analytics histogram test run completed"
6868
@@ -73,7 +73,7 @@ jobs:
7373
test_type: "Sepal_Histogram_Analytics"
7474

7575
test_fed_analytics_smokers_health:
76-
name: With TLS (federated_analytics/smokers_health, 3.12) # DO NOT change this name.
76+
name: With gRPC (federated_analytics/smokers_health, 3.12)
7777
runs-on: ubuntu-22.04
7878
timeout-minutes: 30
7979
if: |

.github/workflows/task_runner_fedeval_e2e.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,8 @@ jobs:
8080
with:
8181
test_type: "FedEval_With_TLS"
8282

83-
test_without_tls:
84-
name: Without TLS
83+
test_without_tls_rest:
84+
name: Without TLS Using Rest Protocol
8585
if: | # Skip for PR pipeline
8686
((github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
8787
(github.event_name == 'workflow_dispatch')) && (github.workflow != 'OpenFL PR Pipeline')
@@ -115,17 +115,17 @@ jobs:
115115
run: |
116116
python -m pytest -s tests/end_to_end/test_suites/tr_with_fedeval_tests.py \
117117
-m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
118-
--num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls
118+
--num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_tls --tr_rest_protocol
119119
echo "Task runner end to end test run completed"
120120
121121
- name: Post test run
122122
uses: ./.github/actions/tr_post_test_run
123123
if: ${{ always() }}
124124
with:
125-
test_type: "FedEval_Without_TLS"
125+
test_type: "FedEval_Without_TLS_REST"
126126

127-
test_without_client_auth:
128-
name: Without Client Auth
127+
test_without_client_auth_rest:
128+
name: Without ClientAuth Using Rest Protocol
129129
if: | # Skip for PR pipeline
130130
((github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
131131
(github.event_name == 'workflow_dispatch')) && (github.workflow != 'OpenFL PR Pipeline')
@@ -159,11 +159,11 @@ jobs:
159159
run: |
160160
python -m pytest -s tests/end_to_end/test_suites/tr_with_fedeval_tests.py \
161161
-m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
162-
--num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth
162+
--num_rounds ${{ env.NUM_ROUNDS }} --num_collaborators ${{ env.NUM_COLLABORATORS }} --disable_client_auth --tr_rest_protocol
163163
echo "Task runner end to end test run completed"
164164
165165
- name: Post test run
166166
uses: ./.github/actions/tr_post_test_run
167167
if: ${{ always() }}
168168
with:
169-
test_type: "FedEval_Without_Client_Auth"
169+
test_type: "FedEval_Without_Client_Auth_REST"

.github/workflows/task_runner_resiliency_e2e.yml

Lines changed: 38 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -53,50 +53,13 @@ env:
5353
COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow
5454

5555
jobs:
56-
input_selection:
57-
if: |
58-
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
59-
(github.event_name == 'workflow_dispatch') ||
60-
(github.event.pull_request.draft == false)
61-
name: Input value selection
62-
runs-on: ubuntu-22.04
63-
outputs:
64-
# Output all the variables related to models and python versions to be used in the matrix strategy
65-
# for different jobs, however their usage depends on the selected job.
66-
selected_models_for_tls: ${{ steps.input_selection.outputs.models_for_tls }}
67-
selected_python_for_tls: ${{ steps.input_selection.outputs.python_for_tls }}
68-
steps:
69-
- name: Job to select input values
70-
id: input_selection
71-
run: |
72-
if [ "${{ env.MODEL_NAME }}" == "all" ]; then
73-
echo "models_for_tls=[\"torch/mnist\", \"keras/mnist\"]" >> "$GITHUB_OUTPUT"
74-
else
75-
echo "models_for_tls=[\"${{env.MODEL_NAME}}\"]" >> "$GITHUB_OUTPUT"
76-
fi
77-
if [ "${{ env.PYTHON_VERSION }}" == "all" ]; then
78-
echo "python_for_tls=[\"3.10\", \"3.11\"]" >> "$GITHUB_OUTPUT"
79-
else
80-
echo "python_for_tls=[\"${{env.PYTHON_VERSION}}\"]" >> "$GITHUB_OUTPUT"
81-
fi
82-
83-
resiliency_in_native:
84-
name: With TLS
85-
needs: input_selection
56+
resiliency_in_native_gRPC:
57+
name: Resiliency in gRPC (torch/mnist, 3.10)
8658
runs-on: ubuntu-22.04
8759
timeout-minutes: 30
88-
strategy:
89-
matrix:
90-
model_name: ${{ fromJson(needs.input_selection.outputs.selected_models_for_tls) }}
91-
python_version: ${{ fromJson(needs.input_selection.outputs.selected_python_for_tls) }}
92-
exclude: # Keras does not support Python 3.12
93-
- model_name: "keras/mnist"
94-
python_version: "3.12"
95-
fail-fast: false # do not immediately fail if one of the combinations fail
96-
9760
env:
98-
MODEL_NAME: ${{ matrix.model_name }}
99-
PYTHON_VERSION: ${{ matrix.python_version }}
61+
MODEL_NAME: ${{ inputs.model_name || 'torch/mnist' }}
62+
PYTHON_VERSION: ${{ inputs.python_version || '3.10' }}
10063

10164
steps:
10265
- name: Checkout OpenFL repository
@@ -121,6 +84,38 @@ jobs:
12184
uses: ./.github/actions/tr_post_test_run
12285
if: ${{ always() }}
12386
with:
124-
test_type: "Resiliency_Native"
87+
test_type: "Resiliency_Native_gRPC"
88+
89+
# Uncomment below once Issue is resolved - https://github.com/securefederatedai/openfl/issues/1646
90+
# resiliency_in_native_rest:
91+
# name: Resiliency in REST (keras/tensorflow/mnist, 3.11)
92+
# runs-on: ubuntu-22.04
93+
# timeout-minutes: 30
94+
# env:
95+
# MODEL_NAME: ${{ inputs.model_name || 'keras/tensorflow/mnist' }}
96+
# PYTHON_VERSION: ${{ inputs.python_version || '3.11' }}
97+
98+
# steps:
99+
# - name: Checkout OpenFL repository
100+
# id: checkout_openfl
101+
# uses: actions/checkout@v4
102+
# with:
103+
# ref: ${{ env.COMMIT_ID }}
104+
105+
# - name: Pre test run
106+
# uses: ./.github/actions/tr_pre_test_run
107+
# if: ${{ always() }}
108+
109+
# - name: Run Task Runner E2E tests with TLS
110+
# id: run_tests
111+
# run: |
112+
# python -m pytest -s tests/end_to_end/test_suites/tr_resiliency_tests.py \
113+
# -m task_runner_basic --model_name ${{ env.MODEL_NAME }} \
114+
# --num_collaborators ${{ env.NUM_COLLABORATORS }} --num_rounds ${{ env.NUM_ROUNDS }} --tr_rest_protocol
115+
# echo "Task runner end to end test run completed"
125116

126-
# TODO - Add dockerized approach as well once we have GitHub runners with higher configurations.
117+
# - name: Post test run
118+
# uses: ./.github/actions/tr_post_test_run
119+
# if: ${{ always() }}
120+
# with:
121+
# test_type: "Resiliency_Native_REST"

.github/workflows/task_runner_straggler_e2e.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,17 @@ jobs:
5151
with:
5252
test_type: "With_TLS_Percentage"
5353

54+
# Move it to Rest once issue is resolved https://github.com/securefederatedai/openfl/issues/1646
5455
test_straggler_cutoff:
55-
name: Cutoff Policy (torch/mnist_straggler_check, 3.10)
56+
name: Cutoff Policy (torch/mnist_straggler_check, 3.11)
5657
if: |
5758
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
5859
(github.event_name == 'workflow_dispatch')
5960
runs-on: ubuntu-22.04
6061
timeout-minutes: 30
6162
env:
6263
MODEL_NAME: 'torch/mnist_straggler_check'
63-
PYTHON_VERSION: '3.10'
64+
PYTHON_VERSION: '3.11'
6465
steps:
6566
- name: Checkout OpenFL repository
6667
id: checkout_openfl

.github/workflows/windows.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,24 @@ jobs:
5454
run: |
5555
python -m tests.github.test_hello_federation --template keras/mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --save-model output_model
5656
57+
torch_mnist_rest: # from taskrunner.yml - torch/mnist
58+
if: github.event.pull_request.draft == false
59+
runs-on: windows-latest
60+
timeout-minutes: 15
61+
steps:
62+
- uses: actions/checkout@v4
63+
- name: Set up Python 3
64+
uses: actions/setup-python@v5
65+
with:
66+
python-version: "3.10"
67+
- name: Install dependencies
68+
run: |
69+
python -m pip install --upgrade pip
70+
pip install .
71+
- name: Test TaskRunner API
72+
run: |
73+
python -m tests.github.test_hello_federation --template torch/mnist --fed_workspace aggregator --col1 col1 --col2 col2 --rounds-to-train 3 --transport-protocol rest --save-model output_model
74+
5775
torch_mnist_eden_compression: # from taskrunner_eden_pipeline.yml - torch/mnist_eden_compression
5876
if: github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'eden_compression')
5977
runs-on: windows-latest

tests/end_to_end/conftest.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def pytest_addoption(parser):
3131
parser.addoption("--num_rounds")
3232
parser.addoption("--model_name")
3333
parser.addoption("--workflow_backend")
34-
parser.addoption("--tr_rest_api", action="store_true")
34+
parser.addoption("--tr_rest_protocol", action="store_true")
3535
parser.addoption("--disable_client_auth", action="store_true")
3636
parser.addoption("--disable_tls", action="store_true")
3737
parser.addoption("--log_memory_usage", action="store_true")
@@ -54,7 +54,7 @@ def pytest_configure(config):
5454
config.use_tls = not args.disable_tls
5555
config.log_memory_usage = args.log_memory_usage
5656
config.secure_agg = args.secure_agg
57-
config.tr_rest_api = args.tr_rest_api
57+
config.tr_rest_protocol = args.tr_rest_protocol
5858
config.workflow_backend = args.workflow_backend
5959
config.results_dir = config.getini("results_dir")
6060

tests/end_to_end/models/aggregator.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,22 @@ class Aggregator():
2121
2. Starting the aggregator
2222
"""
2323

24-
def __init__(self, agg_domain_name, workspace_path, eval_scope=False, container_id=None):
24+
def __init__(self, agg_domain_name, workspace_path, transport_protocol, eval_scope=False, container_id=None):
2525
"""
2626
Initialize the Aggregator class
2727
Args:
2828
agg_domain_name (str): Aggregator domain name
2929
workspace_path (str): Workspace path
3030
container_id (str): Container ID
3131
eval_scope (bool, optional): Scope of aggregator is evaluation. Default is False.
32+
transport_protocol (str): Transport protocol (default: "gRPC")
3233
"""
3334
self.name = "aggregator"
3435
self.agg_domain_name = agg_domain_name
3536
self.workspace_path = workspace_path
3637
self.eval_scope = eval_scope
3738
self.container_id = container_id
39+
self.transport_protocol = transport_protocol
3840
self.tensor_db_file = os.path.join(self.workspace_path, "local_state", "tensor.db")
3941
self.res_file = None # Result file to track the logs
4042
self.start_process = None # Process associated with the aggregator start command
@@ -87,7 +89,7 @@ def start(self):
8789
cmd=command,
8890
work_dir=self.workspace_path,
8991
redirect_to_file=bg_file,
90-
check_sleep=60,
92+
check_sleep=30,
9193
env=env
9294
)
9395

0 commit comments

Comments
 (0)