Refactor pytest Actions workflow and group integration tests

.github/workflows/pytest.yml

@@ -12,59 +12,128 @@ on:
       - '.github/dependabot.yml'
   workflow_call:
 
+# Cancel workflow runs if a new commit is pushed
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  test:
-    name: pytest with coverage
+  unit-test:
+    name: Run unit tests
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 10
+    strategy:
+      matrix:
+        # Don't test in 3.12 until #1035 is fixed
+        # https://github.com/simonsobs/socs/issues/1035
+        python: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
-    - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.13.1
+    - name: Set up Python ${{ matrix.python }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Clone socs
+      uses: actions/checkout@v6
+
+    - name: Install socs
+      run: |
+        pip3 install -r requirements.txt
+        pip3 install -e .
+
+    - name: Run unit tests
+      working-directory: ./tests
+      run: |
+        COVERAGE_FILE=.coverage.unit-${{ matrix.python }} python3 -m pytest --cov -m 'not integtest'
+
+    - name: Upload coverage file artifact
+      uses: actions/upload-artifact@v7
       with:
-        access_token: ${{ github.token }}
+        name: coverage-${{ matrix.python }}
+        include-hidden-files: true
+        path: ./tests/.coverage.*
 
+  integ-test:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        marker:
+          - 'http'
+          - 'hwp'
+          - 'pysmurf'
+          - 'serial'
+          - 'snmp'
+          - 'tcp'
+
+    steps:
     - name: Install system packages
       run: |
         sudo apt-get install -y socat
 
-    - name: Set up Python 3.8
+    # Run on a single version, since these tests are run in Docker
+    - name: Set up Python 3.11
       uses: actions/setup-python@v6
       with:
-        python-version: 3.8
+        python-version: 3.11
 
-    # Install
-    - name: Install so3g
-      run: |
-        pip3 install so3g
-
-    - name: clone socs
+    - name: Clone socs
       uses: actions/checkout@v6
 
     - name: Install socs
       run: |
         pip3 install -r requirements.txt
         pip3 install -e .
 
-    # Unit Tests
-    - name: Run unit tests
+    - name: Run ${{ matrix.marker }} integration tests
       working-directory: ./tests
       run: |
-        COVERAGE_FILE=.coverage.unit python3 -m pytest --cov -m 'not integtest'
+        COVERAGE_FILE=.coverage.int.${{ matrix.marker }} python3 -m pytest --cov -m 'integtest and ${{ matrix.marker }}'
 
-    # Integration Tests
-    - name: Run integration tests
-      working-directory: ./tests
+    - name: Upload coverage file artifact
+      uses: actions/upload-artifact@v7
+      with:
+        name: coverage-integ-${{ matrix.marker }}
+        include-hidden-files: true
+        path: ./tests/.coverage.*
+
+  upload-coverage:
+    name: Upload coverage
+    needs: [unit-test, integ-test]
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v6
+      with:
+        python-version: 3.11
+
+    - name: Install coverage
       run: |
-        COVERAGE_FILE=.coverage.int python3 -m pytest --cov -m 'integtest'
+        python -m pip install coverage
+
+    # Coverage report doesn't work unless we have a checkout
+    - name: Clone socs
+      uses: actions/checkout@v6
+
+    # Download after checkout, else artifacts get deleted
+    - name: Download coverage artifacts
+      uses: actions/download-artifact@v8
 
-    # Coverage
-    - name: Report test coverage
+    - name: List coverage output for upload
+      run: |
+        ls -lthra
+        ls -lthra coverage*
+        pwd
+
+    - name: Report and upload test coverage
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        mv ./tests/.coverage.* ./
-        pip install -U coveralls
+        mv ./coverage-*/.coverage.* ./
         coverage combine
         coverage report
+        pip install -U coveralls
         coveralls --service=github

.github/workflows/skipped-pytest.yml

@@ -12,8 +12,15 @@ on:
       - '.github/dependabot.yml'
 
 jobs:
-  test:
-    name: pytest with coverage
+  unit-test:
+    name: Run unit tests (3.11)
+    runs-on: ubuntu-latest
+
+    steps:
+      - run: 'echo "No build required" '
+
+  integ-test:
+    name: Run integration tests
     runs-on: ubuntu-latest
 
     steps:

pyproject.toml

@@ -10,13 +10,12 @@ name = "socs"
 dynamic = ["version"]
 description = "Simons Observatory Control System"
 readme = "README.rst"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 classifiers = [
     "Framework :: Twisted",
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: BSD License",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",

tests/integration/test_cryomech_cpa_agent_integration.py

@@ -25,6 +25,7 @@
 emulator = create_device_emulator({init_msg: init_res}, relay_type='tcp', port=5502, encoding=None)
 
 
+@pytest.mark.tcp
 @pytest.mark.integtest
 def test_cryomech_cpa_init(wait_for_crossbar, emulator, run_agent,
                            client):
@@ -36,6 +37,7 @@ def test_cryomech_cpa_init(wait_for_crossbar, emulator, run_agent,
     assert resp.session['op_code'] == OpCode.SUCCEEDED.value
 
 
+@pytest.mark.tcp
 @pytest.mark.integtest
 @pytest.mark.parametrize("state,command", [('on', b'\t\x99\x00\x00\x00\x06\x01\x06\x00\x01\x00\x01'),
                                            ('off', b'\t\x99\x00\x00\x00\x06\x01\x06\x00\x01\x00\xff')])
@@ -54,6 +56,7 @@ def test_cryomech_cpa_power_ptc(wait_for_crossbar, emulator, run_agent,
     assert resp.session['op_code'] == OpCode.SUCCEEDED.value
 
 
+@pytest.mark.tcp
 @pytest.mark.integtest
 def test_cryomech_cpa_auto_start_acq(wait_for_crossbar, emulator, run_agent_acq, client):
     client.init()
@@ -65,6 +68,7 @@ def test_cryomech_cpa_auto_start_acq(wait_for_crossbar, emulator, run_agent_acq,
     assert resp.status == ocs.OK
 
 
+@pytest.mark.tcp
 @pytest.mark.integtest
 def test_cryomech_cpa_acq(wait_for_crossbar, emulator, run_agent, client):
     client.init()
@@ -82,6 +86,7 @@ def test_cryomech_cpa_acq(wait_for_crossbar, emulator, run_agent, client):
     print(resp)
 
 
+@pytest.mark.tcp
 @pytest.mark.integtest
 @pytest.mark.parametrize("state,command", [('on', b'\t\x99\x00\x00\x00\x06\x01\x06\x00\x01\x00\x01')])
 def test_cryomech_cpa_release_reacquire(wait_for_crossbar, emulator, run_agent_acq,

tests/integration/test_hwp_pid_agent_integration.py

@@ -67,12 +67,7 @@ def wait_for_main(client):
         time.sleep(0.2)
 
 
-@pytest.mark.integtest
-def test_testing(wait_for_crossbar):
-    """Just a quick test to make sure we can bring up crossbar."""
-    assert True
-
-
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_get_state(wait_for_crossbar, pid_agent, client):
     resp = client.get_state()
@@ -81,6 +76,7 @@ def test_hwp_rotation_get_state(wait_for_crossbar, pid_agent, client):
     assert len(state.keys()) > 0
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_direction(wait_for_crossbar, pid_agent, client):
     wait_for_main(client)
@@ -97,6 +93,7 @@ def test_hwp_rotation_set_direction(wait_for_crossbar, pid_agent, client):
     assert data['direction'] == 1
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_pid(wait_for_crossbar, pid_agent, client):
     wait_for_main(client)
@@ -107,6 +104,7 @@ def test_hwp_rotation_set_pid(wait_for_crossbar, pid_agent, client):
     assert resp.session['op_code'] == OpCode.SUCCEEDED.value
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_tune_stop(wait_for_crossbar, pid_agent, client):
     wait_for_main(client)
@@ -117,6 +115,7 @@ def test_hwp_rotation_tune_stop(wait_for_crossbar, pid_agent, client):
     assert resp.session['op_code'] == OpCode.SUCCEEDED.value
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_scale(wait_for_crossbar, pid_agent, client):
     wait_for_main(client)
@@ -127,6 +126,7 @@ def test_hwp_rotation_set_scale(wait_for_crossbar, pid_agent, client):
     assert resp.session['op_code'] == OpCode.SUCCEEDED.value
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_declare_freq(wait_for_crossbar, pid_agent, client):
     wait_for_main(client)
@@ -137,6 +137,7 @@ def test_hwp_rotation_declare_freq(wait_for_crossbar, pid_agent, client):
     assert resp.session['op_code'] == OpCode.SUCCEEDED.value
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_tune_freq(wait_for_crossbar, pid_agent, client):
     wait_for_main(client)

tests/integration/test_hwp_pmx_agent_integration.py

@@ -23,12 +23,7 @@
 kikusui_emu = create_device_emulator(responses, relay_type='tcp', port=5025)
 
 
-@pytest.mark.integtest
-def test_testing(wait_for_crossbar):
-    """Just a quick test to make sure we can bring up crossbar."""
-    assert True
-
-
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_main(wait_for_crossbar, kikusui_emu, run_agent, client):
     client.main.stop()
@@ -38,6 +33,7 @@ def test_hwp_rotation_main(wait_for_crossbar, kikusui_emu, run_agent, client):
     assert resp.session['success']
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_off(wait_for_crossbar, kikusui_emu, run_agent, client):
     kikusui_emu.update_responses({
@@ -50,6 +46,7 @@ def test_hwp_rotation_set_off(wait_for_crossbar, kikusui_emu, run_agent, client)
     assert resp.session['success']
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_on(wait_for_crossbar, kikusui_emu, run_agent, client):
     kikusui_emu.update_responses({
@@ -62,6 +59,7 @@ def test_hwp_rotation_set_on(wait_for_crossbar, kikusui_emu, run_agent, client):
     assert resp.session['success']
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_i(wait_for_crossbar, kikusui_emu, run_agent, client):
     kikusui_emu.update_responses(
@@ -74,6 +72,7 @@ def test_hwp_rotation_set_i(wait_for_crossbar, kikusui_emu, run_agent, client):
     assert resp.session['success']
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_i_lim(wait_for_crossbar, kikusui_emu, run_agent, client):
     kikusui_emu.update_responses(
@@ -85,6 +84,7 @@ def test_hwp_rotation_set_i_lim(wait_for_crossbar, kikusui_emu, run_agent, clien
     assert resp.session['success']
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_v(wait_for_crossbar, kikusui_emu, run_agent, client):
     kikusui_emu.update_responses(
@@ -96,6 +96,7 @@ def test_hwp_rotation_set_v(wait_for_crossbar, kikusui_emu, run_agent, client):
     assert resp.session['success']
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_set_v_lim(wait_for_crossbar, kikusui_emu, run_agent, client):
     kikusui_emu.update_responses({
@@ -109,6 +110,7 @@ def test_hwp_rotation_set_v_lim(wait_for_crossbar, kikusui_emu, run_agent, clien
     assert resp.session['success']
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_use_ext(wait_for_crossbar, kikusui_emu, run_agent, client):
     kikusui_emu.update_responses({
@@ -122,6 +124,7 @@ def test_hwp_rotation_use_ext(wait_for_crossbar, kikusui_emu, run_agent, client)
     assert resp.session['success']
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_rotation_ign_ext(wait_for_crossbar, kikusui_emu, run_agent, client):
     kikusui_emu.update_responses({

tests/integration/test_hwp_supervisor_agent_integration.py

@@ -166,6 +166,7 @@ def get_hwp_state(client) -> Dict[str, Any]:
     return client.query_hwp_state().session["data"]["state"]
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_supervisor_grip(hwp_em, supervisor_agent, sup_client) -> None:
     state = get_hwp_state(sup_client)
@@ -183,6 +184,7 @@ def test_supervisor_grip(hwp_em, supervisor_agent, sup_client) -> None:
         assert False
 
 
+@pytest.mark.hwp
 @pytest.mark.integtest
 def test_hwp_spinup(supervisor_agent, sup_client) -> None:
     assert not get_hwp_state(sup_client)["is_spinning"]