trying subprocess instead of multiprocessing, refactoring testsetup

.gitattributes

@@ -1,3 +1,4 @@
 * text=auto
+*.py text eol=lf
 # GitHub syntax highlighting
 pixi.lock linguist-language=YAML

.github/workflows/testing-all-oses.yml

@@ -18,17 +18,29 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["macos-14", "macos-15", "ubuntu-latest"]
+        os: ["windows-latest", "macos-14", "macos-15", "ubuntu-latest"]
     steps:
     - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
     - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6
       with:
         pixi-version: latest
         cache: true
         environments: dev
+    - name: Check CI resource limits
+      shell: bash
+      run: |
+        echo "=== Resource Limits ==="
+        echo "Open files: $(ulimit -n 2>/dev/null || echo 'N/A')"
     - name: Run tests
       timeout-minutes: 40
-      run: pixi run -e dev env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib tests
+      shell: bash
+      run: |
+        mkdir -p /tmp/mpl_cache
+        # Set stricter resource limits for tests on Linux
+        if [ "$(uname)" = "Linux" ]; then
+          ulimit -n 16384 2>/dev/null || true
+        fi
+        pixi run -e dev env QT_QPA_PLATFORM=offscreen MPLCONFIGDIR=/tmp/mpl_cache pytest -v -n logical --durations=20 --cov=mslib tests
     - run: pixi run -e dev coverage xml
     - name: Send coverage to Coveralls (parallel)
       uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7

conftest.py

@@ -24,24 +24,54 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 """
-
+import shutil
 import importlib.util
 import os
 import sys
+import tempfile
+from pathlib import Path
 # Disable pyc files
 sys.dont_write_bytecode = True
 
+_tmpdir_kwargs = {"ignore_cleanup_errors": True} if sys.version_info >= (3, 10) else {}
+_tmp_dir = tempfile.TemporaryDirectory(**_tmpdir_kwargs)
+ROOT_DIR = Path(_tmp_dir.name)
+
+MSWMS_SERVER_CONFIG_FILE = "mswms_settings.py"
+MSWMS_SERVER_CONFIG_DIR = ROOT_DIR / "mswms"
+MSWMS_DATA_DIR = MSWMS_SERVER_CONFIG_DIR / "testdata"
+MSWMS_SERVER_CONFIG_FILE_PATH = MSWMS_SERVER_CONFIG_DIR / MSWMS_SERVER_CONFIG_FILE
+
+if not MSWMS_DATA_DIR.exists():
+    MSWMS_DATA_DIR.mkdir(parents=True)
+
+MSCOLAB_CONFIG_FILE = "mscolab_settings.py"
+MSCOLAB_AUTH_FILE = "mscolab_auth.py"
+MSCOLAB_SERVER_CONFIG_DIR = ROOT_DIR / "mscolab"
+MSCOLAB_DATA_DIR = MSCOLAB_SERVER_CONFIG_DIR / "filedata"
+MSCOLAB_SERVER_CONFIG_FILE_PATH = MSCOLAB_SERVER_CONFIG_DIR / MSCOLAB_CONFIG_FILE
+
+if not MSCOLAB_DATA_DIR.exists():
+    MSCOLAB_DATA_DIR.mkdir(parents=True)
+
+MSUI_CONFIG_PATH = ROOT_DIR / "msui"
+os.environ["MSUI_CONFIG_PATH"] = str(MSUI_CONFIG_PATH.resolve())
+MSUI_CONFIG_FILE_PATH = MSUI_CONFIG_PATH / "msui_settings.json"
+
+if not MSUI_CONFIG_PATH.exists():
+    MSUI_CONFIG_PATH.mkdir(parents=True)
+
+_xdg_cache_home_temporary_directory = tempfile.TemporaryDirectory(**_tmpdir_kwargs)
+os.environ["XDG_CACHE_HOME"] = _xdg_cache_home_temporary_directory.name
+
 import pytest
-import shutil
 import keyring
 from mslib.mswms.seed import DataFiles
-import tests.constants as constants
 from mslib.utils.loggerdef import configure_mpl_logger
 
 matplotlib_logger = configure_mpl_logger()
 
 # This import must come after importing tests.constants due to MSUI_CONFIG_PATH being set there
-from mslib.utils.config import read_config_file
 
 
 class TestKeyring(keyring.backend.KeyringBackend):
@@ -77,22 +107,22 @@ def keyring_reset():
 
 
 def generate_initial_config():
-    """Generate an initial state for the configuration directory in tests.constants.ROOT_FS
+    """Generate an initial state for the configuration directory in ROOT_DIR.
     """
     # make a copy for mscolab test, so that we read different paths during parallel tests.
     sample_path = os.path.join(os.path.dirname(__file__), "tests", "data")
-    shutil.copy(os.path.join(sample_path, "example.ftml"), constants.ROOT_DIR)
+    shutil.copy(os.path.join(sample_path, "example.ftml"), ROOT_DIR)
 
-    if not constants.MSWMS_SERVER_CONFIG_FILE_PATH.exists():
+    if not MSWMS_SERVER_CONFIG_FILE_PATH.exists():
         print('\n configure testdata')
         # ToDo check pytest tmpdir_factory
-        print(constants.MSWMS_DATA_DIR)
-        examples = DataFiles(mswms_data_dir=constants.MSWMS_DATA_DIR,
-                             mswms_server_config_dir=constants.MSWMS_SERVER_CONFIG_DIR)
+        print(MSWMS_DATA_DIR)
+        examples = DataFiles(mswms_data_dir=MSWMS_DATA_DIR,
+                             mswms_server_config_dir=MSWMS_SERVER_CONFIG_DIR)
         examples.create_server_config(detailed_information=True)
         examples.create_data()
 
-    if not constants.MSCOLAB_SERVER_CONFIG_FILE_PATH.exists():
+    if not MSCOLAB_SERVER_CONFIG_FILE_PATH.exists():
         config_string = f'''
 # SQLALCHEMY_DATABASE_URI = 'mysql://user:pass@127.0.0.1/mscolab'
 import os
@@ -101,9 +131,9 @@ def generate_initial_config():
 from pathlib import Path
 from urllib.parse import urljoin
 
-ROOT_DIR = "{constants.ROOT_DIR}"
+ROOT_DIR = "{ROOT_DIR.as_posix()}"
 # directory where mss output files are stored
-DATA_DIR = "{constants.MSCOLAB_DATA_DIR}"
+DATA_DIR = "{MSCOLAB_DATA_DIR.as_posix()}"
 # this will be removed
 OPERATIONS_DATA = Path(DATA_DIR)
 BASE_DIR = ROOT_DIR
@@ -147,7 +177,11 @@ def generate_initial_config():
 # enable verification by Mail
 MAIL_ENABLED = False
 
-SQLALCHEMY_DATABASE_URI = 'sqlite:///' + urljoin(DATA_DIR, 'mscolab.db')
+SQLALCHEMY_DATABASE_URI = 'sqlite:///{MSCOLAB_SERVER_CONFIG_DIR.as_posix()}/mscolab.db'
+
+# Extend SQLite busy-wait timeout (seconds) so concurrent workers don't
+# immediately fail with "database is locked" during Alembic migrations.
+SQLALCHEMY_ENGINE_OPTIONS = {{"connect_args": {{"timeout": 30}}}}
 
 # enable SQLALCHEMY_ECHO
 SQLALCHEMY_ECHO = False
@@ -161,53 +195,41 @@ def generate_initial_config():
 # enable login by identity provider
 USE_SAML2 = False
 '''
-        MSCOLAB_CONFIG = constants.MSCOLAB_SERVER_CONFIG_FILE_PATH
+        MSCOLAB_CONFIG = MSCOLAB_SERVER_CONFIG_FILE_PATH
         MSCOLAB_CONFIG.write_text(config_string)
-        MSCOLAB_AUTH_FILE = constants.MSCOLAB_SERVER_CONFIG_DIR / constants.MSCOLAB_AUTH_FILE
-        if not MSCOLAB_AUTH_FILE.exists():
+        mscolab_auth_file = MSCOLAB_SERVER_CONFIG_DIR / MSCOLAB_AUTH_FILE
+        if not mscolab_auth_file.exists():
             config_string = '''
 import hashlib
 
 class mscolab_auth:
      password = "testvaluepassword"
      allowed_users = [("user", hashlib.md5(password.encode('utf-8')).hexdigest())]
 '''
-            MSCOLAB_AUTH_FILE.write_text(config_string)
+            mscolab_auth_file.write_text(config_string)
 
     def _load_module(module_name, path):
         spec = importlib.util.spec_from_file_location(module_name, path)
         module = importlib.util.module_from_spec(spec)
         sys.modules[module_name] = module
         spec.loader.exec_module(module)
 
-    _load_module("mswms_settings", constants.MSWMS_SERVER_CONFIG_FILE_PATH)
-    _load_module("mscolab_settings", constants.MSCOLAB_SERVER_CONFIG_FILE_PATH)
-
-
-generate_initial_config()
+    _load_module("mswms_settings", MSWMS_SERVER_CONFIG_FILE_PATH)
+    _load_module("mscolab_settings", MSCOLAB_SERVER_CONFIG_FILE_PATH)
 
 
-# This import must come after the call to generate_initial_config, otherwise SQLAlchemy will have a wrong database path
-from tests.utils import create_msui_settings_file
+def pytest_configure(config):
+    """In xdist workers, give each worker its own log file to avoid closed-stream
+    errors when multiple workers share the same pytest.log file handle."""
+    worker_id = os.environ.get("PYTEST_XDIST_WORKER")
+    if worker_id:
+        log_file = getattr(config.option, "log_file", None) or config.getini("log_file")
+        if log_file:
+            base, ext = os.path.splitext(log_file)
+            config.option.log_file = f"{base}_{worker_id}{ext}"
 
 
-@pytest.fixture(autouse=True)
-def reset_config():
-    """Reset the configuration directory used in the tests (tests.constants.ROOT_FS) after every test
-    """
-    # Ideally this would just be shutil.rmtree(constants.MSCOLAB_SERVER_CONFIG_DIR),
-    # but SQLAlchemy complains if the SQLite file is deleted.
-    for item_name in constants.MSCOLAB_SERVER_CONFIG_DIR.iterdir():
-        if item_name.is_dir():
-            shutil.rmtree(item_name)
-        else:
-            if item_name.name != "mscolab.db":
-                item_name.unlink()
-
-    generate_initial_config()
-    create_msui_settings_file("{}")
-    read_config_file()
-
+generate_initial_config()
 
 # Make fixtures available everywhere
 from tests.fixtures import *  # noqa: F401, F403

docs/samples/config/msui/autoplot_dockwidget.json.sample

@@ -1,5 +1,4 @@
 {
-    "mscolab_skip_verify_user_token": true,
     "filepicker_default": "default",
     "data_dir": "~/mssdata",
     "layout": {

mslib/mscolab/events.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+"""
+    mslib.mscolab.events
+    ~~~~~~~~~~~~~~~~~~~~
+
+    This module defines all socket event names used in the mscolab module.
+
+    This file is part of MSS.
+
+    :copyright: Copyright 2019 Shivashis Padhi
+    :copyright: Copyright 2019-2026 by the MSS team, see AUTHORS.
+    :license: APACHE-2.0, see LICENSE for details.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+"""
+
+
+class SocketEvents:
+    """Registry of all Socket.IO event names used in MSColab."""
+
+    # Connection events
+    CONNECT = 'connect'
+    DISCONNECT = 'disconnect'
+    START = 'start'
+
+    # Chat events
+    CHAT_MESSAGE = 'chat-message'
+    CHAT_MESSAGE_CLIENT = 'chat-message-client'
+    CHAT_MESSAGE_REPLY_CLIENT = 'chat-message-reply-client'
+    EDIT_MESSAGE = 'edit-message'
+    EDIT_MESSAGE_CLIENT = 'edit-message-client'
+    DELETE_MESSAGE = 'delete-message'
+    DELETE_MESSAGE_CLIENT = 'delete-message-client'
+
+    # File events
+    FILE_SAVE = 'file-save'
+    FILE_CHANGED = 'file-changed'
+
+    # Permission events
+    ADD_USER_TO_OPERATION = 'add-user-to-operation'
+    # ToDo rename to same word order
+    UPDATE_OPERATION_LIST = 'operation-list-update'
+    OPERATION_SELECTED = 'operation-selected'
+
+    # Active user events
+    ACTIVE_USER_UPDATE = 'active-user-update'
+
+    # Permission management events
+    NEW_PERMISSION = 'new-permission'
+    UPDATE_PERMISSION = 'update-permission'
+    REVOKE_PERMISSION = 'revoke-permission'
+    OPERATION_PERMISSIONS_UPDATED = 'operation-permissions-updated'
+
+    # Operation management events
+    OPERATION_DELETED = 'operation-deleted'
+
+    @classmethod
+    def get_all_events(cls):
+        """Return a set of all event names."""
+        return {
+            getattr(cls, attr)
+            for attr in dir(cls)
+            if not attr.startswith('_') and isinstance(getattr(cls, attr), str)
+        }

mslib/mscolab/file_manager.py

@@ -112,6 +112,7 @@ def create_operation(self, path, description, user, last_used=None, content=None
             r.git.clear_cache()
             r.index.add(['main.ftml'])
             r.index.commit("initial commit")
+            r.close()
             return True
 
     def get_operation_details(self, op_id, user):
@@ -313,12 +314,13 @@ def upload_file(self, file, subfolder=None, identifier=None, include_prefix=Fals
         with file_path.open(mode="wb") as f:
             file.save(f)
 
-        # Relative File path
+        # Relative File path — always use forward slashes so the path can be
+        # used as a URL segment on both Windows and Linux.
         if include_prefix:  # ToDo: add a namespace for the chat attachments, similar as for profile images
             static_dir = Path(upload_folder).name
-            static_file_path = str(Path(static_dir) / str(subfolder) / file_name)
+            static_file_path = '/'.join([static_dir, str(subfolder), file_name])
         else:
-            static_file_path = str(Path(file_path).relative_to(Path(upload_folder)))
+            static_file_path = Path(file_path).relative_to(Path(upload_folder)).as_posix()
 
         logging.debug(f'Relative Path: {static_file_path}')
         return static_file_path
@@ -373,12 +375,7 @@ def update_operation(self, op_id, attribute, value, user):
             if new_path.exists():
                 return False
 
-            new_path.mkdir(parents=True, exist_ok=True)
-
-            try:
-                old_path.rename(new_path)
-            except OSError:
-                shutil.move(str(old_path), str(new_path))
+            shutil.move(str(old_path), str(new_path))
 
             if value.endswith(APP.config['GROUP_POSTFIX']):
                 # getting the category
@@ -468,6 +465,7 @@ def save_file(self, op_id, content, user, version_name=None, comment=""):
                 repo.git.clear_cache()
                 repo.index.add(['main.ftml'])
                 cm = repo.index.commit("committing changes")
+                repo.close()
                 # change db table
                 change = Change(op_id, user.id, cm.hexsha, version_name=version_name)
                 db.session.add(change)
@@ -489,7 +487,7 @@ def get_file(self, op_id, user):
         op_lock = self._get_operation_lock(op_id)
         with op_lock:
             operation_path = Path(self.data_dir) / operation.path / 'main.ftml'
-            with operation_path.open() as data:
+            with operation_path.open(encoding="utf-8") as data:
                 operation_data = data.read()
             return operation_data
 
@@ -545,7 +543,8 @@ def get_change_content(self, ch_id, user):
         operation_path = Path(self.data_dir) / operation.path
         repo = git.Repo(operation_path)
         change_content = repo.git.show(f'{change.commit_hash}:main.ftml')
-        return change_content
+        repo.close()
+        return change_content.replace('\r\n', '\n')
 
     def set_version_name(self, ch_id, op_id, u_id, version_name):
         if (not self.is_admin(u_id, op_id) and not self.is_creator(u_id, op_id) and not
@@ -582,6 +581,7 @@ def undo_changes(self, ch_id, user):
             repo.git.clear_cache()
             try:
                 file_content = repo.git.show(f'{ch.commit_hash}:main.ftml')
+                file_content = file_content.replace('\r\n', '\n')
                 main_ftml_path = operation_path / 'main.ftml'
                 main_ftml_path.write_text(file_content, encoding='utf-8')
 
@@ -594,6 +594,8 @@ def undo_changes(self, ch_id, user):
             except Exception as ex:
                 logging.debug(ex)
                 return False
+            finally:
+                repo.close()
 
     def fetch_users_without_permission(self, op_id, u_id):
         if not self.is_admin(u_id, op_id) and not self.is_creator(u_id, op_id):