feat(scripts): sync pre-commit linter versions with conda in update.py

.pre-commit-config.yaml

@@ -1,15 +1,15 @@
 repos:
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.13.2
+    rev: 8.0.1
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 24.10.0
+    rev: 26.1.0
     hooks:
       - id: black
         language_version: python3
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.19.1
     hooks:
       - id: mypy
         additional_dependencies:

custom_command.py

@@ -1,4 +1,4 @@
-""" This file aims to demonstrate how to write custom commands in OpenWPM
+"""This file aims to demonstrate how to write custom commands in OpenWPM
 
 Steps to have a custom command run as part of a CommandSequence
 

docs/Release-Checklist.md

@@ -4,15 +4,15 @@ We aim to release a new version of OpenWPM with each new Firefox release (~1 rel
 
 1. Run `python scripts/update.py` — this will:
     - Repin the conda environment
+    - Sync linter versions in `.pre-commit-config.yaml` to match `environment.yaml` (black, isort, mypy)
     - Bump all npm dependencies to their latest compatible versions
     - Rebuild the extension
     - Automatically detect and apply any new Firefox release to `scripts/install-firefox.sh`
-2. Check if we can unpin our python version (this requires Linux to test) See <https://github.com/openwpm/OpenWPM/issues/1111>
-3. Run `./scripts/install-firefox.sh` to install the updated Firefox locally and verify the test suite passes.
-4. Increment the version number in [VERSION](../VERSION)
-5. Add a summary of changes since the last version to [CHANGELOG](../CHANGELOG.md)
-6. Squash and merge the release PR to master.
-7. Publish a new release from <https://github.com/openwpm/OpenWPM/releases>:
+2. Run `./scripts/install-firefox.sh` to install the updated Firefox locally and verify the test suite passes.
+3. Increment the version number in [VERSION](../VERSION)
+4. Add a summary of changes since the last version to [CHANGELOG](../CHANGELOG.md)
+5. Squash and merge the release PR to master.
+6. Publish a new release from <https://github.com/openwpm/OpenWPM/releases>:
     1. Click "Draft a new release".
     2. Enter the "Tag version" and "Release title" as `vX.X.X`.
     3. In the description:

openwpm/browser_manager.py

@@ -156,7 +156,7 @@ def check_queue(launch_status: Dict[str, bool]) -> Any:
                 "BROWSER %i: Spawn attempt %i " % (self.browser_id, unsuccessful_spawns)
             )
             # Resets the command/status queues
-            (self.command_queue, self.status_queue) = (Queue(), Queue())
+            self.command_queue, self.status_queue = (Queue(), Queue())
 
             # builds and launches the browser_manager
 

openwpm/deploy_browsers/configure_firefox.py

@@ -1,4 +1,4 @@
-""" Set prefs and load extensions in Firefox """
+"""Set prefs and load extensions in Firefox"""
 
 from selenium.webdriver.firefox.options import Options
 

openwpm/deploy_browsers/deploy_firefox.py

@@ -88,12 +88,10 @@ def deploy_firefox(
             display.start()
             display_pid, display_port = display.pid, display.display
         except EasyProcessError:
-            raise RuntimeError(
-                "Xvfb could not be started. \
+            raise RuntimeError("Xvfb could not be started. \
                 Please ensure it's on your path. \
                 See www.X.org for full details. \
-                Commonly solved on ubuntu with `sudo apt install xvfb`"
-            )
+                Commonly solved on ubuntu with `sudo apt install xvfb`")
     # Must do this for all display modes,
     # because status_queue is read off no matter what.
     status_queue.put(("STATUS", "Display", (display_pid, display_port)))

openwpm/errors.py

@@ -1,4 +1,4 @@
-""" OpenWPM Custom Errors """
+"""OpenWPM Custom Errors"""
 
 
 class CommandExecutionError(Exception):

openwpm/js_instrumentation.py

@@ -32,12 +32,10 @@ def _validate(python_list_to_validate):
             propertiesToInstrument = set(propertiesToInstrument)
             excludedProperties = set(setting["logSettings"]["excludedProperties"])
             if len(propertiesToInstrument.intersection(excludedProperties)) != 0:
-                raise ValueError(
-                    f"excludedProperties and \
+                raise ValueError(f"excludedProperties and \
                     propertiesToInstrument collide. This \
                     may have occurred after a merge. \
-                    Setting with collision: {setting}."
-                )
+                    Setting with collision: {setting}.")
     return True
 
 
@@ -127,11 +125,9 @@ def _build_full_settings_object(setting):
 
     elif isinstance(setting, dict):
         if len(setting.keys()) != 1:
-            raise ValueError(
-                f"Invalid settings request. \
+            raise ValueError(f"Invalid settings request. \
                 Settings item is a dictionary with more \
-                than one key. Received {setting}."
-            )
+                than one key. Received {setting}.")
 
         setting_key = list(setting.keys())[0]
         props = setting[setting_key]
@@ -142,18 +138,14 @@ def _build_full_settings_object(setting):
             for k, v in props.items():
                 logSettings[k] = v
         else:
-            raise ValueError(
-                f"Invalid settings request. \
+            raise ValueError(f"Invalid settings request. \
                 Setting was a dictionary. Settings value \
                 must be a list or a dictionary. Received \
-                {setting}."
-            )
+                {setting}.")
     else:
-        raise ValueError(
-            f"Invalid settings request. \
+        raise ValueError(f"Invalid settings request. \
             Must be a string or a dictionary. \
-            Received {setting}."
-        )
+            Received {setting}.")
 
     return {
         "object": obj,
@@ -228,10 +220,8 @@ def clean_js_instrumentation_settings(
 
     """
     if not isinstance(user_requested_settings, list):
-        raise TypeError(
-            f"js_instrumentation_settings must be a list. \
-              Received {user_requested_settings}"
-        )
+        raise TypeError(f"js_instrumentation_settings must be a list. \
+              Received {user_requested_settings}")
     settings = []
     for setting in user_requested_settings:
         if isinstance(setting, str) and (setting in shortcut_specs):

openwpm/socket_interface.py

@@ -41,7 +41,7 @@ def _accept(self):
         """Listen for connections and pass handling to a new thread"""
         while True:
             try:
-                (client, address) = self.sock.accept()
+                client, address = self.sock.accept()
                 thread = threading.Thread(
                     target=self._handle_conn, args=(client, address)
                 )

openwpm/storage/storage_controller.py

@@ -122,20 +122,16 @@ async def handler(
             record_type, data = record
 
             if record_type == RECORD_TYPE_CREATE:
-                raise RuntimeError(
-                    f"""{RECORD_TYPE_CREATE} is no longer supported.
+                raise RuntimeError(f"""{RECORD_TYPE_CREATE} is no longer supported.
                     Please change the schema before starting the StorageController.
                     For an example of that see test/test_custom_function.py
-                    """
-                )
+                    """)
 
             if record_type == RECORD_TYPE_CONTENT:
                 assert len(data) == 2
                 if self.unstructured_storage is None:
-                    self.logger.error(
-                        """Tried to save content while not having
-                        provided any unstructured storage provider."""
-                    )
+                    self.logger.error("""Tried to save content while not having
+                        provided any unstructured storage provider.""")
                     continue
                 content, content_hash = data
                 content = base64.b64decode(content)

openwpm/utilities/build_cookie_table.py

@@ -153,17 +153,14 @@ def build_http_cookie_table(database, verbose=False):
     cur1 = con.cursor()
     cur2 = con.cursor()
 
-    cur1.execute(
-        "CREATE TABLE IF NOT EXISTS http_request_cookies ( \
+    cur1.execute("CREATE TABLE IF NOT EXISTS http_request_cookies ( \
                     id INTEGER PRIMARY KEY AUTOINCREMENT, \
                     browser_id INTEGER NOT NULL, \
                     header_id INTEGER NOT NULL, \
                     name VARCHAR(200) NOT NULL, \
                     value TEXT NOT NULL, \
-                    accessed DATETIME);"
-    )
-    cur1.execute(
-        "CREATE TABLE IF NOT EXISTS http_response_cookies ( \
+                    accessed DATETIME);")
+    cur1.execute("CREATE TABLE IF NOT EXISTS http_response_cookies ( \
                     id INTEGER PRIMARY KEY AUTOINCREMENT, \
                     browser_id INTEGER NOT NULL, \
                     header_id INTEGER NOT NULL, \
@@ -177,20 +174,17 @@ def build_http_cookie_table(database, verbose=False):
                     secure BOOLEAN, \
                     comment VARCHAR(200), \
                     version VARCHAR(100), \
-                    accessed DATETIME);"
-    )
+                    accessed DATETIME);")
     con.commit()
 
     # Parse http request cookies
     commit = 0
     last_commit = 0
 
-    cur1.execute(
-        """SELECT id, browser_id, headers, time_stamp
+    cur1.execute("""SELECT id, browser_id, headers, time_stamp
                     FROM http_requests
                     WHERE id NOT IN (SELECT header_id FROM
-                    http_request_cookies)"""
-    )
+                    http_request_cookies)""")
 
     row = cur1.fetchone()
     if row is None:
@@ -225,12 +219,10 @@ def build_http_cookie_table(database, verbose=False):
     # Parse http response cookies
     commit = 0
     last_commit = 0
-    cur1.execute(
-        """SELECT id, browser_id, url, headers, time_stamp
+    cur1.execute("""SELECT id, browser_id, url, headers, time_stamp
                     FROM http_responses
                     WHERE id NOT IN (SELECT header_id
-                    FROM http_response_cookies)"""
-    )
+                    FROM http_response_cookies)""")
 
     row = cur1.fetchone()
     if row is None:

openwpm/utilities/cookie.py

@@ -207,7 +207,6 @@
 Finis.
 """
 
-
 # Import our required modules
 #
 
@@ -641,9 +640,7 @@ def js_output(self, attrs=None):
         document.cookie = \"%s\";
         // end hiding -->
         </script>
-        """ % (
-            self.OutputString(attrs).replace('"', r"\""),
-        )
+        """ % (self.OutputString(attrs).replace('"', r"\""),)
 
     # end js_output()
 

scripts/update.py

@@ -3,10 +3,11 @@
 Steps
 -----
 1. Repin the conda environment (scripts/repin.sh)
-2. Update root npm dependencies to latest, resolving peer dep conflicts
-3. Update Extension npm dependencies to latest, resolving peer dep conflicts
-4. Rebuild the extension
-5. Check hg.mozilla.org for a newer Firefox and update install-firefox.sh if found
+2. Sync pre-commit linter revs to the freshly pinned conda env
+3. Update root npm dependencies to latest, resolving peer dep conflicts
+4. Update Extension npm dependencies to latest, resolving peer dep conflicts
+5. Rebuild the extension
+6. Check hg.mozilla.org for a newer Firefox and update install-firefox.sh if found
 
 Run from the project root:
     python scripts/update.py
@@ -23,6 +24,14 @@
 
 _MAX_RESOLVE_ATTEMPTS = 10
 
+# Mapping: conda package name → (pre-commit repo URL, rev prefix)
+# The prefix is prepended to the conda version to form the pre-commit rev tag.
+_LINTER_MAP: dict[str, tuple[str, str]] = {
+    "black": ("https://github.com/psf/black", ""),
+    "isort": ("https://github.com/timothycrosley/isort", ""),
+    "mypy": ("https://github.com/pre-commit/mirrors-mypy", "v"),
+}
+
 
 def run(*cmd: str, cwd: Path = ROOT) -> None:
     print(f"+ {' '.join(cmd)}")
@@ -33,6 +42,97 @@ def conda_run(*cmd: str, cwd: Path = ROOT) -> None:
     run("conda", "run", "-n", "openwpm", *cmd, cwd=cwd)
 
 
+def _versions_from_conda_list_json(json_str: str) -> dict[str, str]:
+    """Parse the output of ``conda list --json`` into ``{name: version}``.
+
+    Separated from the subprocess call so it can be unit-tested without
+    invoking conda.
+    """
+    data = json.loads(json_str)
+    return {pkg["name"]: pkg["version"] for pkg in data}
+
+
+def _query_conda_versions(env_name: str = "openwpm") -> dict[str, str]:
+    """Return ``{package_name: version}`` for all packages in the conda env.
+
+    Uses ``conda list -n <env> --json`` so we get authoritative, fully-resolved
+    versions without re-parsing environment.yaml ourselves.
+    """
+    try:
+        result = subprocess.run(
+            ["conda", "list", "-n", env_name, "--json"],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+    except FileNotFoundError as e:
+        raise RuntimeError(
+            "conda not found on PATH. Install it via ./install.sh or activate "
+            "your conda installation before running this script."
+        ) from e
+
+    if result.returncode != 0:
+        raise RuntimeError(
+            f"`conda list -n {env_name} --json` failed (exit {result.returncode}). "
+            f"Does the env exist? Run ./install.sh to create it.\n"
+            f"stderr: {result.stderr.strip()}"
+        )
+
+    return _versions_from_conda_list_json(result.stdout)
+
+
+def sync_precommit_linter_versions(env_name: str = "openwpm") -> None:
+    """Sync linter revs in ``.pre-commit-config.yaml`` to the conda env.
+
+    Fails loud if a linter from ``_LINTER_MAP`` is missing from the env or
+    if its hook is missing from ``.pre-commit-config.yaml`` — environment.yaml
+    is supposed to be fully pinned, so silent gaps would mask real bugs.
+    """
+    precommit_yaml = ROOT / ".pre-commit-config.yaml"
+
+    print("\n=== Syncing pre-commit linter versions with conda ===")
+
+    conda_versions = _query_conda_versions(env_name)
+    content = precommit_yaml.read_text()
+
+    updated = False
+    for pkg, (repo_url, prefix) in _LINTER_MAP.items():
+        if pkg not in conda_versions:
+            raise RuntimeError(f"{pkg} not found in conda env '{env_name}'.")
+
+        target_rev = f"{prefix}{conda_versions[pkg]}"
+
+        # Match the repo URL line followed by the rev line, preserving whitespace.
+        pattern = re.compile(
+            rf"(- repo: {re.escape(repo_url)}\s*\n\s*rev:\s*)\S+",
+        )
+        match = pattern.search(content)
+        if not match:
+            raise RuntimeError(
+                f"No rev: line found for {repo_url} in .pre-commit-config.yaml"
+            )
+
+        current_rev = content[match.start(0) + len(match.group(1)) : match.end(0)]
+        if current_rev == target_rev:
+            print(f"  {pkg}: already in sync ({current_rev})")
+            continue
+
+        print(f"  {pkg}: {current_rev} -> {target_rev}")
+        content = (
+            content[: match.start(0)]
+            + match.group(1)
+            + target_rev
+            + content[match.end(0) :]
+        )
+        updated = True
+
+    if updated:
+        precommit_yaml.write_text(content)
+        print("Updated .pre-commit-config.yaml")
+    else:
+        print("All linter versions already in sync.")
+
+
 def _npm_install(cwd: Path) -> tuple[str, bool]:
     """Run npm install and return (combined output, success).
 
@@ -150,6 +250,9 @@ def main() -> None:
     # Repin the conda environment from unpinned sources
     run("./repin.sh", cwd=SCRIPTS)
 
+    # Sync pre-commit linter versions to match the freshly pinned conda env
+    sync_precommit_linter_versions()
+
     # Bump npm deps to latest and resolve peer dep conflicts
     npm_bump_and_resolve(ROOT)
     npm_bump_and_resolve(ROOT / "Extension")