Client: relaunch MasterTech while RemoteExec is armed

Mastertech4.0/src/main.rs

@@ -416,6 +416,8 @@ async fn run_gui(opts: LaunchOptions, minidump: MinidumpArgs) -> eframe::Result<
     // This path can fall back to terminal mode, so it needs the same guarantees.
     mtech_tui::panic_guard::install_hook();
     init_logging(vec![egui_logger, tui_drain_logger()], opts, "gui");
+    #[cfg(target_os = "windows")]
+    utilities::windows::watchdog::log_launch();
 
     // A stale job the registry still calls running belongs to a dead process.
     remote_exec::recover_on_start();
@@ -454,6 +456,8 @@ async fn run_gui(opts: LaunchOptions, minidump: MinidumpArgs) -> eframe::Result<
     };
 
     if gui_ok {
+        #[cfg(target_os = "windows")]
+        utilities::windows::watchdog::remove_on_exit();
         displays::signal_shutdown();
         tokio::time::sleep(std::time::Duration::from_millis(750)).await;
         log::info!("main -> GUI closed; forcing process exit to release the launching terminal");
@@ -470,9 +474,13 @@ async fn run_gui(opts: LaunchOptions, minidump: MinidumpArgs) -> eframe::Result<
         if !console::ensure_console() {
             error!("terminal mode has no console to draw on; nothing further can be shown");
         }
+        #[cfg(target_os = "windows")]
+        utilities::windows::watchdog::set_terminal_mode();
         if let Err(e) = terminal_mode::run_terminal_mode().await {
             error!("Error running terminal app: {e:?}");
         }
+        #[cfg(target_os = "windows")]
+        utilities::windows::watchdog::remove_on_exit();
     }
     Ok(())
 }
@@ -522,11 +530,29 @@ fn cli() -> clap::Command {
                 .help("Disable the glass backdrop-blur pass (same as setting MTECH_NO_FROST=1)")
                 .action(clap::ArgAction::SetTrue),
         )
+        .arg(
+            clap::Arg::new("watchdog")
+                .long("watchdog")
+                .help("Scheduled relaunch: exit at once when MasterTech is already running")
+                .hide(true)
+                .action(clap::ArgAction::SetTrue),
+        )
         .args(MinidumpArgs::clap_args())
 }
 
 #[tokio::main]
 async fn main() -> eframe::Result<()> {
+    #[cfg(target_os = "windows")]
+    {
+        use utilities::windows::watchdog;
+        if !std::env::args().any(|a| a == "--mcp-stdio")
+            && !watchdog::claim_instance()
+            && std::env::args().any(|a| a == watchdog::WATCHDOG_FLAG)
+        {
+            std::process::exit(0);
+        }
+    }
+
     let _ = rustls::crypto::ring::default_provider().install_default();
 
     // Adopt the launching terminal's console, if there is one. Never allocates, so a double-click
@@ -619,8 +645,15 @@ async fn main() -> eframe::Result<()> {
             ..opts
         };
         init_terminal_mode_logging(opts);
+        #[cfg(target_os = "windows")]
+        {
+            utilities::windows::watchdog::set_terminal_mode();
+            utilities::windows::watchdog::log_launch();
+        }
         let res = terminal_mode::run_terminal_mode().await;
         log::info!("TERM MODE: {res:?}");
+        #[cfg(target_os = "windows")]
+        utilities::windows::watchdog::remove_on_exit();
     } else {
         run_gui(opts, minidump).await?;
     }

Mastertech4.0/src/remote_exec/mod.rs

@@ -98,6 +98,11 @@ pub fn arm(
 ) -> GateStatus {
     journal::record_gate("armed", &tech, &diagnostic_session_id, Some(ttl_secs));
     let status = gate::arm(session_id, tech, diagnostic_session_id, reason, ttl_secs);
+    #[cfg(target_os = "windows")]
+    if status.armed {
+        let lease = std::time::Duration::from_secs(status.expires_in_secs.unwrap_or(ttl_secs));
+        crate::utilities::windows::watchdog::arm(lease);
+    }
     wake_ui();
     status
 }
@@ -109,6 +114,8 @@ pub fn disarm(kill_running: bool) -> GateStatus {
         log::warn!("[remote_exec] disarm terminated {killed} running job(s)");
     }
     gate::disarm();
+    #[cfg(target_os = "windows")]
+    crate::utilities::windows::watchdog::disarm();
     wake_ui();
     gate::status(registry::running_count())
 }

Mastertech4.0/src/utilities/windows/mod.rs

@@ -7,6 +7,8 @@ pub mod net_adapter;
 pub mod power;
 #[cfg(target_os = "windows")]
 pub mod reboot;
+#[cfg(target_os = "windows")]
+pub mod watchdog;
 pub mod registry;
 pub mod windows_update;
 pub mod drivers;

Mastertech4.0/src/utilities/windows/reboot.rs

@@ -4,7 +4,7 @@ pub const RELAUNCH_TASK_NAME: &str = "MastertechAutoRestart";
 
 /// Escapes a value for a single-quoted PowerShell string literal.
 #[cfg(target_os = "windows")]
-fn ps_quote(s: &str) -> String {
+pub(crate) fn ps_quote(s: &str) -> String {
     s.replace('\'', "''")
 }
 
@@ -20,10 +20,11 @@ pub async fn schedule_mastertech_relaunch(terminal_mode: bool) -> anyhow::Result
 
     let exe = ps_quote(&exe_path.to_string_lossy());
     let dir = ps_quote(&exe_dir.to_string_lossy());
+    let flag = super::watchdog::WATCHDOG_FLAG;
     let action = if terminal_mode {
-        format!("New-ScheduledTaskAction -Execute '{exe}' -Argument '-t' -WorkingDirectory '{dir}'")
+        format!("New-ScheduledTaskAction -Execute '{exe}' -Argument '-t {flag}' -WorkingDirectory '{dir}'")
     } else {
-        format!("New-ScheduledTaskAction -Execute '{exe}' -WorkingDirectory '{dir}'")
+        format!("New-ScheduledTaskAction -Execute '{exe}' -Argument '{flag}' -WorkingDirectory '{dir}'")
     };
     let script = format!(
         "$a={action};$t=New-ScheduledTaskTrigger -AtLogOn;Register-ScheduledTask -TaskName '{RELAUNCH_TASK_NAME}' -Action $a -Trigger $t -RunLevel Highest -Force | Out-Null"

Mastertech4.0/src/utilities/windows/watchdog.rs

@@ -0,0 +1,231 @@
+//! Scheduled task that relaunches MasterTech while RemoteExec is armed.
+
+use std::os::windows::process::CommandExt;
+use std::sync::Mutex;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::time::{Duration, Instant};
+
+use super::reboot::ps_quote;
+
+pub const WATCHDOG_TASK_NAME: &str = "MastertechWatchdog";
+
+/// Marks a scheduled launch, which exits when MasterTech is already running.
+pub const WATCHDOG_FLAG: &str = "--watchdog";
+
+/// Minutes between relaunch checks.
+const CHECK_MINUTES: u32 = 5;
+
+/// How long the task outlives the RemoteExec lease that armed it.
+const GRACE: Duration = Duration::from_secs(3600);
+
+/// Opened by every GUI and terminal instance for the life of the process.
+const INSTANCE_MUTEX: windows::core::PCWSTR = windows::core::w!("Local\\MasterTech.Instance");
+
+const CREATE_NO_WINDOW: u32 = 0x0800_0000;
+
+static TERMINAL_MODE: AtomicBool = AtomicBool::new(false);
+
+#[derive(Clone, Copy)]
+enum Desired {
+    Until(Instant),
+    Removed,
+}
+
+static DESIRED: Mutex<Desired> = Mutex::new(Desired::Removed);
+
+/// Serialises task updates so the last request wins.
+static APPLY: Mutex<()> = Mutex::new(());
+
+/// Opens the instance mutex and keeps it open; false when another instance already has it.
+pub fn claim_instance() -> bool {
+    use windows::Win32::Foundation::{ERROR_ALREADY_EXISTS, GetLastError};
+    use windows::Win32::System::Threading::CreateMutexW;
+    match unsafe { CreateMutexW(None, false, INSTANCE_MUTEX) } {
+        Ok(_) => {
+            let last = unsafe { GetLastError() };
+            last != ERROR_ALREADY_EXISTS
+        }
+        Err(e) => {
+            log::warn!("instance mutex unavailable: {e}");
+            true
+        }
+    }
+}
+
+/// Records that this process runs terminal mode, so a relaunch passes `-t`.
+pub fn set_terminal_mode() {
+    TERMINAL_MODE.store(true, Ordering::Relaxed);
+}
+
+/// Logs a launch made by the watchdog or the reboot relaunch task.
+pub fn log_launch() {
+    if std::env::args().any(|a| a == WATCHDOG_FLAG) {
+        log::warn!("started by a scheduled relaunch ({WATCHDOG_FLAG})");
+    }
+}
+
+/// Relaunches MasterTech until `lease` plus [`GRACE`] from now.
+pub fn arm(lease: Duration) {
+    if std::env::var_os("MTECH_NO_WATCHDOG").is_some() {
+        log::info!("{WATCHDOG_TASK_NAME}: MTECH_NO_WATCHDOG is set; not registering");
+        return;
+    }
+    request(Desired::Until(Instant::now() + lease + GRACE));
+}
+
+pub fn disarm() {
+    request(Desired::Removed);
+}
+
+/// Deletes the task when the user closes MasterTech; an ending Windows session keeps it.
+pub fn remove_on_exit() {
+    if session_ending() {
+        log::info!("{WATCHDOG_TASK_NAME}: Windows session is ending; keeping the task");
+        return;
+    }
+    *DESIRED.lock().unwrap_or_else(|e| e.into_inner()) = Desired::Removed;
+    apply();
+}
+
+fn request(desired: Desired) {
+    *DESIRED.lock().unwrap_or_else(|e| e.into_inner()) = desired;
+    std::thread::spawn(apply);
+}
+
+fn apply() {
+    let _serial = APPLY.lock().unwrap_or_else(|e| e.into_inner());
+    let desired = *DESIRED.lock().unwrap_or_else(|e| e.into_inner());
+    let result = match desired {
+        Desired::Until(at) => register(at.saturating_duration_since(Instant::now())),
+        Desired::Removed => remove_task(),
+    };
+    if let Err(e) = result {
+        log::warn!("{WATCHDOG_TASK_NAME}: {e}");
+    }
+}
+
+fn register(window: Duration) -> anyhow::Result<()> {
+    let exe = std::env::current_exe()?;
+    let dir = exe
+        .parent()
+        .ok_or_else(|| anyhow::anyhow!("exe path has no parent directory"))?;
+    let script = register_script(
+        &exe.to_string_lossy(),
+        &dir.to_string_lossy(),
+        window.as_secs(),
+        TERMINAL_MODE.load(Ordering::Relaxed),
+    );
+    let out = std::process::Command::new("powershell")
+        .args(["-NoProfile", "-NonInteractive", "-Command", &script])
+        .creation_flags(CREATE_NO_WINDOW)
+        .output()?;
+    if !out.status.success() {
+        anyhow::bail!(
+            "Register-ScheduledTask failed: {}",
+            String::from_utf8_lossy(&out.stderr).trim()
+        );
+    }
+    log::info!(
+        "{WATCHDOG_TASK_NAME}: relaunching {} for the next {} min",
+        exe.display(),
+        window.as_secs() / 60
+    );
+    Ok(())
+}
+
+fn remove_task() -> anyhow::Result<()> {
+    let out = std::process::Command::new("schtasks")
+        .args(["/delete", "/tn", WATCHDOG_TASK_NAME, "/f"])
+        .creation_flags(CREATE_NO_WINDOW)
+        .output()?;
+    if out.status.success() {
+        log::info!("{WATCHDOG_TASK_NAME}: removed");
+    }
+    Ok(())
+}
+
+fn session_ending() -> bool {
+    use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_SHUTTINGDOWN};
+    unsafe { GetSystemMetrics(SM_SHUTTINGDOWN) != 0 }
+}
+
+/// PowerShell that registers the logon and repeating triggers, both expiring `secs` from now.
+fn register_script(exe: &str, dir: &str, secs: u64, terminal: bool) -> String {
+    let exe = ps_quote(exe);
+    let dir = ps_quote(dir);
+    let args = if terminal {
+        format!("-t {WATCHDOG_FLAG}")
+    } else {
+        WATCHDOG_FLAG.to_string()
+    };
+    let secs = secs.max(u64::from(CHECK_MINUTES) * 60);
+    format!(
+        "$end=[DateTimeOffset]::Now.AddSeconds({secs}).ToString('yyyy-MM-ddTHH:mm:sszzz',[Globalization.CultureInfo]::InvariantCulture);\
+         $a=New-ScheduledTaskAction -Execute '{exe}' -Argument '{args}' -WorkingDirectory '{dir}';\
+         $l=New-ScheduledTaskTrigger -AtLogOn;$l.EndBoundary=$end;\
+         $r=New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes({CHECK_MINUTES}) \
+         -RepetitionInterval (New-TimeSpan -Minutes {CHECK_MINUTES}) -RepetitionDuration (New-TimeSpan -Seconds {secs});\
+         $r.EndBoundary=$end;\
+         $s=New-ScheduledTaskSettingsSet -MultipleInstances IgnoreNew -AllowStartIfOnBatteries \
+         -DontStopIfGoingOnBatteries -StartWhenAvailable -ExecutionTimeLimit ([TimeSpan]::Zero) \
+         -DeleteExpiredTaskAfter ([TimeSpan]::Zero);\
+         Register-ScheduledTask -TaskName '{WATCHDOG_TASK_NAME}' -Action $a -Trigger $l,$r -Settings $s \
+         -RunLevel Highest -Force | Out-Null"
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn the_task_relaunches_with_the_watchdog_flag() {
+        let s = register_script(r"C:\Tools\MasterTech.exe", r"C:\Tools", 7200, false);
+        assert!(s.contains(r"-Execute 'C:\Tools\MasterTech.exe' -Argument '--watchdog' -WorkingDirectory 'C:\Tools'"));
+        assert!(s.contains("-TaskName 'MastertechWatchdog'"));
+    }
+
+    #[test]
+    fn terminal_mode_relaunches_in_terminal_mode() {
+        let s = register_script(r"C:\MasterTech.exe", r"C:\", 7200, true);
+        assert!(s.contains("-Argument '-t --watchdog'"));
+    }
+
+    #[test]
+    fn both_triggers_expire_and_the_expired_task_deletes_itself() {
+        let s = register_script(r"C:\MasterTech.exe", r"C:\", 9000, false);
+        assert!(s.contains("AddSeconds(9000)"));
+        assert!(s.contains("$l.EndBoundary=$end"));
+        assert!(s.contains("$r.EndBoundary=$end"));
+        assert!(s.contains("-DeleteExpiredTaskAfter ([TimeSpan]::Zero)"));
+    }
+
+    #[test]
+    fn a_relaunched_instance_is_never_timed_out_or_doubled() {
+        let s = register_script(r"C:\MasterTech.exe", r"C:\", 7200, false);
+        assert!(s.contains("-ExecutionTimeLimit ([TimeSpan]::Zero)"));
+        assert!(s.contains("-MultipleInstances IgnoreNew"));
+        assert!(s.contains("-RunLevel Highest"));
+    }
+
+    #[test]
+    fn quotes_in_paths_are_escaped() {
+        let s = register_script(r"C:\Bob's\MasterTech.exe", r"C:\Bob's", 7200, false);
+        assert!(s.contains(r"-Execute 'C:\Bob''s\MasterTech.exe'"));
+        assert!(s.contains(r"-WorkingDirectory 'C:\Bob''s'"));
+    }
+
+    #[test]
+    fn the_window_covers_at_least_one_check() {
+        let s = register_script(r"C:\MasterTech.exe", r"C:\", 10, false);
+        assert!(s.contains("AddSeconds(300)"));
+    }
+
+    #[test]
+    fn the_command_line_accepts_the_watchdog_flag() {
+        let m = crate::cli()
+            .try_get_matches_from(["MasterTech", WATCHDOG_FLAG])
+            .expect("--watchdog must parse");
+        assert!(m.get_flag("watchdog"));
+    }
+}

displays/src/plugins/mcp_bridge.rs

@@ -4587,7 +4587,9 @@ impl PluginToolProvider {
         description = "Open the consent gate on a client so RemoteExec jobs may run. Fails closed: until the client paints its consent banner \
                        (which names you and your stated reason to whoever is at the machine), every remote_exec_start is refused. \
                        If start keeps reporting 'consent banner not rendering', the client's UI is minimised, wedged, or on a build without the banner. \
-                       Arm once per diagnostic session, not per job, and call remote_exec_disarm when you are done."
+                       Arm once per diagnostic session, not per job, and call remote_exec_disarm when you are done. \
+                       While armed, the client relaunches MasterTech within about five minutes if it dies or Windows restarts, until an hour \
+                       after the lease ends; it comes back disarmed with its jobs gone, so re-arm once it reconnects."
     )]
     async fn remote_exec_arm(
         &self,