Improve auto-cache robustness

- Detect corrupt caches and automatically recreate them.
- Enables fully offline work with auto-cache.: Skip remote fetches when
the revision is already cached.
- Support temporary offline remotes: Skip remote fetches when the remote
is unreachable.

Author: thorsten-klein

src/west/app/project.py

@@ -1177,9 +1177,9 @@ def do_add_parser(self, parser_adder):
             '--auto-cache',
             help='''automatically setup local cache repositories
                     in a flat folder hierarchy, but with an additional
-                    subfolder (hashed name) for different remote URLs.
-                    Each local cache repository is automatically cloned
-                    on first usage and synced on subsequent clones.
+                    subfolder (hashed name) for different remote URLs. Each
+                    local cache repository is automatically cloned on first
+                    usage and synced on subsequent fetches (on demand).
                     This cache has the lowest priority (Prio 2).''',
         )
 
@@ -1203,7 +1203,8 @@ def do_add_parser(self, parser_adder):
         workspace setup.
         Only in case of auto-cache the 'west update' process updates the local
         caches first, which then serve as the source for pulling changes into
-        the workspace.
+        the workspace. Thereby, the auto-cache only fetches updates from remote
+        if the specified revision is not already present in the local cache.
 
         Example: Assume your manifest describes this workspace structure:
           (workspace)
@@ -1233,7 +1234,8 @@ def do_add_parser(self, parser_adder):
           folder hierarchy is setup automatically. Each repository is stored under a
           directory named after the basename of its remote URL. To prevent conflicts
           between repos with same name, a hash of the remote URL is used as subfolder.
-          Note: Each local cache repo is automatically synced on subsequent updates.
+          Note: Each local cache repo is automatically synced on subsequent updates
+          on demand (if the used revision is not cached yet).
             (auto cache directory)
             ├── bar.git
             │   ├── <hash>
@@ -1757,13 +1759,60 @@ def handle_auto_cache(self, project):
             # Then clone the repository into the local cache.
             cache_dir_parent = Path(cache_dir).parent
             cache_dir_parent.mkdir(parents=True, exist_ok=True)
+            self.dbg(f'{project.name}: Setup auto-cache for {project.url} in {cache_dir}')
             project.git(
                 ['clone', '--mirror', '--', project.url, os.fspath(cache_dir)], cwd=cache_dir_parent
             )
             self.create_auto_cache_info(project, cache_dir)
         else:
-            # The local cache already exists. Sync it with remote.
-            project.git(['remote', 'update', '--prune'], cwd=cache_dir)
+            # The local cache already exists
+
+            # helper function to check if git commands run successful
+            def check_git(cmd) -> bool:
+                '''
+                run git command in the auto-cache directory without suppressed output
+                return True if the command was successful (returncode 0), otherwise False
+                '''
+                p = project.git(
+                    cmd, cwd=cache_dir, capture_stdout=True, capture_stderr=True, check=False
+                )
+                return not p.returncode
+
+            # Check if the auto-cache is corrupt
+            corrupt = not check_git(['cat-file', '-e', 'HEAD'])
+
+            # Check if the revision is already contained in the auto-cache.
+            contains_rev = check_git(['cat-file', '-e', f'{project.revision}^{{commit}}'])
+
+            # Check if the remote is reachable
+            reachable = check_git(['ls-remote'])
+
+            if not corrupt:
+                # early exit if auto-cache is intact, but remote is not reachable (even if
+                # fetch stratefy 'always' is specified), since we know that remote update will fail.
+                if not reachable:
+                    self.dbg(
+                        f'{project.name}: remote ({project.url}) not reachable. '
+                        'Skip auto-cache update with remote.'
+                    )
+                    return
+
+                # early exit if auto-cache is intact and revision is already contained
+                if contains_rev and self.fs != 'always':
+                    self.dbg(
+                        f'{project.name}: auto-cache remote update is skipped '
+                        f'as it already contains revision {project.revision}'
+                    )
+                    return
+
+                # The auto-cache needs to be updated.
+                self.dbg(f'{project.name}: update auto-cache ({cache_dir}) with remote')
+                project.git(['remote', 'update', '--prune'], cwd=cache_dir, check=False)
+            else:
+                # Remove the auto-cache and freshly setup again.
+                self.small_banner(f'{project.name}: remove corrupt auto-cache {cache_dir}')
+                shutil.rmtree(cache_dir)
+                self.handle_auto_cache(project)
 
     def init_project(self, project):
         # update() helper. Initialize an uncloned project repository.