Skip package sets that may not exist in older nixpkgs revisions.

When running nix-index on old nixpkgs, scopes like rPackages are missing,
so instead of exiting with an error, this change issues an error to stderr and
skips such package set.

Motivation: while it's possible to use an older version of nix-index to create
the index for older nixpkgs, newer nix-index is much more performant
so it's useful to be able to use it.

Author: vkryachko

src/errors.rs

@@ -6,8 +6,9 @@ use crate::{hydra, nixpkgs, package::StorePath};
 
 #[derive(Error, Debug)]
 pub enum Error {
-    #[error("querying available packages failed: {source}")]
+    #[error("querying available packages in set {} failed: {source}", package_set.as_ref().unwrap_or(&".".to_string()))]
     QueryPackages {
+        package_set: Option<String>,
         #[source]
         source: nixpkgs::Error,
     },
@@ -36,13 +37,13 @@ pub enum Error {
     #[error("writing the paths.cache file failed: {source}")]
     WritePathsCache {
         #[source]
-        source: Box<dyn std::error::Error>,
+        source: Box<dyn std::error::Error + Send>,
     },
     #[error("creating the database at '{path:?}' failed: {source}")]
     CreateDatabase {
         path: PathBuf,
         #[source]
-        source: Box<dyn std::error::Error>,
+        source: Box<dyn std::error::Error + Send>,
     },
     #[error("creating the directory for the database at '{path:?}' failed: {source}")]
     CreateDatabaseDir {

src/listings.rs

@@ -166,10 +166,23 @@ pub fn fetch<'a>(
     let all_paths = all_queries
         .par_iter()
         .flat_map_iter(|&(system, scope)| {
-            nixpkgs::query_packages(nixpkgs, system, scope.as_deref(), show_trace)
+            nixpkgs::query_packages(nixpkgs, system, scope.as_deref(), show_trace).map(|x| {
+                x.map_err(|e| Error::QueryPackages {
+                    package_set: scope.as_deref().map(|s| s.to_string()),
+                    source: e,
+                })
+            })
         })
-        .collect::<std::result::Result<_, nixpkgs::Error>>()
-        .map_err(|e| Error::QueryPackages { source: e })?;
+        .filter_map(|res| match res {
+            Ok(path) => Some(path),
+            Err(e) => {
+                // Older versions of nixpkgs may not have all scopes, so we skip them instead
+                // of completely bailing out.
+                eprintln!("Error getting package set: {e}");
+                None
+            }
+        })
+        .collect::<Vec<_>>();
 
     Ok(fetch_listings_impl(fetcher, jobs, all_paths))
 }

src/nixpkgs.rs

@@ -64,6 +64,7 @@ pub fn query_packages(
         parser: None,
         child: None,
         cmd: Some(cmd),
+        had_error: false,
     }
 }
 
@@ -74,6 +75,7 @@ pub struct PackagesQuery<R: Read> {
     parser: Option<PackagesParser<R>>,
     child: Option<Child>,
     cmd: Option<Command>,
+    had_error: bool,
 }
 
 impl PackagesQuery<ChildStdout> {
@@ -125,6 +127,11 @@ impl Iterator for PackagesQuery<ChildStdout> {
     type Item = Result<StorePath, Error>;
 
     fn next(&mut self) -> Option<Self::Item> {
+        // if we emitted an error in the previous call to next,
+        // there is nothing meaningful we can emit, so signal that we have no more elements to emit.
+        if self.had_error {
+            return None;
+        }
         if let Err(e) = self.ensure_initialized() {
             return Some(Err(e));
         }
@@ -137,7 +144,10 @@ impl Iterator for PackagesQuery<ChildStdout> {
                     //
                     // If the subprocess returned an error, then the parser probably tried to parse garbage output
                     // so we will ignore the parser error and instead return the error printed by the subprocess.
-                    v.map_err(|e| self.check_error().unwrap_or_else(|| Error::from(e)))
+                    v.map_err(|e| {
+                        self.had_error = true;
+                        self.check_error().unwrap_or_else(|| Error::from(e))
+                    })
                 })
                 .or_else(|| {
                     self.parser = None;