diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 475d091..f5c6fbb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -4,7 +4,7 @@ name: Integration Tests (Standalone) # # This workflow runs integration tests independently of the release process. # Use it for: -# - Manual testing before creating a release branch +# - Manual testing before triggering a release via workflow_dispatch # - Weekly regression testing # - Debugging integration issues # diff --git a/CHANGELOG.md b/CHANGELOG.md index 072b9ca..a78d33e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1341,11 +1341,20 @@ GRPM (Go Resource Package Manager) is a modern reimplementation of Gentoo's Port ## Links - **Repository**: https://github.com/grpmsoft/grpm -- **Documentation**: https://github.com/grpmsoft/grpm/tree/master/docs +- **Documentation**: https://github.com/grpmsoft/grpm/tree/main/docs - **Issues**: https://github.com/grpmsoft/grpm/issues - **License**: [Apache-2.0](LICENSE) -[Unreleased]: https://github.com/grpmsoft/grpm/compare/v0.7.11...HEAD +[Unreleased]: https://github.com/grpmsoft/grpm/compare/v0.9.3...HEAD +[0.9.3]: https://github.com/grpmsoft/grpm/compare/v0.9.2...v0.9.3 +[0.9.2]: https://github.com/grpmsoft/grpm/compare/v0.9.1...v0.9.2 +[0.9.1]: https://github.com/grpmsoft/grpm/compare/v0.9.0...v0.9.1 +[0.9.0]: https://github.com/grpmsoft/grpm/compare/v0.8.4...v0.9.0 +[0.8.4]: https://github.com/grpmsoft/grpm/compare/v0.8.3...v0.8.4 +[0.8.3]: https://github.com/grpmsoft/grpm/compare/v0.8.2...v0.8.3 +[0.8.2]: https://github.com/grpmsoft/grpm/compare/v0.8.1...v0.8.2 +[0.8.1]: https://github.com/grpmsoft/grpm/compare/v0.8.0...v0.8.1 +[0.8.0]: https://github.com/grpmsoft/grpm/compare/v0.7.11...v0.8.0 [0.7.11]: https://github.com/grpmsoft/grpm/compare/v0.7.10...v0.7.11 [0.7.10]: https://github.com/grpmsoft/grpm/compare/v0.7.9...v0.7.10 [0.7.9]: https://github.com/grpmsoft/grpm/compare/v0.7.8...v0.7.9 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7d97847..2d0c971 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -120,9 +120,9 @@ Look for issues labeled `good first issue` or `help wanted`. ### Areas Needing Help -- EAPI 8 support -- CMake/Meson build systems -- Eclass implementations +- Complex eclass support (kernel, LLVM, Java) +- CMake/Meson build system edge cases +- Real-world package testing and bug reports - Test coverage improvements - Documentation diff --git a/README.md b/README.md index 6a9ec93..5ff2475 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,9 @@ GRPM (Go Resource Package Manager) is a modern source-based package manager writ | **SAT-based Dependency Resolution** | Boolean satisfiability solver for guaranteed conflict-free resolution | | **Binary Package Support** | Full GPKG (.gpkg.tar) and legacy TBZ2 (.tbz2) format support | | **Transactional Updates** | Btrfs/ZFS snapshot-based rollbacks for safe system updates | -| **Source Building** | Complete ebuild execution with autotools, CMake, and Meson | -| **Build Systems** | cmake.eclass, meson.eclass, toolchain-funcs, flag-o-matic | -| **Language Ecosystems** | Python (distutils-r1), Rust (cargo.eclass), Go (go-module.eclass) | +| **Source Building** | Ebuild execution with autotools; CMake and Meson support (basic) | +| **Build Systems** | toolchain-funcs, flag-o-matic, cmake.eclass, meson.eclass | +| **Language Ecosystems** | Python (distutils-r1), Rust (cargo.eclass), Go (go-module.eclass) — basic support | | **Multilib Support** | 32-bit/64-bit library support with ABI management | | **Package Sets** | @world, @system, @selected in ALL commands (resolve, install, emerge, fetch) | | **Distfile Fetching** | Automatic source downloading with mirror failover | @@ -57,9 +57,10 @@ GRPM (Go Resource Package Manager) is a modern source-based package manager writ ### Install from Binary ```bash -# Download latest release -wget https://github.com/grpmsoft/grpm/releases/latest/download/grpm_linux_amd64.tar.gz -tar -xzf grpm_linux_amd64.tar.gz +# Download latest release (check https://github.com/grpmsoft/grpm/releases for VERSION) +VERSION="0.9.3" +wget "https://github.com/grpmsoft/grpm/releases/download/v${VERSION}/grpm_${VERSION}_linux_x86_64.tar.gz" +tar -xzf "grpm_${VERSION}_linux_x86_64.tar.gz" sudo install -m 0755 grpm /usr/bin/grpm # Verify @@ -235,8 +236,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for development guidelines and [AGENTS.md **Completed Features:** - ✅ SAT-based dependency resolution -- ✅ Full ebuild execution (autotools, CMake, Meson) -- ✅ Language ecosystems (Python, Rust, Go) +- ✅ Ebuild execution (autotools full; CMake, Meson basic) +- ✅ Language ecosystem support (Python, Rust, Go — basic) - ✅ Multilib support (32-bit/64-bit) - ✅ Binary package support (GPKG, TBZ2) - ✅ Repository sync (rsync, git) with GPG verification diff --git a/ROADMAP.md b/ROADMAP.md index 359abe1..be964c9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -42,7 +42,7 @@ Key insight: **Eclasses don't need Go implementations.** They are loaded dynamic ## Current Status -### What's Implemented (~75% Tree Coverage) +### What's Implemented (~98% Tree Coverage) | Category | Features | |----------|----------| @@ -301,7 +301,7 @@ grpm emerge --check-tools @world # Optional pre-validation ## Roadmap to v1.0.0 ``` -v0.9.0 ← CURRENT (Pre-Release Testing) +v0.9.3 ← CURRENT (Pre-Release Testing) │ ✅ v0.6.0: Distfile fetching, debug helpers, coverage analyzer │ ✅ v0.7.x: Portage compatibility, security fixes │ ✅ v0.8.0: Configuration management (make.conf, repos.conf, package.use) @@ -456,4 +456,4 @@ Key deliverables: --- *This roadmap evolves based on community feedback and project needs.* -*Last updated: 2026-01-19 (v0.9.0 enterprise tool check)* +*Last updated: 2026-02-08 (v0.9.3 documentation audit)* diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ab4bd5e..19c4c1e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -11,7 +11,7 @@ flowchart TB subgraph Daemon["Daemon Layer"] GRPC_SERVER[gRPC Server
unix:///var/run/grpm.sock] - REST_API[REST API
:8080] + REST_API[REST API
unix:///var/run/grpm-rest.sock] JOB_QUEUE[Job Queue] WORKERS[Worker Pool] end @@ -88,7 +88,7 @@ flowchart TB ### Daemon Layer - **gRPC Server**: Unix socket server for CLI communication -- **REST API**: HTTP API for external integrations +- **REST API**: Unix socket API for external integrations (TCP optional) - **Job Queue**: Prioritized queue with conflict detection - **Worker Pool**: Parallel job execution diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index bc2c6cf..6f82a77 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -17,6 +17,7 @@ Complete command-line reference for GRPM. See [CHANGELOG](../CHANGELOG.md) for v - [fetch](#fetch) - [build](#build) - [update](#update) + - [depclean](#depclean) - [analyze](#analyze) - [tools](#tools) - [completion](#completion) @@ -220,11 +221,16 @@ grpm emerge [options] ... | `--mock` | Use mock repository for testing | `false` | | `--pretend`, `-p` | Show build plan without building | `false` | | `--ask`, `-a` | Ask for confirmation before building | `false` | -| `--jobs ` | Number of parallel make jobs | From MAKEOPTS or 4 | +| `--jobs `, `-j ` | Number of packages to build in parallel | `1` | +| `--make-jobs ` | Number of parallel make jobs per package | From MAKEOPTS or 4 | +| `--keep-going`, `-k` | Continue building remaining packages on failure | `false` | | `--keep-work` | Keep work directory after build | `false` | | `--test` | Run test phase (make check/test) | `false` | +| `--replace`, `-R` | Replace existing package (unmerge old before merge) | `false` | +| `--force`, `-f` | Force installation (skip collision checks) | `false` | | `--onlydeps`, `-o` | Build dependencies only, skip target | `false` | | `--check-tools` | Perform optional pre-build tool availability check | `false` | +| `--info` | Show system environment information (like `emerge --info`) | `false` | | `--deep`, `-D` | Traverse dependencies of already-installed packages | `false` | | `--with-bdeps` | Include build-time dependencies for installed packages | `false` | | `--emptytree`, `-e` | Assume no packages installed (full dependency tree) | `false` | @@ -567,9 +573,10 @@ grpm update [options] | `--repo ` | Path to Portage repository | `/var/db/repos/gentoo` | | `--mock` | Use mock repository | `false` | | `--pretend`, `-p` | Show what would be updated | `false` | +| `--ask`, `-a` | Ask for confirmation before updating | `false` | | `--deep`, `-D` | Include dependencies | `false` | - -**Note:** Full update functionality is planned for future releases. +| `--newuse`, `-N` | Recalculate USE flags for installed packages | `false` | +| `--changed-use`, `-U` | Only update packages with changed USE flags | `false` | **Examples:** @@ -579,10 +586,53 @@ grpm update --pretend # Include dependencies grpm update --deep --pretend + +# Update with USE flag recalculation +grpm update --deep --newuse --pretend + +# Update @world with confirmation +sudo grpm update --ask --deep --newuse ``` --- +### depclean + +Remove unused/orphaned packages from the system. + +``` +grpm depclean [options] [package...] +``` + +**Options:** + +| Option | Description | Default | +|--------|-------------|---------| +| `--pretend`, `-p` | Show what would be removed | `false` | +| `--ask`, `-a` | Ask for confirmation before removing | `false` | +| `--exclude ` | Exclude packages from removal (repeatable) | | + +**Examples:** + +```bash +# Show orphaned packages (dry-run) +grpm depclean --pretend + +# Remove unused packages with confirmation +sudo grpm depclean --ask + +# Exclude specific packages from removal +sudo grpm depclean --exclude sys-libs/glibc --exclude sys-devel/gcc +``` + +**Notes:** +- Depclean removes packages not in @world/@system that no other package depends on +- Always use `--pretend` first to review what would be removed +- Use `--exclude` to protect packages from removal +- Also accessible via `grpm remove --depclean` / `grpm remove -c` + +--- + ### analyze Analyze repository coverage and compatibility. @@ -913,7 +963,7 @@ grpm daemon The daemon provides: - gRPC server on Unix socket (`/var/run/grpm.sock`) -- REST API on HTTP (`127.0.0.1:8080`) +- REST API on Unix socket (`/var/run/grpm-rest.sock`) - Job queue with conflict detection - Background monitoring diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 61de5ef..472d56c 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -46,7 +46,7 @@ This guide covers installation of GRPM on Gentoo Linux and compatible distributi ```bash # Set version and architecture VERSION="0.9.3" -ARCH="amd64" # Options: amd64, arm64, arm_7, arm_6, 386 +ARCH="x86_64" # Options: x86_64, arm64, armv7, armv6, i386 # Download binary wget "https://github.com/grpmsoft/grpm/releases/download/v${VERSION}/grpm_${VERSION}_linux_${ARCH}.tar.gz" @@ -71,11 +71,11 @@ grpm -V | Architecture | Filename | Description | |--------------|----------|-------------| -| x86_64 | `grpm_*_linux_amd64.tar.gz` | 64-bit Intel/AMD (most common) | +| x86_64 | `grpm_*_linux_x86_64.tar.gz` | 64-bit Intel/AMD (most common) | | ARM64 | `grpm_*_linux_arm64.tar.gz` | 64-bit ARM (Apple Silicon, AWS Graviton) | -| ARMv7 | `grpm_*_linux_arm_7.tar.gz` | 32-bit ARM with hardware float | -| ARMv6 | `grpm_*_linux_arm_6.tar.gz` | Raspberry Pi Zero/1 | -| i386 | `grpm_*_linux_386.tar.gz` | 32-bit Intel/AMD | +| ARMv7 | `grpm_*_linux_armv7.tar.gz` | 32-bit ARM with hardware float | +| ARMv6 | `grpm_*_linux_armv6.tar.gz` | Raspberry Pi Zero/1 | +| i386 | `grpm_*_linux_i386.tar.gz` | 32-bit Intel/AMD | --- diff --git a/docs/PMS_COMPLIANCE.md b/docs/PMS_COMPLIANCE.md index 7fcdd30..13787be 100644 --- a/docs/PMS_COMPLIANCE.md +++ b/docs/PMS_COMPLIANCE.md @@ -310,7 +310,7 @@ internal/solver/gophersat_adapter.go # SAT encoding | pkg_postrm | Full | | | pkg_config | Partial | | | pkg_info | Partial | | -| pkg_nofetch | Not Yet | | +| pkg_nofetch | Full | Default implementation per PMS 9.1.16 | ### Default Phase Implementations diff --git a/go.mod b/go.mod index 8c196d4..20ced11 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( require ( github.com/coregx/ahocorasick v0.1.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect diff --git a/go.sum b/go.sum index aade80d..d2c8329 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= diff --git a/internal/cli/analyze.go b/internal/cli/analyze.go index 045a039..d4d8a82 100644 --- a/internal/cli/analyze.go +++ b/internal/cli/analyze.go @@ -49,7 +49,7 @@ func (a *App) runAnalyze(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("analyze")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } diff --git a/internal/cli/app.go b/internal/cli/app.go index a4ad831..ef10315 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -239,7 +239,7 @@ func (a *App) runResolve(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("resolve")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -415,7 +415,7 @@ func (a *App) runInstall(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("install")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -852,7 +852,7 @@ func (a *App) runSync(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("sync")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -1002,7 +1002,7 @@ func (a *App) runCompletion(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("completion")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -1121,7 +1121,7 @@ View generated man page: grpm doc man emerge | man -l -`) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -1188,3 +1188,19 @@ func (a *App) generateAllManPages(gen *ManPageGenerator, dir string) error { a.log.Success("Generated %d man page(s) in %s", count, dir) return nil } + +// reorderArgs moves flag-like arguments (starting with "-") before positional +// arguments so that Go's flag package can parse them correctly. +// Go's flag.Parse stops at the first non-flag argument, but Portage allows +// flags anywhere: "emerge @world --deep" should work the same as "emerge --deep @world". +func reorderArgs(args []string) []string { + var flags, positional []string + for _, arg := range args { + if len(arg) > 0 && arg[0] == '-' { + flags = append(flags, arg) + } else { + positional = append(positional, arg) + } + } + return append(flags, positional...) +} diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 0512c7c..895bd93 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -33,7 +33,7 @@ func (a *App) runSearch(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("search")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -150,7 +150,7 @@ func (a *App) runInfo(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("info")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -299,7 +299,7 @@ func (a *App) runUpdate(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("update")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -663,7 +663,7 @@ func (a *App) runRemove(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("remove")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -730,7 +730,7 @@ func (a *App) runBuild(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("build")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } @@ -814,7 +814,7 @@ func (a *App) runDepclean(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("depclean")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } diff --git a/internal/cli/emerge.go b/internal/cli/emerge.go index a2f051c..b903155 100644 --- a/internal/cli/emerge.go +++ b/internal/cli/emerge.go @@ -76,7 +76,7 @@ func (a *App) runEmerge(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("emerge")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil // Help was requested, not an error } @@ -103,6 +103,11 @@ func (a *App) runEmerge(args []string) error { return nil } + // When using --deep, installed packages will be in the solution — implicitly enable replace + if *deep && !*replace { + *replace = true + } + // Validate parallel builds count if *parallelBuilds < 1 { *parallelBuilds = 1 @@ -202,7 +207,7 @@ func (a *App) runEmerge(args []string) error { } // Sequential build (original behavior) - return a.buildAndInstallPackages(solution, *repoPath, *distDir, *tmpDir, *makeJobs, *keepWork, *enableTests, *replace, *force, *rootPath, fetcher) + return a.buildAndInstallPackages(solution, *repoPath, *distDir, *tmpDir, *makeJobs, *keepWork, *enableTests, *replace, *force, *rootPath, *keepGoing, fetcher) } // parallelBuildOptions holds options for parallel build execution. @@ -457,11 +462,13 @@ func (a *App) createFetcher(distDir string) fetch.Fetcher { } // buildAndInstallPackages builds packages from source and installs them. -func (a *App) buildAndInstallPackages(solution map[string]*pkg.Package, repoPath, distDir, tmpDir string, jobs int, keepWork, enableTests, replace, force bool, root string, fetcher fetch.Fetcher) error { +func (a *App) buildAndInstallPackages(solution map[string]*pkg.Package, repoPath, distDir, tmpDir string, jobs int, keepWork, enableTests, replace, force bool, root string, keepGoing bool, fetcher fetch.Fetcher) error { logging.Action("Starting source build...") builtCount := 0 + failedCount := 0 totalPackages := len(solution) + var failedPkgs []string // Get package database (with root prefix) db, err := a.getOrCreatePackageDBWithRoot(root) @@ -473,28 +480,60 @@ func (a *App) buildAndInstallPackages(solution map[string]*pkg.Package, repoPath installer := install.NewInstaller(root, db) installer.Verbose = a.verbose + pkgNum := 0 for name, p := range solution { - logging.Action("(%d/%d) Emerging %s-%s", builtCount+1, totalPackages, name, p.Version) - - // Build from source (fetcher will download sources automatically) - imageDir, err := a.buildPackageFromSource(p, repoPath, distDir, tmpDir, jobs, keepWork, enableTests, fetcher) - if err != nil { - return fmt.Errorf("failed to build %s: %w", name, err) - } - - // Install to system - if err := a.installFromImageDir(installer, p, imageDir, keepWork, replace, force); err != nil { - return fmt.Errorf("failed to install %s: %w", name, err) + pkgNum++ + logging.Action("(%d/%d) Emerging %s-%s", pkgNum, totalPackages, name, p.Version) + + buildErr := a.buildAndInstallSingle(name, p, installer, repoPath, distDir, tmpDir, jobs, keepWork, enableTests, replace, force, fetcher) + if buildErr != nil { + if keepGoing { + logging.Error("failed to emerge %s: %v", name, buildErr) + failedCount++ + failedPkgs = append(failedPkgs, name) + continue + } + return fmt.Errorf("failed to emerge %s: %w", name, buildErr) } builtCount++ logging.Action("%s-%s merged successfully (%d/%d)", name, p.Version, builtCount, totalPackages) } + if failedCount > 0 { + logging.Error("Emerge completed with %d failure(s) out of %d package(s):", failedCount, totalPackages) + for _, name := range failedPkgs { + logging.Error(" - %s", name) + } + return fmt.Errorf("%d package(s) failed to build", failedCount) + } + logging.Action("Emerge completed successfully: %d package(s) built and installed", builtCount) return nil } +// buildAndInstallSingle builds and installs a single package with panic recovery. +// This prevents interpreter panics (e.g., unsupported bash features) from +// crashing the entire emerge process when --keep-going is used. +func (a *App) buildAndInstallSingle(name string, p *pkg.Package, installer *install.Installer, repoPath, distDir, tmpDir string, jobs int, keepWork, enableTests, replace, force bool, fetcher fetch.Fetcher) (buildErr error) { + defer func() { + if r := recover(); r != nil { + buildErr = fmt.Errorf("internal error (panic): %v", r) + } + }() + + imageDir, err := a.buildPackageFromSource(p, repoPath, distDir, tmpDir, jobs, keepWork, enableTests, fetcher) + if err != nil { + return fmt.Errorf("build failed: %w", err) + } + + if err := a.installFromImageDir(installer, p, imageDir, keepWork, replace, force); err != nil { + return fmt.Errorf("install failed: %w", err) + } + + return nil +} + // buildPackageFromSource builds a package from source using ebuild executor. // // Returns the image directory (D) where files are installed. diff --git a/internal/cli/errors.go b/internal/cli/errors.go index b13602d..3119087 100644 --- a/internal/cli/errors.go +++ b/internal/cli/errors.go @@ -51,6 +51,12 @@ func (e *UserError) Error() string { } } + if e.Technical != nil { + sb.WriteString("\n Technical: ") + sb.WriteString(e.Technical.Error()) + sb.WriteString("\n") + } + return sb.String() } diff --git a/internal/cli/fetch.go b/internal/cli/fetch.go index 8dba675..73161b2 100644 --- a/internal/cli/fetch.go +++ b/internal/cli/fetch.go @@ -52,7 +52,7 @@ func (a *App) runFetch(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("fetch")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } diff --git a/internal/cli/tools.go b/internal/cli/tools.go index ad6a443..87c0dff 100644 --- a/internal/cli/tools.go +++ b/internal/cli/tools.go @@ -48,7 +48,7 @@ func (a *App) runTools(args []string) error { // Set custom help handler fs.Usage = func() { fmt.Print(GetCommandHelp("tools")) } - if err := fs.Parse(args); err != nil { + if err := fs.Parse(reorderArgs(args)); err != nil { if errors.Is(err, flag.ErrHelp) { return nil } diff --git a/internal/cli/useflags.go b/internal/cli/useflags.go index a595d6a..dc98b24 100644 --- a/internal/cli/useflags.go +++ b/internal/cli/useflags.go @@ -17,15 +17,20 @@ import ( // Common USE_EXPAND prefixes that should be displayed separately. // These are standard Portage USE_EXPAND variables. var defaultUSEExpandPrefixes = []string{ + "abi_mips_", + "abi_s390_", + "abi_x86_", "cpu_flags_x86_", "cpu_flags_arm_", - "python_targets_", + "input_devices_", + "l10n_", + "llvm_targets_", + "lua_single_target_", + "lua_targets_", "python_single_target_", + "python_targets_", "ruby_targets_", - "lua_targets_", - "l10n_", "video_cards_", - "input_devices_", } // FormatUSEFlags formats USE flags for display in emerge --pretend output. @@ -164,6 +169,11 @@ func resolvePackageUSE(p *pkg.Package, cfg *config.Config) (enabled, disabled [] } } + // Apply USE_EXPAND from make.conf (e.g., PYTHON_SINGLE_TARGET, PYTHON_TARGETS) + if cfg != nil { + applyUSEExpandToFlags(cfg, p.UseFlags, flagState) + } + // Apply per-package USE from package.use if cfg != nil { // Extract category and package name for pattern matching @@ -204,6 +214,35 @@ func resolvePackageUSE(p *pkg.Package, cfg *config.Config) (enabled, disabled [] return enabled, disabled } +// useExpandMapping maps USE_EXPAND variable names to their flag prefixes. +var useExpandMapping = []struct { + varName string + prefix string +}{ + {"PYTHON_SINGLE_TARGET", "python_single_target_"}, + {"PYTHON_TARGETS", "python_targets_"}, + {"LUA_SINGLE_TARGET", "lua_single_target_"}, + {"LUA_TARGETS", "lua_targets_"}, + {"RUBY_TARGETS", "ruby_targets_"}, +} + +// applyUSEExpandToFlags enables USE_EXPAND flags in flagState based on +// make.conf variables (e.g., PYTHON_TARGETS="python3_12" -> python_targets_python3_12=true). +func applyUSEExpandToFlags(cfg *config.Config, iuseFlags map[string]bool, flagState map[string]bool) { + for _, uev := range useExpandMapping { + value := cfg.GetVariable(uev.varName) + if value == "" { + continue + } + for _, val := range strings.Fields(value) { + flag := uev.prefix + strings.ToLower(val) + if _, exists := iuseFlags[flag]; exists { + flagState[flag] = true + } + } + } +} + // getUSEExpandPrefix returns the USE_EXPAND prefix if the flag matches one, // or empty string otherwise. // diff --git a/internal/ebuild/helpers_unpack.go b/internal/ebuild/helpers_unpack.go index 6708eee..6265341 100644 --- a/internal/ebuild/helpers_unpack.go +++ b/internal/ebuild/helpers_unpack.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/klauspost/compress/zstd" "github.com/ulikunitz/xz" ) @@ -23,7 +24,7 @@ import ( // Usage: unpack file.tar.gz // Usage: unpack ${A} // -// Supported formats: .tar.gz, .tar.bz2, .tar.xz, .tar, .zip +// Supported formats: .tar.gz, .tar.bz2, .tar.xz, .tar.zst, .tar, .zip, .gz, .bz2, .xz, .zst // Pure Go implementation, no external commands. func (h *Helpers) Unpack(args []string) error { if len(args) < 1 { @@ -93,10 +94,20 @@ func (h *Helpers) unpackArchive(archivePath, destDir string) error { return h.unpackTarBz2(archivePath, destDir) case strings.HasSuffix(lowerPath, ".tar.xz") || strings.HasSuffix(lowerPath, ".txz"): return h.unpackTarXz(archivePath, destDir) + case strings.HasSuffix(lowerPath, ".tar.zst") || strings.HasSuffix(lowerPath, ".tar.zstd"): + return h.unpackTarZst(archivePath, destDir) case strings.HasSuffix(lowerPath, ".tar"): return h.unpackTar(archivePath, destDir) case strings.HasSuffix(lowerPath, ".zip"): return h.unpackZip(archivePath, destDir) + case strings.HasSuffix(lowerPath, ".gz"): + return h.unpackSingleGz(archivePath, destDir) + case strings.HasSuffix(lowerPath, ".bz2"): + return h.unpackSingleBz2(archivePath, destDir) + case strings.HasSuffix(lowerPath, ".xz"): + return h.unpackSingleXz(archivePath, destDir) + case strings.HasSuffix(lowerPath, ".zst") || strings.HasSuffix(lowerPath, ".zstd"): + return h.unpackSingleZst(archivePath, destDir) default: return fmt.Errorf("unsupported archive format: %s", archivePath) } @@ -147,6 +158,103 @@ func (h *Helpers) unpackTarXz(archivePath, destDir string) error { return h.extractTar(tar.NewReader(xzReader), destDir) } +// unpackTarZst extracts a .tar.zst archive. +func (h *Helpers) unpackTarZst(archivePath, destDir string) error { + file, err := os.Open(archivePath) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + decoder, err := zstd.NewReader(file) + if err != nil { + return fmt.Errorf("zstd reader: %w", err) + } + defer decoder.Close() + + return h.extractTar(tar.NewReader(decoder), destDir) +} + +// unpackSingleGz decompresses a standalone .gz file (not a tar archive). +func (h *Helpers) unpackSingleGz(archivePath, destDir string) error { + file, err := os.Open(archivePath) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + gz, err := gzip.NewReader(file) + if err != nil { + return fmt.Errorf("gzip reader: %w", err) + } + defer func() { _ = gz.Close() }() + + outName := strings.TrimSuffix(filepath.Base(archivePath), ".gz") + return h.writeDecompressed(gz, filepath.Join(destDir, outName)) +} + +// unpackSingleBz2 decompresses a standalone .bz2 file. +func (h *Helpers) unpackSingleBz2(archivePath, destDir string) error { + file, err := os.Open(archivePath) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + outName := strings.TrimSuffix(filepath.Base(archivePath), ".bz2") + return h.writeDecompressed(bzip2.NewReader(file), filepath.Join(destDir, outName)) +} + +// unpackSingleXz decompresses a standalone .xz file. +func (h *Helpers) unpackSingleXz(archivePath, destDir string) error { + file, err := os.Open(archivePath) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + xzReader, err := xz.NewReader(file) + if err != nil { + return fmt.Errorf("xz reader: %w", err) + } + + outName := strings.TrimSuffix(filepath.Base(archivePath), ".xz") + return h.writeDecompressed(xzReader, filepath.Join(destDir, outName)) +} + +// unpackSingleZst decompresses a standalone .zst file. +func (h *Helpers) unpackSingleZst(archivePath, destDir string) error { + file, err := os.Open(archivePath) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + decoder, err := zstd.NewReader(file) + if err != nil { + return fmt.Errorf("zstd reader: %w", err) + } + defer decoder.Close() + + outName := strings.TrimSuffix(filepath.Base(archivePath), ".zst") + outName = strings.TrimSuffix(outName, ".zstd") + return h.writeDecompressed(decoder, filepath.Join(destDir, outName)) +} + +// writeDecompressed writes decompressed data from a reader to a file. +func (h *Helpers) writeDecompressed(r io.Reader, outPath string) error { + out, err := os.Create(outPath) + if err != nil { + return fmt.Errorf("create output: %w", err) + } + defer func() { _ = out.Close() }() + + if _, err := io.Copy(out, r); err != nil { + return fmt.Errorf("decompress: %w", err) + } + return nil +} + // unpackTar extracts a plain .tar archive. func (h *Helpers) unpackTar(archivePath, destDir string) error { file, err := os.Open(archivePath) diff --git a/internal/ebuild/phases_impl.go b/internal/ebuild/phases_impl.go index b5c20b5..923b7bb 100644 --- a/internal/ebuild/phases_impl.go +++ b/internal/ebuild/phases_impl.go @@ -131,7 +131,7 @@ func (e *Executor) phaseSetup() (string, error) { // // Uses the $A variable (populated from Manifest) to determine which // archives to extract. Falls back to pattern matching if $A is empty. -// Supports: .tar.gz, .tar.bz2, .tar.xz +// Supports: .tar.gz, .tar.bz2, .tar.xz, .tar.zst, .tar, .zip, .gz, .bz2, .xz, .zst func (e *Executor) phaseUnpack() (string, error) { var archives []string @@ -160,9 +160,20 @@ func (e *Executor) phaseUnpack() (string, error) { return "No source tarball found (ebuild-only package or binary)", nil } - // Extract each archive + // Extract each archive using helpers unpack (supports all formats). + // Skip non-archive files like PGP signatures and patches. + helpers := &Helpers{} var extracted []string for _, archive := range archives { + // Skip non-archive files (PGP signatures, patches, etc.) + lower := strings.ToLower(archive) + if strings.HasSuffix(lower, ".asc") || strings.HasSuffix(lower, ".sig") || + strings.HasSuffix(lower, ".sign") || strings.HasSuffix(lower, ".patch") || + strings.HasSuffix(lower, ".diff") { + logging.Debug("[ebuild] Skipping non-archive file: %s", archive) + continue + } + tarball := filepath.Join(e.Env.DISTDIR, archive) if _, err := os.Stat(tarball); err != nil { logging.Debug("[ebuild] Archive %s not found in DISTDIR, skipping", archive) @@ -170,7 +181,7 @@ func (e *Executor) phaseUnpack() (string, error) { } logging.Debug("[ebuild] Extracting %s to %s", archive, e.Env.WORKDIR) - if err := extractTarball(tarball, e.Env.WORKDIR); err != nil { + if err := helpers.unpackArchive(tarball, e.Env.WORKDIR); err != nil { return "", fmt.Errorf("failed to extract %s: %w", archive, err) } extracted = append(extracted, archive) diff --git a/internal/profile/parser.go b/internal/profile/parser.go index 6e869e5..c298ddf 100644 --- a/internal/profile/parser.go +++ b/internal/profile/parser.go @@ -28,11 +28,13 @@ func (p *Profile) loadEAPI() error { // loadMakeDefaults loads variables from the "make.defaults" file. // -// Format: +// Supports shell-style variable expansion: ${VAR} and $VAR references +// are replaced with previously defined values. This is critical for +// profiles like default/linux/make.defaults which use incremental assignment: // -// KEY="value" -// USE="ssl unicode" -// CFLAGS="-O2 -pipe" +// USE="crypt ipv6 pam ssl" +// USE="${USE} seccomp" +// USE="${USE} iconv" func (p *Profile) loadMakeDefaults() error { makeDefaultsPath := filepath.Join(p.Path, "make.defaults") file, err := os.Open(makeDefaultsPath) @@ -62,12 +64,54 @@ func (p *Profile) loadMakeDefaults() error { // Remove quotes value = strings.Trim(value, `"'`) + // Expand variable references: ${VAR} and $VAR + value = p.expandVariables(value) + p.MakeDefaults[key] = value } return scanner.Err() } +// expandVariables expands ${VAR} and $VAR references in a value string +// using previously stored MakeDefaults values. +func (p *Profile) expandVariables(value string) string { + // Expand ${VAR} references + for strings.Contains(value, "${") { + start := strings.Index(value, "${") + end := strings.Index(value[start:], "}") + if end == -1 { + break + } + end += start + varName := value[start+2 : end] + varValue := p.MakeDefaults[varName] + value = value[:start] + varValue + value[end+1:] + } + + // Expand $VAR references (without braces) — only uppercase var names + // to avoid false positives with e.g. $1 + for i := 0; i < len(value); i++ { + if value[i] != '$' || i+1 >= len(value) || value[i+1] == '{' { + continue + } + // Find end of variable name (uppercase letters + underscore + digits) + j := i + 1 + for j < len(value) && (value[j] >= 'A' && value[j] <= 'Z' || value[j] == '_' || value[j] >= '0' && value[j] <= '9') { + j++ + } + if j == i+1 { + continue + } + varName := value[i+1 : j] + varValue := p.MakeDefaults[varName] + value = value[:i] + varValue + value[j:] + i += len(varValue) - 1 + } + + return value +} + // loadUSEMask loads masked USE flags from the "use.mask" file. // // Format (one USE flag per line): diff --git a/internal/profile/profile_test.go b/internal/profile/profile_test.go index be16b35..f05619d 100644 --- a/internal/profile/profile_test.go +++ b/internal/profile/profile_test.go @@ -244,6 +244,37 @@ func TestGetUSEFlags(t *testing.T) { } } +func TestGetUSEFlags_IncrementalExpansion(t *testing.T) { + // Simulates default/linux/make.defaults which uses ${USE} expansion: + // USE="crypt ipv6 pam ssl" + // USE="${USE} seccomp" + // USE="${USE} iconv" + files := map[string]string{ + "eapi": "8", + "make.defaults": `USE="crypt ipv6 pam ssl" +USE="${USE} seccomp" +USE="${USE} iconv"`, + } + + dir := setupTestProfile(t, "incremental_use", files) + prof, err := LoadProfile(dir) + if err != nil { + t.Fatal(err) + } + + flags := prof.GetUSEFlags() + flagMap := make(map[string]bool) + for _, flag := range flags { + flagMap[flag] = true + } + + for _, expected := range []string{"crypt", "ipv6", "pam", "ssl", "seccomp", "iconv"} { + if !flagMap[expected] { + t.Errorf("Expected USE flag %q not found in %v", expected, flags) + } + } +} + func TestGetSystemPackages(t *testing.T) { files := map[string]string{ "eapi": "8", diff --git a/internal/repo/ebuild_parser.go b/internal/repo/ebuild_parser.go index abbfb86..78e0c33 100644 --- a/internal/repo/ebuild_parser.go +++ b/internal/repo/ebuild_parser.go @@ -14,9 +14,19 @@ var ( // Pattern: ^VARNAME="value" (single line) ebuildVarRe = coregex.MustCompile(`(?m)^([A-Z_][A-Z0-9_]*)="([^"]*(?:\\"[^"]*)*)"`) + // ebuildVarAppendRe matches VAR+="value" (bash append operator, single line). + // Pattern: ^VARNAME+="value" — appends to existing variable. + // Used in ebuilds like sys-libs/pam: BDEPEND+="acct-group/shadow ..." + ebuildVarAppendRe = coregex.MustCompile(`(?m)^([A-Z_][A-Z0-9_]*)\+="([^"]*(?:\\"[^"]*)*)"`) + // ebuildMultiLineVarRe matches multi-line variable assignments. // Pattern: VARNAME="line1\nline2\nline3" - ebuildMultiLineVarRe = coregex.MustCompile(`(?s)^([A-Z_][A-Z0-9_]*)="(.*?)"`) + // Uses (?ms): m=multiline ^/$, s=dotall (.=\n) + ebuildMultiLineVarRe = coregex.MustCompile(`(?ms)^([A-Z_][A-Z0-9_]*)="(.*?)"`) + + // ebuildMultiLineVarAppendRe matches multi-line VAR+="..." (append). + // Uses (?ms): m=multiline ^/$, s=dotall (.=\n) + ebuildMultiLineVarAppendRe = coregex.MustCompile(`(?ms)^([A-Z_][A-Z0-9_]*)\+="(.*?)"`) // ebuildVarRefRe matches variable references: ${VAR} or ${VAR:-default} ebuildVarRefRe = coregex.MustCompile(`\$\{([A-Z_][A-Z0-9_]*)(?::-([^}]*))?\}`) @@ -196,7 +206,8 @@ func (ep *EbuildParser) ParseDependencies() ([]ParsedDependency, error) { } // extractAllVariables extracts all variables from ebuild content -// Populates ep.variables map with variable name -> raw value mappings +// Populates ep.variables map with variable name -> raw value mappings. +// Handles both VAR="..." assignment and VAR+="..." append operators. func (ep *EbuildParser) extractAllVariables() { // Find all variable assignments using precompiled regex matches := ebuildVarRe.FindAllStringSubmatch(ep.content, -1) @@ -223,6 +234,37 @@ func (ep *EbuildParser) extractAllVariables() { } } } + + // Handle VAR+="..." append operator (single-line) + appendMatches := ebuildVarAppendRe.FindAllStringSubmatch(ep.content, -1) + for _, match := range appendMatches { + if len(match) >= 3 { + varName := match[1] + appendValue := match[2] + if existing, exists := ep.variables[varName]; exists { + ep.variables[varName] = existing + " " + appendValue + } else { + ep.variables[varName] = appendValue + } + } + } + + // Handle multi-line VAR+="..." append operator + multiAppendMatches := ebuildMultiLineVarAppendRe.FindAllStringSubmatch(ep.content, -1) + for _, match := range multiAppendMatches { + if len(match) >= 3 { + varName := match[1] + appendValue := strings.TrimSpace(match[2]) + // Check if already appended by single-line regex + if existing, exists := ep.variables[varName]; exists { + if !strings.Contains(existing, appendValue) { + ep.variables[varName] = existing + " " + appendValue + } + } else { + ep.variables[varName] = appendValue + } + } + } } // expandVariables recursively expands ${VAR} references in a string diff --git a/internal/repo/portage.go b/internal/repo/portage.go index 1cff5fa..41962df 100644 --- a/internal/repo/portage.go +++ b/internal/repo/portage.go @@ -152,6 +152,12 @@ func (pr *PortageRepository) getEffectiveUSE(category, pkgName, version, slot st } } + // 3b. Apply USE_EXPAND variables (PYTHON_SINGLE_TARGET, PYTHON_TARGETS, etc.) + // Profile defines USE_EXPAND="PYTHON_SINGLE_TARGET PYTHON_TARGETS ..." + // make.conf sets PYTHON_SINGLE_TARGET="python3_12" + // This expands to USE flag: python_single_target_python3_12 + pr.applyUSEExpand(effectiveUSE) + // 4. Apply package.use per-package USE flags if pr.config != nil { packageUSE := pr.config.GetPackageUSEForPackage(category, pkgName, version, slot) @@ -195,6 +201,58 @@ func (pr *PortageRepository) isUSEConditionalActive(useConditional string, effec return effectiveUSE[useConditional] } +// applyUSEExpand expands critical USE_EXPAND variables into USE flags. +// +// In Portage, USE_EXPAND variables are expanded into lowercase USE flags: +// +// PYTHON_SINGLE_TARGET="python3_12" → python_single_target_python3_12 +// PYTHON_TARGETS="python3_12" → python_targets_python3_12 +// +// Currently handles Python-related variables which are the most commonly used +// in dependency conditions. Full USE_EXPAND support (ABI_X86, ELIBC, etc.) +// requires additional implicit variable handling and is planned for later. +// +// Sources checked (in priority order): +// 1. make.conf variables +// 2. Profile make.defaults variables +func (pr *PortageRepository) applyUSEExpand(effectiveUSE map[string]bool) { + // Expand only well-known USE_EXPAND variables that appear in dependency conditions. + // Full USE_EXPAND expansion requires proper handling of implicit variables + // (ELIBC, KERNEL, ARCH) and ABI flags, which is deferred. + useExpandVars := []string{ + "PYTHON_SINGLE_TARGET", + "PYTHON_TARGETS", + "LUA_SINGLE_TARGET", + "LUA_TARGETS", + "RUBY_TARGETS", + } + + for _, varName := range useExpandVars { + prefix := strings.ToLower(varName) + "_" + + // Check make.conf first (higher priority) + var value string + if pr.config != nil { + value = pr.config.GetVariable(varName) + } + + // Fall back to profile make.defaults + if value == "" && pr.profile != nil { + value = pr.profile.MakeDefaults[varName] + } + + if value == "" { + continue + } + + // Expand each value into a USE flag + for _, val := range strings.Fields(value) { + flag := prefix + strings.ToLower(val) + effectiveUSE[flag] = true + } + } +} + func (pr *PortageRepository) LoadPackages(names []string) ([]*pkg.Package, error) { var packages []*pkg.Package @@ -284,12 +342,21 @@ func (pr *PortageRepository) LoadPackageVersion(name, version string) (*pkg.Pack //nolint:gocyclo // Complexity inherent to ebuild format parsing (IUSE, SLOT, deps, USE filtering) func (pr *PortageRepository) parseEbuild(name, path string) (*pkg.Package, error) { - // Check cache first + // Check in-memory cache first if cached, ok := pr.cache.Load(path); ok { logging.Debug("Cache hit for ebuild: %s", path) return cached.(*pkg.Package), nil } + // Try Portage metadata cache (md5-cache) first — it has correct eclass-merged values + if p, err := pr.parseFromMetadataCache(name, path); err != nil { + logging.Debug("Metadata cache error for %s: %v", name, err) + } else if p != nil { + // Store in in-memory cache and return + pr.cache.Store(path, p) + return p, nil + } + logging.Debug("Parsing ebuild: %s", path) content, err := os.ReadFile(path) if err != nil { @@ -701,3 +768,138 @@ func (pr *PortageRepository) loadDependenciesWithEclass(ebuildPath string, p *pk return allDeps, nil } + +// parseFromMetadataCache attempts to build a Package from the Portage metadata cache. +// The metadata cache (metadata/md5-cache/) contains pre-computed values generated by +// egencache, which correctly handles eclass inheritance, variable merging (+=), +// and dynamic variable generation. This is much more accurate than regex-based parsing. +// +// Cache format: simple key=value per line. Keys include DEPEND, RDEPEND, BDEPEND, +// IDEPEND, PDEPEND, IUSE, KEYWORDS, SLOT, REQUIRED_USE, INHERIT, etc. +// +// Returns nil, nil if cache file does not exist (caller should fall back to regex parsing). +func (pr *PortageRepository) parseFromMetadataCache(name, ebuildPath string) (*pkg.Package, error) { + // Extract category, package name, and version + category, pkgName, found := strings.Cut(name, "/") + if !found { + return nil, nil // Can't parse cache without proper name + } + + filename := filepath.Base(ebuildPath) + version := strings.TrimSuffix(filename, ".ebuild") + version = strings.TrimPrefix(version, pkgName+"-") + + // Construct metadata cache path: {repo}/metadata/md5-cache/{category}/{pkgName}-{version} + cachePath := filepath.Join(pr.Path, "metadata", "md5-cache", category, pkgName+"-"+version) + + cacheData, err := os.ReadFile(cachePath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil // No cache, fall back to ebuild parsing + } + return nil, nil // Other error, fall back silently + } + + // Parse cache file into key-value map + cacheVars := make(map[string]string) + for _, line := range strings.Split(string(cacheData), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "_") { + continue // Skip empty lines and internal keys (_eclasses_, _md5_) + } + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + cacheVars[key] = value + } + + // Build package from cache + p := &pkg.Package{ + Name: name, + Version: version, + Slot: pkg.Slot{Name: "0"}, + UseFlags: make(map[string]bool), + Keywords: make([]string, 0), + Deps: make([]pkg.Constraint, 0), + Provides: make([]pkg.Constraint, 0), + } + + // Parse SLOT + if slot := cacheVars["SLOT"]; slot != "" { + p.Slot = pkg.ParseSlot(slot) + } + + // Parse KEYWORDS + if keywords := cacheVars["KEYWORDS"]; keywords != "" { + p.Keywords = strings.Fields(keywords) + } + + // Parse IUSE and compute effective USE flags + iuseDefaults := make(map[string]bool) + if iuse := cacheVars["IUSE"]; iuse != "" { + for _, flag := range strings.Fields(iuse) { + if strings.HasPrefix(flag, "+") { + cleanFlag := strings.TrimPrefix(flag, "+") + iuseDefaults[cleanFlag] = true + p.UseFlags[cleanFlag] = true + } else if strings.HasPrefix(flag, "-") { + cleanFlag := strings.TrimPrefix(flag, "-") + iuseDefaults[cleanFlag] = false + p.UseFlags[cleanFlag] = true + } else { + iuseDefaults[flag] = false + p.UseFlags[flag] = true + } + } + } + + // Compute effective USE flags + effectiveUSE := pr.getEffectiveUSE(category, pkgName, p.Version, p.Slot.String(), iuseDefaults) + + // Parse dependencies from cache using EbuildParser + meta := NewPackageMetadata(category, pkgName, version) + parser := NewEbuildParserWithMetadata("", meta) + + depVars := map[string]DependencyType{ + "DEPEND": DepTypeBuild, + "RDEPEND": DepTypeRuntime, + "BDEPEND": DepTypeBuildtime, + "IDEPEND": DepTypeInstall, + "PDEPEND": DepTypePostMerge, + } + + for varName, depType := range depVars { + depStr := cacheVars[varName] + if depStr == "" { + continue + } + + deps, err := parser.parseDependencyString(depStr, depType) + if err != nil { + logging.Debug("Warning: failed to parse %s from cache for %s: %v", varName, name, err) + continue + } + + for _, pd := range deps { + if pd.IsBlocker { + continue + } + + // Filter by USE conditional + if !pr.isUSEConditionalActive(pd.UseFlag, effectiveUSE) { + continue + } + + constraint := pd.Constraint + constraint.OrGroupID = pd.OrGroupID + constraint.DepType = convertDepType(pd.DepType) + p.Deps = append(p.Deps, constraint) + } + } + + logging.Debug("Loaded %s-%s from metadata cache (%d deps, KEYWORDS=%v)", + name, version, len(p.Deps), p.Keywords) + + return p, nil +} diff --git a/internal/solver/resolver.go b/internal/solver/resolver.go index 775e3b1..2e22778 100644 --- a/internal/solver/resolver.go +++ b/internal/solver/resolver.go @@ -101,6 +101,24 @@ func (r *PortageResolver) isInstalled(name string) bool { return r.installedDB.IsInstalled(name) } +// sortAlternativesByInstalled reorders OR-group alternatives to put +// already-installed packages first. This creates a preference for keeping +// installed packages when the SAT solver picks from OR alternatives. +func (r *PortageResolver) sortAlternativesByInstalled(alternatives []pkg.Constraint) []pkg.Constraint { + sorted := make([]pkg.Constraint, 0, len(alternatives)) + var notInstalled []pkg.Constraint + + for _, alt := range alternatives { + if r.isInstalled(alt.Name) { + sorted = append(sorted, alt) + } else { + notInstalled = append(notInstalled, alt) + } + } + + return append(sorted, notInstalled...) +} + // groupDependenciesByOrGroupID groups dependencies by their OrGroupID // Returns required dependencies (OrGroupID=0) and OR-groups (OrGroupID>0) func groupDependenciesByOrGroupID(deps []pkg.Constraint) (requiredDeps []pkg.Constraint, orGroups map[int][]pkg.Constraint) { @@ -180,8 +198,9 @@ func (r *PortageResolver) collectDependencies(p *pkg.Package, allPackages map[st } } - // For OR-groups: Register all alternatives but DON'T collect their dependencies yet - // The SAT solver will choose ONE alternative from each group + // For OR-groups: add alternative packages to allPackages (so SAT solver knows + // about them), but DON'T recursively collect their dependencies yet. + // After SAT solving, we'll collect deps for chosen alternatives in a second pass. for groupID, alternatives := range orGroups { logging.Debug("OR-group %d for %s: %d alternatives", groupID, p.Name, len(alternatives)) for _, alt := range alternatives { @@ -193,10 +212,17 @@ func (r *PortageResolver) collectDependencies(p *pkg.Package, allPackages map[st if r.isInstalled(alt.Name) && !r.options.Deep { continue } - // Just ensure the alternative package exists in the repository - // Use loadUnmaskedPackage to filter masked alternatives - if _, err := r.loadUnmaskedPackage(alt.Name); err != nil { + altPkg, err := r.loadUnmaskedPackage(alt.Name) + if err != nil { logging.Debug("Warning: OR-alternative %s not found or masked: %v", alt.Name, err) + continue + } + // Add to allPackages so SAT solver has variables for it, + // but don't recurse into its deps (those would be required + // only if this alternative is chosen) + if _, exists := allPackages[altPkg.Name]; !exists { + copyPkg := *altPkg + allPackages[altPkg.Name] = ©Pkg } } } @@ -240,9 +266,13 @@ func (r *PortageResolver) addPackageConstraints(adapter *GophersatAdapter, p *pk } // Add OR-group constraints (OR logic) + // Prefer installed alternatives by putting them first in the clause. + // SAT solvers typically assign positive values to earlier variables, + // so this creates a soft preference for already-installed packages. for groupID, alternatives := range orGroups { logging.Debug("Adding OR-group %d with %d alternatives", groupID, len(alternatives)) - if err := adapter.AddOrGroupConstraint(alternatives); err != nil { + sorted := r.sortAlternativesByInstalled(alternatives) + if err := adapter.AddOrGroupConstraint(sorted); err != nil { logging.Debug("Warning: failed to add OR-group constraint: %v", err) } } @@ -325,6 +355,7 @@ func (r *PortageResolver) loadPackageFromAtom(atomStr string) (*pkg.Package, err return r.loadUnmaskedPackage(atom.CP()) } +//nolint:gocyclo // Complexity inherent to multi-pass Portage-compatible resolution with OR-group support func (r *PortageResolver) Resolve(packages []string) (map[string]*pkg.Package, error) { adapter := NewGophersatAdapter() allPackages := make(map[string]*pkg.Package) @@ -384,6 +415,65 @@ func (r *PortageResolver) Resolve(packages []string) (map[string]*pkg.Package, e return nil, err } + // Post-SAT pass: iteratively resolve transitive deps for all packages in the + // result. OR-group alternatives were added shallowly (without recursing into + // their deps), so we need to fill in the gaps now. Loop until convergence. + for pass := 1; pass <= 10; pass++ { + added := 0 + // Snapshot current result keys to avoid modifying map while iterating + currentPkgs := make([]*pkg.Package, 0, len(result)) + for _, p := range result { + currentPkgs = append(currentPkgs, p) + } + + for _, p := range currentPkgs { + // Group deps by OR-group + requiredDeps, orGroups := groupDependenciesByOrGroupID(p.Deps) + + // Required deps + for _, dep := range requiredDeps { + if _, inResult := result[dep.Name]; inResult { + continue + } + depPkg, err := r.loadUnmaskedPackage(dep.Name) + if err != nil { + continue + } + result[dep.Name] = depPkg + added++ + } + + // OR-groups: pick first available alternative (Portage default behavior) + for _, alternatives := range orGroups { + // Check if any alternative is already in result + satisfied := false + for _, alt := range alternatives { + if _, inResult := result[alt.Name]; inResult { + satisfied = true + break + } + } + if satisfied { + continue + } + // Pick first available alternative + for _, alt := range alternatives { + altPkg, err := r.loadUnmaskedPackage(alt.Name) + if err != nil { + continue + } + result[alt.Name] = altPkg + added++ + break // Take first available + } + } + } + logging.Debug("Post-pass %d: added %d packages", pass, added) + if added == 0 { + break + } + } + // Output formatted package list logging.Info("Resolved packages:") for name, p := range result {