diff --git a/README.md b/README.md index 2caf96be..349b29a6 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ PackageCompiler is a Julia package with three main purposes: 3. Creating a relocatable C library bundle form of Julia code. + 4. Creating custom Julia distributions that look like the official binaries but ship with extra packages baked into the sysimage. + For installation and usage instructions, see the [documentation][docs-stable-url]. [docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg diff --git a/src/PackageCompiler.jl b/src/PackageCompiler.jl index 6fa7a565..d16e4664 100644 --- a/src/PackageCompiler.jl +++ b/src/PackageCompiler.jl @@ -12,7 +12,7 @@ using TOML using Glob using p7zip_jll: p7zip_path -export create_sysimage, create_app, create_library +export create_sysimage, create_app, create_distribution, create_library include("juliaconfig.jl") include("../ext/TerminalSpinners.jl") @@ -168,6 +168,18 @@ function gather_stdlibs_project(ctx) return stdlib_names end +function gather_dependency_entries(ctx; include_stdlibs::Bool=false) + manifest = ctx.env.manifest + manifest === nothing && return Pkg.Types.PackageEntry[] + entries = Pkg.Types.PackageEntry[] + for entry in values(manifest) + include_stdlibs || entry.name in _STDLIBS && continue + push!(entries, entry) + end + sort!(entries, by = entry -> something(entry.name, entry.uuid === nothing ? "" : string(entry.uuid))) + return entries +end + function check_packages_in_project(ctx, packages) packages_in_project = collect(keys(ctx.env.project.deps)) if ctx.env.pkg !== nothing @@ -398,6 +410,7 @@ function create_sysimg_object_file(object_file::String, script::Union{Nothing, String}, sysimage_build_args::Cmd, extra_precompiles::String, + release_banner::Union{Nothing, String}, incremental::Bool, import_into_main::Bool) julia_code_buffer = IOBuffer() @@ -425,8 +438,12 @@ function create_sysimg_object_file(object_file::String, push!(precompile_files, tracefile) end append!(precompile_files, abspath.(precompile_statements_file)) + banner_code = release_banner === nothing ? "" : """ + @eval Base const TAGGED_RELEASE_BANNER = $(repr(release_banner)) + """ precompile_code = """ # This @eval prevents symbols from being put into Main + $banner_code @eval Module() begin using Base.Meta PrecompileStagingArea = Module() @@ -619,6 +636,7 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector soname=nothing, compat_level::String="major", extra_precompiles::String = "", + release_banner::Union{Nothing, String}=nothing, import_into_main::Bool=true, ) # We call this at the very beginning to make sure that the user has a compiler available. Therefore, if no compiler @@ -725,6 +743,7 @@ function create_sysimage(packages::Union{Nothing, Symbol, Vector{String}, Vector script, sysimage_build_args, extra_precompiles, + release_banner, incremental, import_into_main) object_files = [object_file] @@ -941,6 +960,7 @@ function create_app(package_dir::String, stdlibs = unique(vcat(stdlibs, map(pkg -> pkg.name, stdlibs_in_default_sysimage()))) end bundle_julia_libraries(app_dir, stdlibs) + bundle_windows_import_libraries(app_dir) bundle_julia_libexec(ctx, app_dir) bundle_julia_executable(app_dir) bundle_artifacts(ctx, app_dir; include_lazy_artifacts) @@ -979,6 +999,95 @@ function create_app(package_dir::String, end end +""" + create_distribution(project_dir::String, dist_dir::String; kwargs...) + +Create a relocatable Julia tree rooted at `dist_dir` that behaves like the official +Julia downloads but with the dependencies of `project_dir` baked into the sysimage. +The baked packages are also exposed as stdlibs so that Pkg treats them as part of +the distribution. Packages are not imported into `Main`, keeping the default Julia +runtime behavior. + +# Keyword arguments + +- `precompile_execution_file`: Same as [`create_app`](@ref), these scripts are executed when generating + the sysimage. +- `precompile_statements_file`: Extra precompile statements appended to the sysimage build. +- `incremental::Bool=true`: Whether to extend the current Julia sysimage instead of creating a fresh one. +- `force::Bool=false`: Overwrite `dist_dir` if it exists. +- `cpu_target::String=default_app_cpu_target()`: CPU target used when compiling the sysimage. +- `include_lazy_artifacts::Bool=false`: If `true`, lazy artifacts referenced by dependencies are bundled. +- `sysimage_build_args::Cmd=```: Additional flags for the Julia process building the sysimage. +- `include_transitive_dependencies::Bool=true`: If `true`, include transitive dependencies in the sysimage. +- `include_preferences::Bool=true`: Bundle package preferences into `share/julia/LocalPreferences.toml`. +- `script::Union{Nothing,String}=nothing`: Optional script executed while generating the sysimage. +- `copy_globs::Vector{String}=String[]`: Glob patterns for copying package files to the stdlib directory. + Patterns are relative to each package root and apply to all packages in the distribution. + Example: `["assets/**", "data/**"]` copies assets and data directories for all packages. +""" +function create_distribution(project_dir::String, + dist_dir::String; + precompile_execution_file::Union{String, Vector{String}}=String[], + precompile_statements_file::Union{String, Vector{String}}=String[], + incremental::Bool=true, + force::Bool=false, + cpu_target::String=default_app_cpu_target(), + include_lazy_artifacts::Bool=false, + sysimage_build_args::Cmd=``, + include_transitive_dependencies::Bool=true, + include_preferences::Bool=true, + script::Union{Nothing, String}=nothing, + copy_globs::Vector{String}=String[], + release_banner::Union{Nothing, String}=nothing) + ctx = create_pkg_context(project_dir) + Pkg.instantiate(ctx, verbose=true, allow_autoprecomp=false) + + try_rm_dir(dist_dir; force) + ensure_default_depot_paths(dist_dir) + + # For distributions, we need to bundle libraries for ALL stdlibs from the running Julia + all_stdlibs = readdir(Sys.STDLIB) + bundle_julia_libraries(dist_dir, all_stdlibs) + bundle_windows_import_libraries(dist_dir) + + manifest_pkg_entries = gather_dependency_entries(ctx) + bundle_default_stdlibs(dist_dir) + bundle_custom_stdlibs(ctx, dist_dir, manifest_pkg_entries, copy_globs) + bundle_julia_test_files(dist_dir) + bundle_julia_base_files(dist_dir) + bundle_julia_compiler_files(dist_dir) + bundle_julia_support_files(dist_dir) + + # Get stdlibs that will be in the sysimage (as deps of custom packages) + stdlib_deps_in_sysimage = gather_dependency_entries(ctx; include_stdlibs=true) + stdlib_deps_names = [pkg.name for pkg in stdlib_deps_in_sysimage] + bundle_stdlib_project(dist_dir, stdlib_deps_names) + + bundle_julia_libexec(ctx, dist_dir) + bundle_julia_executable(dist_dir) + bundle_artifacts(ctx, dist_dir; include_lazy_artifacts) + include_preferences && bundle_preferences(ctx, dist_dir) + bundle_cert(dist_dir) + + sysimage_path = joinpath(dist_dir, "lib", "julia", "sys." * Libdl.dlext) + project = dirname(ctx.env.project_file) + create_sysimage(; sysimage_path, project, + incremental, + filter_stdlibs=false, + precompile_execution_file, + precompile_statements_file, + cpu_target, + sysimage_build_args, + include_transitive_dependencies, + script, + release_banner, + import_into_main=false) + + precompile_stdlibs(dist_dir, sysimage_path, cpu_target) + + return nothing +end + function create_executable_from_sysimg(exe_path::String, c_driver_program::String, @@ -1158,6 +1267,7 @@ function create_library(package_or_project::String, stdlibs = unique(vcat(stdlibs, map(pkg -> pkg.name, stdlibs_in_default_sysimage()))) end bundle_julia_libraries(dest_dir, stdlibs) + bundle_windows_import_libraries(dest_dir) bundle_julia_libexec(ctx, dest_dir) bundle_artifacts(ctx, dest_dir; include_lazy_artifacts) bundle_headers(dest_dir, header_files) @@ -1299,6 +1409,237 @@ function bundle_project(ctx, dir) Pkg.Types.write_project(d, joinpath(julia_share, "Project.toml")) end +function ensure_default_depot_paths(dest_dir) + mkpath(joinpath(dest_dir, "share", "julia")) + mkpath(joinpath(dest_dir, "local", "share", "julia")) +end + +function bundle_julia_test_files(dest_dir) + src_test = abspath(Sys.BINDIR, "..", "share", "julia", "test") + if isdir(src_test) + dest_test = joinpath(dest_dir, "share", "julia", "test") + if isdir(dest_test) + rm(dest_test; recursive=true, force=true) + end + cp(src_test, dest_test; force=true) + end +end + +function bundle_julia_base_files(dest_dir) + src_base = abspath(Sys.BINDIR, "..", "share", "julia", "base") + if isdir(src_base) + dest_base = joinpath(dest_dir, "share", "julia", "base") + if isdir(dest_base) + rm(dest_base; recursive=true, force=true) + end + cp(src_base, dest_base; force=true) + end +end + +function bundle_julia_compiler_files(dest_dir) + src_compiler = abspath(Sys.BINDIR, "..", "share", "julia", "Compiler") + if isdir(src_compiler) + dest_compiler = joinpath(dest_dir, "share", "julia", "Compiler") + if isdir(dest_compiler) + rm(dest_compiler; recursive=true, force=true) + end + cp(src_compiler, dest_compiler; force=true) + end +end + +function bundle_julia_support_files(dest_dir) + src_share = abspath(Sys.BINDIR, "..", "share", "julia") + dest_share = joinpath(dest_dir, "share", "julia") + + # Bundle individual files and directories + for item in ["julia-config.jl", "juliac", "terminfo"] + src_item = joinpath(src_share, item) + dest_item = joinpath(dest_share, item) + if isfile(src_item) + cp(src_item, dest_item; force=true) + elseif isdir(src_item) + if isdir(dest_item) + rm(dest_item; recursive=true, force=true) + end + cp(src_item, dest_item; force=true) + end + end +end + +function bundle_stdlib_project(dest_dir, packages_in_sysimage::Vector{String}) + # Dynamically generate stdlib Project.toml and Manifest.toml from running Julia's stdlibs + # Include all stdlibs in manifest for dependency resolution, but only list non-sysimage ones in Project.toml + dest_stdlib = joinpath(dest_dir, "share", "julia", "stdlib") + mkpath(dest_stdlib) + + # Get list of packages in the sysimage + sysimage_pkgs = Set(map(pkg -> pkg.name, stdlibs_in_default_sysimage())) + union!(sysimage_pkgs, packages_in_sysimage) + + # Collect stdlib packages for Project.toml (only non-sysimage) + stdlib_deps = Dict{String, String}() + # Collect ALL stdlib packages for Manifest.toml (needed for dependency resolution) + all_stdlib_deps = Dict{String, String}() + for pkg_name in readdir(Sys.STDLIB) + pkg_project = joinpath(Sys.STDLIB, pkg_name, "Project.toml") + if isfile(pkg_project) + proj = TOML.parsefile(pkg_project) + if haskey(proj, "uuid") + all_stdlib_deps[pkg_name] = proj["uuid"] + # Only add to Project.toml if not in sysimage + if !(pkg_name in sysimage_pkgs) + stdlib_deps[pkg_name] = proj["uuid"] + end + end + end + end + + # Generate Project.toml + project = Dict("deps" => stdlib_deps) + open(joinpath(dest_stdlib, "Project.toml"), "w") do io + TOML.print(io, project) + end + + # Generate Manifest.toml + manifest = Dict{String, Any}() + manifest["julia_version"] = string(VERSION) + manifest["manifest_format"] = "2.0" + manifest["deps"] = Dict{String, Any}() + + # Add each stdlib as a manifest entry (use all_stdlib_deps for complete dependency resolution) + for (name, uuid) in all_stdlib_deps + pkg_proj = TOML.parsefile(joinpath(Sys.STDLIB, name, "Project.toml")) + entry = Dict{String, Any}("uuid" => uuid) + if haskey(pkg_proj, "version") + entry["version"] = pkg_proj["version"] + end + if haskey(pkg_proj, "deps") + entry["deps"] = collect(keys(pkg_proj["deps"])) + end + if haskey(pkg_proj, "weakdeps") + entry["weakdeps"] = collect(keys(pkg_proj["weakdeps"])) + end + if haskey(pkg_proj, "extensions") + entry["extensions"] = pkg_proj["extensions"] + end + manifest["deps"][name] = [entry] + end + + open(joinpath(dest_stdlib, "Manifest.toml"), "w") do io + TOML.print(io, manifest) + end +end + +function precompile_stdlibs(dist_dir, sysimage_path, cpu_target) + julia_exe = joinpath(dist_dir, "bin", "julia") + depot_path = joinpath(dist_dir, "share", "julia") + stdlib_dir = joinpath(depot_path, "stdlib") + compiled_dir = joinpath(depot_path, "compiled") + mkpath(compiled_dir) + + # Precompile all packages in the stdlib that aren't in the sysimage + precompile_code = """ + Base.Precompilation.precompilepkgs(configs=[ + `` => Base.CacheFlags(debug_level=2, opt_level=3), + `` => Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3) + ]; io=stdout) + """ + + # Match Julia's pkgimage.mk approach: + # - JULIA_LOAD_PATH has @stdlib AND the stdlib dir (which contains Project.toml/Manifest.toml) + # - JULIA_PROJECT is NOT set (important for correct extension handling) + pathsep = Sys.iswindows() ? ";" : ":" + env = Dict( + "JULIA_DEPOT_PATH" => depot_path, + "JULIA_LOAD_PATH" => "@stdlib$(pathsep)$(stdlib_dir)", + "JULIA_CPU_TARGET" => cpu_target + ) + + cmd = setenv(`$julia_exe --sysimage=$sysimage_path --startup-file=no -e $precompile_code`, env) + + @info "Precompiling stdlibs for the distribution..." + run(cmd) +end + +function bundle_default_stdlibs(dest_dir) + src_stdlib = abspath(Sys.BINDIR, "..", "share", "julia", "stdlib") + dest_stdlib = joinpath(dest_dir, "share", "julia", "stdlib") + mkpath(dirname(dest_stdlib)) + if isdir(dest_stdlib) + rm(dest_stdlib; recursive=true, force=true) + end + cp(src_stdlib, dest_stdlib; force=true) +end + +function bundle_custom_stdlibs(ctx, dest_dir, packages::Vector{Pkg.Types.PackageEntry}, + copy_globs::Vector{String}=String[]) + isempty(packages) && return + version_dir = joinpath(dest_dir, "share", "julia", "stdlib", string('v', VERSION.major, '.', VERSION.minor)) + mkpath(version_dir) + for pkg in packages + pkg_source_path = source_path(ctx, pkg) + pkg_source_path === nothing && error("Unable to locate source for $(pkg.name); ensure the package exists in the current project.") + project_toml = joinpath(pkg_source_path, "Project.toml") + isfile(project_toml) || error("Project.toml for package $(pkg.name) not found at $(project_toml)") + pkg_name = something(pkg.name, string(pkg.uuid)) + pkg_stdlib_dir = joinpath(version_dir, pkg_name) + if isdir(pkg_stdlib_dir) + @debug "Stdlib directory $(pkg_stdlib_dir) already exists; not overwriting" + continue + end + mkpath(pkg_stdlib_dir) + cp(project_toml, joinpath(pkg_stdlib_dir, "Project.toml"); force=true) + + # Copy files matching glob patterns if specified + if !isempty(copy_globs) + for pattern in copy_globs + # For patterns ending with ** or **/*, copy entire directory tree recursively + if endswith(pattern, "**") || endswith(pattern, "**/*") + base_dir = replace(replace(pattern, "**/*" => ""), "**" => "") + base_dir = rstrip(base_dir, ['/', '\\']) + source_dir = joinpath(pkg_source_path, base_dir) + + if !isdir(source_dir) + @warn "Directory $source_dir not found for pattern $pattern in package $pkg_name" + continue + end + + # Recursively copy all files in the directory + for (root, dirs, files) in walkdir(source_dir) + for file in files + src_file = joinpath(root, file) + rel_path = relpath(src_file, pkg_source_path) + dest_file = joinpath(pkg_stdlib_dir, rel_path) + mkpath(dirname(dest_file)) + cp(src_file, dest_file; force=true) + end + end + else + # Use glob for specific patterns + matched_files = glob(pattern, pkg_source_path) + for src_file in matched_files + if isfile(src_file) + rel_path = relpath(src_file, pkg_source_path) + dest_file = joinpath(pkg_stdlib_dir, rel_path) + mkpath(dirname(dest_file)) + cp(src_file, dest_file; force=true) + end + end + end + end + else + # Create stub if no globs specified + stub_dir = joinpath(pkg_stdlib_dir, "src") + mkpath(stub_dir) + stub_path = joinpath(stub_dir, string(pkg_name, ".jl")) + open(stub_path, "w") do io + println(io, "# Autogenerated placeholder for $(pkg_name).") + println(io, "# The module implementation currently ships inside the sysimage.") + end + end + end +end + function bundle_julia_executable(dir::String) bindir = joinpath(dir, "bin") name = Base.julia_exename() @@ -1483,25 +1824,64 @@ function bundle_julia_libraries(dest_dir, stdlibs) return end -function bundle_julia_libexec(ctx, dest_dir) - # We only bundle the `7z` executable at the moment - @assert ctx.env.manifest !== nothing - if !any(x -> x.name == "p7zip_jll", values(ctx.env.manifest)) - return +# On Windows, bundle import libraries (.a files) needed for linking during package precompilation +function bundle_windows_import_libraries(dest_dir) + Sys.iswindows() || return + + # Import libraries are in lib/ and lib/julia/ + # Note: julia_libdir() returns bin/ on Windows (where DLLs are), but import libraries are in lib/ + src_lib_dir = abspath(Sys.BINDIR, Base.LIBDIR) + src_libjulia_dir = joinpath(src_lib_dir, "julia") + + # Destination mirrors the source structure + dest_lib_dir = joinpath(dest_dir, "lib") + dest_libjulia_dir = joinpath(dest_lib_dir, "julia") + mkpath(dest_lib_dir) + mkpath(dest_libjulia_dir) + + # Import libraries in lib/ (libjulia.dll.a, libjulia-internal.dll.a, libopenlibm.dll.a, libssp.dll.a) + for file in readdir(src_lib_dir) + endswith(file, ".dll.a") || continue + src = joinpath(src_lib_dir, file) + dest = joinpath(dest_lib_dir, file) + isfile(dest) && continue + cp(src, dest; force=true) + end + + # Import/static libraries in lib/julia/ (libgcc_s.a, libgcc.a, libmsvcrt.a, libssp.dll.a) + for file in readdir(src_libjulia_dir) + endswith(file, ".a") || continue + src = joinpath(src_libjulia_dir, file) + dest = joinpath(dest_libjulia_dir, file) + isfile(dest) && continue + cp(src, dest; force=true) end +end - # Use Julia-private `libexec` folder if it exsts +function bundle_julia_libexec(ctx, dest_dir) + # Use Julia-private `libexec` folder if it exists # (normpath is required in case `bin` does not exist in `dest_dir`) libexecdir_rel = if isdefined(Base, :PRIVATE_LIBEXECDIR) Base.PRIVATE_LIBEXECDIR else Base.LIBEXECDIR end + + source_libexec_dir = joinpath(Sys.BINDIR, libexecdir_rel) + if !isdir(source_libexec_dir) + return + end + bundle_libexec_dir = normpath(joinpath(dest_dir, "bin", libexecdir_rel)) mkpath(bundle_libexec_dir) - p7zip_exe = basename(p7zip_path) - cp(p7zip_path, joinpath(bundle_libexec_dir, p7zip_exe)) + # Copy all files from the libexec directory (7z, dsymutil, lld, etc.) + for file in readdir(source_libexec_dir) + src_path = joinpath(source_libexec_dir, file) + if isfile(src_path) + cp(src_path, joinpath(bundle_libexec_dir, file); force=true) + end + end return end diff --git a/test/runtests.jl b/test/runtests.jl index 8473c2e7..6f70b542 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,4 @@ -using PackageCompiler: PackageCompiler, create_sysimage, create_app, create_library +using PackageCompiler: PackageCompiler, create_sysimage, create_app, create_distribution, create_library using Test using Libdl using Pkg @@ -183,23 +183,53 @@ end end # testset if !is_slow_ci - # Test library creation - lib_source_dir = joinpath(@__DIR__, "..", "examples/MyLib") - lib_target_dir = joinpath(tmp, "MyLibCompiled") - - # This is why we have to skip this test on 1.12: - incremental = false - - filter = true - lib_name = "inc" - - tmp_lib_src_dir = joinpath(tmp, "MyLib") - cp(lib_source_dir, tmp_lib_src_dir) - create_library(tmp_lib_src_dir, lib_target_dir; incremental=incremental, force=true, filter_stdlibs=filter, - precompile_execution_file=joinpath(lib_source_dir, "build", "generate_precompile.jl"), - precompile_statements_file=joinpath(lib_source_dir, "build", "additional_precompile.jl"), - lib_name=lib_name, version=v"1.0.0") - rm(tmp_lib_src_dir; recursive=true) + @testset "create_distribution" begin + dist_source_dir = joinpath(@__DIR__, "..", "examples/MyApp/") + tmp_dist_source_dir = joinpath(tmp, "MyAppDistSource") + cp(dist_source_dir, tmp_dist_source_dir) + ctx = PackageCompiler.create_pkg_context(tmp_dist_source_dir) + expected_entries = PackageCompiler.gather_dependency_entries(ctx) + expected_names = [something(entry.name, string(entry.uuid)) for entry in expected_entries] + dist_target_dir = joinpath(tmp, "CustomJulia") + try + create_distribution(tmp_dist_source_dir, dist_target_dir; force=true, include_lazy_artifacts=true) + finally + rm(tmp_dist_source_dir; recursive=true) + rm(joinpath(new_depot, "packages"); recursive=true, force=true) + rm(joinpath(new_depot, "compiled"); recursive=true, force=true) + rm(joinpath(new_depot, "artifacts"); recursive=true, force=true) + end + julia_bin = joinpath(dist_target_dir, "bin", Base.julia_exename()) + output = read(`$(julia_bin) -e 'using Example; print(Example.hello("distribution"))'`, String) + @test occursin("Hello, distribution", output) + stdlib_version_dir = joinpath(dist_target_dir, "share", "julia", "stdlib", string('v', VERSION.major, '.', VERSION.minor)) + for name in expected_names + project_path = joinpath(stdlib_version_dir, name, "Project.toml") + stub_path = joinpath(stdlib_version_dir, name, "src", string(name, ".jl")) + @test isfile(project_path) + @test isfile(stub_path) + end + end + + @testset "create_library" begin + # Test library creation + lib_source_dir = joinpath(@__DIR__, "..", "examples/MyLib") + lib_target_dir = joinpath(tmp, "MyLibCompiled") + + # This is why we have to skip this test on 1.12: + incremental = false + + filter = true + lib_name = "inc" + + tmp_lib_src_dir = joinpath(tmp, "MyLib") + cp(lib_source_dir, tmp_lib_src_dir) + create_library(tmp_lib_src_dir, lib_target_dir; incremental=incremental, force=true, filter_stdlibs=filter, + precompile_execution_file=joinpath(lib_source_dir, "build", "generate_precompile.jl"), + precompile_statements_file=joinpath(lib_source_dir, "build", "additional_precompile.jl"), + lib_name=lib_name, version=v"1.0.0") + rm(tmp_lib_src_dir; recursive=true) + end end # Test creating an empty sysimage