diff --git a/packages/core/package.json b/packages/core/package.json index 5cb1429b6a..78b4a78271 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -49,6 +49,7 @@ "get-ready": "catalog:", "globby": "catalog:", "is-type-of": "catalog:", + "multimatch": "catalog:", "node-homedir": "catalog:", "performance-ms": "catalog:", "ready-callback": "catalog:", diff --git a/packages/core/src/loader/egg_loader.ts b/packages/core/src/loader/egg_loader.ts index 57b892ea55..fcc0dfb3b0 100644 --- a/packages/core/src/loader/egg_loader.ts +++ b/packages/core/src/loader/egg_loader.ts @@ -13,7 +13,7 @@ import { isAsyncFunction, isClass, isGeneratorFunction, isObject, isPromise } fr import { homedir } from 'node-homedir'; import { now, diff } from 'performance-ms'; import { register as tsconfigPathsRegister } from 'tsconfig-paths'; -import { getParamNames, readJSONSync, readJSON, exists } from 'utility'; +import { getParamNames, readJSONSync } from 'utility'; import type { BaseContextClass } from '../base_context_class.ts'; import type { Context, EggCore, MiddlewareFunc } from '../egg.ts'; @@ -24,7 +24,7 @@ import { sequencify } from '../utils/sequencify.ts'; import { Timing } from '../utils/timing.ts'; import { type ContextLoaderOptions, ContextLoader } from './context_loader.ts'; import { type FileLoaderOptions, CaseStyle, FULLPATH, FileLoader } from './file_loader.ts'; -import { RealLoaderFS, type LoaderFS } from './loader_fs.ts'; +import { ManifestLoaderFS, RealLoaderFS, type LoaderFS } from './loader_fs.ts'; import { ManifestStore, type StartupManifest } from './manifest.ts'; const debug = debuglog('egg/core/loader/egg_loader'); @@ -95,7 +95,10 @@ export class EggLoader { */ constructor(options: EggLoaderOptions) { this.options = options; - this.loaderFS = this.options.loaderFS ?? new RealLoaderFS(); + const bundleStore = ManifestStore.getBundleStore(); + this.loaderFS = + this.options.loaderFS ?? + (bundleStore?.baseDir === this.options.baseDir ? new ManifestLoaderFS(bundleStore) : new RealLoaderFS()); assert(fs.existsSync(this.options.baseDir), `${this.options.baseDir} not exists`); assert(this.options.app, 'options.app is required'); assert(this.options.logger, 'options.logger is required'); @@ -176,6 +179,9 @@ export class EggLoader { this.manifest = ManifestStore.load(this.options.baseDir, this.serverEnv, this.serverScope) ?? ManifestStore.createCollector(this.options.baseDir); + if (!this.options.loaderFS && !(this.loaderFS instanceof ManifestLoaderFS)) { + this.loaderFS = new ManifestLoaderFS(this.manifest, this.loaderFS); + } } get app(): EggCore { @@ -644,8 +650,8 @@ export class EggLoader { let pkg: any; let eggPluginConfig: any; const pluginPackage = path.join(plugin.path as string, 'package.json'); - if (await utils.existsPath(pluginPackage)) { - pkg = await readJSON(pluginPackage); + if (this.loaderFS.exists(pluginPackage)) { + pkg = this.loaderFS.readJSON(pluginPackage); eggPluginConfig = pkg.eggPlugin; if (pkg.version) { plugin.version = pkg.version; @@ -848,7 +854,7 @@ export class EggLoader { } else if (exports.require) { realPluginPath = path.join(pluginPath, exports.require); } - if (exports.typescript && isSupportTypeScript() && !(await exists(realPluginPath))) { + if (exports.typescript && isSupportTypeScript() && !this.loaderFS.exists(realPluginPath)) { // if require/import path not exists, use typescript path for development stage realPluginPath = path.join(pluginPath, exports.typescript); debug('[formatPluginPathFromPackageJSON] use typescript path %o', realPluginPath); @@ -1796,6 +1802,7 @@ export class EggLoader { */ #collectConventionalDynamicFiles(manifest: StartupManifest): void { for (const unit of this.getLoadUnits()) { + this.#collectConventionFile(manifest, path.join(unit.path, 'package.json')); for (const load of CONVENTIONAL_MANIFEST_LOADS) { const target = path.join(unit.path, ...load.path); if (load.type === 'resolve') { @@ -1838,6 +1845,19 @@ export class EggLoader { return manifest.fileDiscovery[dirKey]; } + #collectConventionFile(manifest: StartupManifest, filepath: string): void { + const fileKey = this.#toManifestRel(filepath); + if (Object.values(manifest.resolveCache).includes(fileKey)) return; + if (!fs.existsSync(filepath) || !fs.statSync(filepath).isFile()) return; + + const dirKey = this.#toManifestRel(path.dirname(filepath)); + const basename = path.basename(filepath); + const files = manifest.fileDiscovery[dirKey] ?? []; + if (!files.includes(basename)) { + manifest.fileDiscovery[dirKey] = [...files, basename].sort(); + } + } + #toManifestRel(filepath: string): string { const rel = path.isAbsolute(filepath) ? path.relative(this.options.baseDir, filepath) : filepath; return rel.replaceAll(path.sep, '/'); diff --git a/packages/core/src/loader/loader_fs.ts b/packages/core/src/loader/loader_fs.ts index 8446c60c1a..f8e3739d97 100644 --- a/packages/core/src/loader/loader_fs.ts +++ b/packages/core/src/loader/loader_fs.ts @@ -1,9 +1,13 @@ import fs, { type Stats } from 'node:fs'; +import path from 'node:path'; +import type {} from '@eggjs/typings/global'; import globby from 'globby'; +import multimatch from 'multimatch'; import { readJSONSync } from 'utility'; import utils from '../utils/index.ts'; +import type { ManifestStore } from './manifest.ts'; export type LoaderFSGlobOptions = globby.GlobbyOptions; @@ -41,3 +45,312 @@ export class RealLoaderFS implements LoaderFS { return utils.loadFile(filepath); } } + +export class ManifestLoaderFS implements LoaderFS { + readonly #manifest: ManifestStore; + readonly #fallback: LoaderFS; + + constructor(manifest: ManifestStore, fallback: LoaderFS = new RealLoaderFS()) { + this.#manifest = manifest; + this.#fallback = fallback; + } + + exists(filepath: string): boolean { + return this.#hasManifestEntry(filepath) || this.#fallback.exists(filepath); + } + + stat(filepath: string): Stats { + const entry = this.#getManifestEntry(filepath); + if (entry) { + return createVirtualStats(entry.type); + } + return this.#fallback.stat(filepath); + } + + realpath(filepath: string): string { + const entry = this.#getManifestEntry(filepath); + if (entry) { + return this.#toAbsolute(entry.rel); + } + return this.#fallback.realpath(filepath); + } + + readJSON(filepath: string): T { + const entry = this.#getManifestEntry(filepath); + if (entry?.type === 'file') { + const bundled = this.#loadBundledModule(entry.rel); + if (bundled !== undefined) { + return unwrapDefaultExport(bundled) as T; + } + } + return this.#fallback.readJSON(filepath); + } + + glob(patterns: string | string[], options?: LoaderFSGlobOptions): string[] { + const cwd = options?.cwd === undefined ? process.cwd() : String(options.cwd); + const absoluteCwd = path.resolve(cwd); + const cwdRel = this.#toRelative(absoluteCwd); + const manifestFiles = this.#listManifestFilesUnder(cwdRel); + if (manifestFiles === undefined) { + return this.#fallback.glob(patterns, options); + } + + const matched = filterManifestGlob(manifestFiles, patterns, options); + if (options?.absolute) { + return matched.map((file) => path.join(absoluteCwd, file)); + } + return matched; + } + + async loadFile(filepath: string): Promise { + const entry = this.#getManifestEntry(filepath); + if (entry?.type === 'file') { + const bundled = this.#loadBundledModule(entry.rel); + if (bundled !== undefined) { + return unwrapDefaultExport(bundled); + } + } + return this.#fallback.loadFile(filepath); + } + + #hasManifestEntry(filepath: string): boolean { + return this.#getManifestEntry(filepath) !== undefined; + } + + #getManifestEntry(filepath: string): ManifestEntry | undefined { + const rel = this.#toRelative(filepath); + const resolved = this.#resolveManifestFile(rel); + if (resolved) { + return { type: 'file', rel: resolved }; + } + if (this.#isManifestDirectory(rel)) { + return { type: 'directory', rel }; + } + } + + #resolveManifestFile(rel: string): string | undefined { + const cache = this.#manifest.data.resolveCache; + if (Object.hasOwn(cache, rel)) { + return cache[rel] ?? undefined; + } + + if (this.#isManifestFile(rel)) { + return rel; + } + + return this.#resolveFromFileDiscovery(rel); + } + + #resolveFromFileDiscovery(rel: string): string | undefined { + let matchedDir: string | undefined; + for (const dir of Object.keys(this.#manifest.data.fileDiscovery)) { + if ((rel === dir || rel.startsWith(dir + '/')) && (!matchedDir || dir.length > matchedDir.length)) { + matchedDir = dir; + } + } + if (!matchedDir || rel === matchedDir) return; + + const request = rel.slice(matchedDir.length + 1); + for (const file of this.#manifest.data.fileDiscovery[matchedDir]) { + if (file === request) { + return path.posix.join(matchedDir, file); + } + + const ext = path.posix.extname(file); + if (!ext || ext === '.map') continue; + + const extensionlessFile = file.slice(0, -ext.length); + if (extensionlessFile === request || extensionlessFile === `${request}/index`) { + return path.posix.join(matchedDir, file); + } + } + } + + #isManifestFile(rel: string): boolean { + for (const [dir, files] of Object.entries(this.#manifest.data.fileDiscovery)) { + if (files.includes(path.posix.relative(dir, rel))) { + return true; + } + } + return Object.values(this.#manifest.data.resolveCache).includes(rel); + } + + #isManifestDirectory(rel: string): boolean { + if (rel === '') { + return this.#hasManifestData(); + } + if (Object.hasOwn(this.#manifest.data.fileDiscovery, rel)) { + return true; + } + + for (const [dir, files] of Object.entries(this.#manifest.data.fileDiscovery)) { + if (dir.startsWith(rel + '/')) { + return true; + } + for (const file of files) { + const fullRel = path.posix.join(dir, file); + if (path.posix.dirname(fullRel) === rel || fullRel.startsWith(rel + '/')) { + return true; + } + } + } + + for (const target of Object.values(this.#manifest.data.resolveCache)) { + if (target && (path.posix.dirname(target) === rel || target.startsWith(rel + '/'))) { + return true; + } + } + return false; + } + + #listManifestFilesUnder(cwdRel: string): string[] | undefined { + if (!this.#isManifestDirectory(cwdRel)) return; + + const files = new Set(); + for (const [dir, entries] of Object.entries(this.#manifest.data.fileDiscovery)) { + const prefix = dir === cwdRel ? '' : relativePrefix(cwdRel, dir); + if (prefix === undefined) continue; + for (const entry of entries) { + files.add(path.posix.join(prefix, entry)); + } + } + + for (const target of Object.values(this.#manifest.data.resolveCache)) { + if (!target) continue; + const prefix = relativePrefix(cwdRel, path.posix.dirname(target)); + if (prefix !== undefined) { + files.add(path.posix.join(prefix, path.posix.basename(target))); + } + } + return [...files].sort(); + } + + #loadBundledModule(rel: string): unknown { + const loader = globalThis.__EGG_BUNDLE_MODULE_LOADER__; + if (!loader) return undefined; + + for (const key of this.#bundleKeys(rel)) { + const loaded = loader(key); + if (loaded !== undefined) { + return loaded; + } + } + } + + #bundleKeys(rel: string): string[] { + const abs = this.#toAbsolute(rel); + return [...new Set([rel, normalizePath(abs)])]; + } + + #hasManifestData(): boolean { + return ( + Object.keys(this.#manifest.data.fileDiscovery).length > 0 || + Object.keys(this.#manifest.data.resolveCache).some((key) => this.#manifest.data.resolveCache[key] !== null) + ); + } + + #toRelative(filepath: string): string { + const rel = path.isAbsolute(filepath) ? path.relative(this.#manifest.baseDir, filepath) : filepath; + return normalizePath(rel); + } + + #toAbsolute(rel: string): string { + return path.isAbsolute(rel) ? rel : path.join(this.#manifest.baseDir, rel); + } +} + +interface ManifestEntry { + type: 'file' | 'directory'; + rel: string; +} + +function normalizePath(filepath: string): string { + return filepath.replaceAll(path.sep, '/'); +} + +function relativePrefix(cwdRel: string, dirRel: string): string | undefined { + if (cwdRel === '') return dirRel; + if (dirRel === cwdRel) return ''; + if (dirRel.startsWith(cwdRel + '/')) return dirRel.slice(cwdRel.length + 1); +} + +function filterManifestGlob(files: string[], patterns: string | string[], options?: LoaderFSGlobOptions): string[] { + const patternList = Array.isArray(patterns) ? patterns : [patterns]; + if (!patternList.some((pattern) => !pattern.startsWith('!'))) return []; + + const ignoreList = Array.isArray(options?.ignore) + ? options.ignore.map(String) + : options?.ignore + ? [String(options.ignore)] + : []; + const normalizedPatterns = patternList + .map(normalizeAlternationGroups) + .concat(ignoreList.map((pattern) => `!${normalizeAlternationGroups(pattern)}`)); + return multimatch(files, normalizedPatterns); +} + +function normalizeAlternationGroups(pattern: string): string { + let normalized = ''; + for (let index = 0; index < pattern.length; index++) { + const char = pattern[index]; + if (char !== '(' || isExtglobPrefix(pattern[index - 1])) { + normalized += char; + continue; + } + + const end = pattern.indexOf(')', index + 1); + if (end === -1) { + normalized += char; + continue; + } + + const group = pattern.slice(index + 1, end); + if (group.includes('|')) { + normalized += `{${group.replaceAll('|', ',')}}`; + index = end; + } else { + normalized += char; + } + } + return normalized; +} + +function isExtglobPrefix(char: string | undefined): boolean { + return char === '@' || char === '!' || char === '?' || char === '+' || char === '*'; +} + +function unwrapDefaultExport(value: unknown): unknown { + let unwrapped = value; + if (isRecord(unwrapped) && isRecord(unwrapped.default) && unwrapped.default.__esModule === true) { + unwrapped = unwrapped.default; + } + if (isRecord(unwrapped) && 'default' in unwrapped) { + return unwrapped.default; + } + return unwrapped; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object'; +} + +function createVirtualStats(type: 'file' | 'directory'): Stats { + const stat = Object.create(fs.Stats.prototype) as Stats; + const isFile = type === 'file'; + const timestamp = new Date(0); + Object.defineProperties(stat, { + size: { value: 0 }, + atimeMs: { value: 0 }, + mtimeMs: { value: 0 }, + ctimeMs: { value: 0 }, + birthtimeMs: { value: 0 }, + atime: { value: timestamp }, + mtime: { value: timestamp }, + ctime: { value: timestamp }, + birthtime: { value: timestamp }, + isFile: { value: () => isFile }, + isDirectory: { value: () => !isFile }, + isSymbolicLink: { value: () => false }, + }); + return stat; +} diff --git a/packages/core/test/__snapshots__/index.test.ts.snap b/packages/core/test/__snapshots__/index.test.ts.snap index fc1467a86e..02ac30dd77 100644 --- a/packages/core/test/__snapshots__/index.test.ts.snap +++ b/packages/core/test/__snapshots__/index.test.ts.snap @@ -18,6 +18,7 @@ exports[`should expose properties 1`] = ` "KoaRequest", "KoaResponse", "Lifecycle", + "ManifestLoaderFS", "ManifestStore", "RealLoaderFS", "Request", diff --git a/packages/core/test/loader/manifest_coverage.test.ts b/packages/core/test/loader/manifest_coverage.test.ts index 23a66288e6..4a7baf5d8e 100644 --- a/packages/core/test/loader/manifest_coverage.test.ts +++ b/packages/core/test/loader/manifest_coverage.test.ts @@ -175,6 +175,10 @@ describe('ManifestStore coverage: FileLoader getter auto-injects manifest', () = manifest.fileDiscovery['node_modules/@eggjs/security/app/middleware']?.includes('securities.js'), 'security middleware should be included in manifest fileDiscovery', ); + assert.ok( + manifest.fileDiscovery['node_modules/@eggjs/security']?.includes('package.json'), + 'plugin package metadata should be included in manifest fileDiscovery', + ); } finally { await testApp.close(); } diff --git a/packages/core/test/loader/manifest_loader_fs.test.ts b/packages/core/test/loader/manifest_loader_fs.test.ts new file mode 100644 index 0000000000..e833e242a4 --- /dev/null +++ b/packages/core/test/loader/manifest_loader_fs.test.ts @@ -0,0 +1,164 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { setBundleModuleLoader } from '@eggjs/utils'; +import { afterEach, describe, it } from 'vitest'; + +import { ManifestLoaderFS, RealLoaderFS, type LoaderFSGlobOptions } from '../../src/loader/loader_fs.ts'; +import { ManifestStore, type StartupManifest } from '../../src/loader/manifest.ts'; + +describe('test/loader/manifest_loader_fs.test.ts', () => { + const createdDirs: string[] = []; + + afterEach(async () => { + setBundleModuleLoader(undefined); + await Promise.all(createdDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); + }); + + it('serves manifest-backed stat, realpath, glob, readJSON, and loadFile from the bundle map', async () => { + const baseDir = await createTempDir(createdDirs, 'egg-manifest-loader-fs-'); + const manifest = createManifest({ + fileDiscovery: { + 'app/service': ['nested/order.ts', 'user.ts', 'user.d.ts'], + config: ['config.default.ts', 'plugin.ts'], + 'node_modules/fake-plugin': ['app.ts', 'package.json'], + }, + resolveCache: { + 'config/plugin': 'config/plugin.ts', + 'node_modules/fake-plugin/app': 'node_modules/fake-plugin/app.ts', + }, + }); + const store = ManifestStore.fromBundle(manifest, baseDir); + const modules: Record = { + 'config/plugin.ts': { default: { fakePlugin: { enable: true, package: 'fake-plugin' } } }, + [normalize(path.join(baseDir, 'config/plugin.ts'))]: { + default: { fakePlugin: { enable: true, package: 'fake-plugin' } }, + }, + 'node_modules/fake-plugin/package.json': { + default: { name: 'fake-plugin', version: '1.0.0', eggPlugin: { name: 'fakePlugin' } }, + }, + [normalize(path.join(baseDir, 'node_modules/fake-plugin/package.json'))]: { + default: { name: 'fake-plugin', version: '1.0.0', eggPlugin: { name: 'fakePlugin' } }, + }, + }; + setBundleModuleLoader((filepath) => modules[filepath]); + + const loaderFS = new ManifestLoaderFS(store); + const serviceFile = path.join(baseDir, 'app/service/user.ts'); + const serviceDir = path.join(baseDir, 'app/service'); + const pluginAlias = path.join(baseDir, 'config/plugin'); + const pluginPackage = path.join(baseDir, 'node_modules/fake-plugin/package.json'); + + assert.equal(loaderFS.exists(serviceFile), true); + assert.equal(loaderFS.exists(serviceDir), true); + assert.equal(loaderFS.exists(pluginAlias), true); + assert.equal(loaderFS.stat(serviceFile).isFile(), true); + const serviceDirStat = loaderFS.stat(serviceDir); + assert.equal(serviceDirStat.isDirectory(), true); + assert.equal(serviceDirStat.mtime.getTime(), 0); + assert.equal(loaderFS.realpath(pluginAlias), path.join(baseDir, 'config/plugin.ts')); + assert.deepEqual(loaderFS.glob(['**/*.(js|ts)', '!**/*.d.ts'], { cwd: serviceDir }), [ + 'nested/order.ts', + 'user.ts', + ]); + assert.deepEqual(loaderFS.glob('user.ts', { cwd: path.relative(process.cwd(), serviceDir), absolute: true }), [ + serviceFile, + ]); + assert.deepEqual(loaderFS.glob(['**/[no]*.ts', '!**/*.d.ts'], { cwd: serviceDir }), ['nested/order.ts']); + assert.deepEqual(loaderFS.readJSON(pluginPackage), { + name: 'fake-plugin', + version: '1.0.0', + eggPlugin: { name: 'fakePlugin' }, + }); + assert.deepEqual(await loaderFS.loadFile(pluginAlias), { fakePlugin: { enable: true, package: 'fake-plugin' } }); + }); + + it('falls back to the real filesystem for paths missing from the manifest', async () => { + const baseDir = await createTempDir(createdDirs, 'egg-manifest-loader-fs-fallback-'); + const realDir = path.join(baseDir, 'real'); + await fs.mkdir(realDir); + await fs.writeFile(path.join(realDir, 'package.json'), JSON.stringify({ name: 'real-package' })); + await fs.writeFile(path.join(realDir, 'config.js'), 'export default { real: true };\n'); + const store = ManifestStore.fromBundle(createManifest(), baseDir); + const loaderFS = new ManifestLoaderFS(store); + + assert.equal(loaderFS.exists(path.join(realDir, 'package.json')), true); + assert.equal(loaderFS.stat(path.join(realDir, 'package.json')).isFile(), true); + assert.deepEqual(loaderFS.glob('**/*.js', { cwd: realDir }), ['config.js']); + assert.deepEqual(loaderFS.readJSON(path.join(realDir, 'package.json')), { name: 'real-package' }); + assert.deepEqual(await loaderFS.loadFile(path.join(realDir, 'config.js')), { real: true }); + }); + + it('prefers manifest and bundle-map results over same-path real files', async () => { + const baseDir = await createTempDir(createdDirs, 'egg-manifest-loader-fs-priority-'); + await fs.mkdir(path.join(baseDir, 'config')); + await fs.writeFile(path.join(baseDir, 'config/plugin.js'), 'export default { source: "real" };\n'); + const manifest = createManifest({ + fileDiscovery: { + config: ['plugin.js'], + }, + }); + const store = ManifestStore.fromBundle(manifest, baseDir); + setBundleModuleLoader((filepath) => { + if (filepath === 'config/plugin.js' || filepath === normalize(path.join(baseDir, 'config/plugin.js'))) { + return { default: { source: 'manifest' } }; + } + }); + + const loaderFS = new ManifestLoaderFS(store); + + assert.deepEqual(await loaderFS.loadFile(path.join(baseDir, 'config/plugin.js')), { source: 'manifest' }); + }); + + it('does not fall back when manifest covers an empty directory', async () => { + const baseDir = await createTempDir(createdDirs, 'egg-manifest-loader-fs-empty-'); + const store = ManifestStore.fromBundle( + createManifest({ + fileDiscovery: { + empty: [], + }, + }), + baseDir, + ); + const loaderFS = new ManifestLoaderFS(store, new ThrowingGlobLoaderFS()); + + assert.deepEqual(loaderFS.glob('**/*.js', { cwd: path.join(baseDir, 'empty') }), []); + }); +}); + +class ThrowingGlobLoaderFS extends RealLoaderFS { + glob(_patterns: string | string[], _options?: LoaderFSGlobOptions): string[] { + throw new Error('unexpected real fs fallback'); + } +} + +async function createTempDir(createdDirs: string[], prefix: string): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + createdDirs.push(dir); + return dir; +} + +function createManifest( + overrides: Partial> = {}, +): StartupManifest { + return { + version: 1, + generatedAt: '2026-05-10T00:00:00.000Z', + invalidation: { + lockfileFingerprint: '', + configFingerprint: '', + serverEnv: 'unittest', + serverScope: '', + typescriptEnabled: true, + }, + extensions: {}, + resolveCache: overrides.resolveCache ?? {}, + fileDiscovery: overrides.fileDiscovery ?? {}, + }; +} + +function normalize(filepath: string): string { + return filepath.replaceAll(path.sep, '/'); +}