CachedInputFileSystem.js 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const nextTick = require("process").nextTick;
  7. /** @typedef {import("./Resolver").FileSystem} FileSystem */
  8. /** @typedef {import("./Resolver").SyncFileSystem} SyncFileSystem */
  9. /** @typedef {any} BaseFileSystem */
  10. /**
  11. * @template T
  12. * @typedef {import("./Resolver").FileSystemCallback<T>} FileSystemCallback<T>
  13. */
  14. /**
  15. * @param {string} path path
  16. * @returns {string} dirname
  17. */
  18. const dirname = path => {
  19. let idx = path.length - 1;
  20. while (idx >= 0) {
  21. const c = path.charCodeAt(idx);
  22. // slash or backslash
  23. if (c === 47 || c === 92) break;
  24. idx--;
  25. }
  26. if (idx < 0) return "";
  27. return path.slice(0, idx);
  28. };
  29. /**
  30. * @template T
  31. * @param {FileSystemCallback<T>[]} callbacks callbacks
  32. * @param {Error | undefined} err error
  33. * @param {T} result result
  34. */
  35. const runCallbacks = (callbacks, err, result) => {
  36. if (callbacks.length === 1) {
  37. callbacks[0](err, result);
  38. callbacks.length = 0;
  39. return;
  40. }
  41. let error;
  42. for (const callback of callbacks) {
  43. try {
  44. callback(err, result);
  45. } catch (e) {
  46. if (!error) error = e;
  47. }
  48. }
  49. callbacks.length = 0;
  50. if (error) throw error;
  51. };
  52. class OperationMergerBackend {
  53. /**
  54. * @param {function} provider async method in filesystem
  55. * @param {function} syncProvider sync method in filesystem
  56. * @param {BaseFileSystem} providerContext call context for the provider methods
  57. */
  58. constructor(provider, syncProvider, providerContext) {
  59. this._provider = provider;
  60. this._syncProvider = syncProvider;
  61. this._providerContext = providerContext;
  62. this._activeAsyncOperations = new Map();
  63. this.provide = this._provider
  64. ? /**
  65. * @param {string} path path
  66. * @param {any} options options
  67. * @param {function} callback callback
  68. * @returns {any} result
  69. */
  70. (path, options, callback) => {
  71. if (typeof options === "function") {
  72. callback = options;
  73. options = undefined;
  74. }
  75. if (options) {
  76. return this._provider.call(
  77. this._providerContext,
  78. path,
  79. options,
  80. callback
  81. );
  82. }
  83. if (typeof path !== "string") {
  84. callback(new TypeError("path must be a string"));
  85. return;
  86. }
  87. let callbacks = this._activeAsyncOperations.get(path);
  88. if (callbacks) {
  89. callbacks.push(callback);
  90. return;
  91. }
  92. this._activeAsyncOperations.set(path, (callbacks = [callback]));
  93. provider(
  94. path,
  95. /**
  96. * @param {Error} err error
  97. * @param {any} result result
  98. */
  99. (err, result) => {
  100. this._activeAsyncOperations.delete(path);
  101. runCallbacks(callbacks, err, result);
  102. }
  103. );
  104. }
  105. : null;
  106. this.provideSync = this._syncProvider
  107. ? /**
  108. * @param {string} path path
  109. * @param {any} options options
  110. * @returns {any} result
  111. */
  112. (path, options) => {
  113. return this._syncProvider.call(this._providerContext, path, options);
  114. }
  115. : null;
  116. }
  117. purge() {}
  118. purgeParent() {}
  119. }
  120. /*
  121. IDLE:
  122. insert data: goto SYNC
  123. SYNC:
  124. before provide: run ticks
  125. event loop tick: goto ASYNC_ACTIVE
  126. ASYNC:
  127. timeout: run tick, goto ASYNC_PASSIVE
  128. ASYNC_PASSIVE:
  129. before provide: run ticks
  130. IDLE --[insert data]--> SYNC --[event loop tick]--> ASYNC_ACTIVE --[interval tick]-> ASYNC_PASSIVE
  131. ^ |
  132. +---------[insert data]-------+
  133. */
  134. const STORAGE_MODE_IDLE = 0;
  135. const STORAGE_MODE_SYNC = 1;
  136. const STORAGE_MODE_ASYNC = 2;
  137. class CacheBackend {
  138. /**
  139. * @param {number} duration max cache duration of items
  140. * @param {function} provider async method
  141. * @param {function} syncProvider sync method
  142. * @param {BaseFileSystem} providerContext call context for the provider methods
  143. */
  144. constructor(duration, provider, syncProvider, providerContext) {
  145. this._duration = duration;
  146. this._provider = provider;
  147. this._syncProvider = syncProvider;
  148. this._providerContext = providerContext;
  149. /** @type {Map<string, FileSystemCallback<any>[]>} */
  150. this._activeAsyncOperations = new Map();
  151. /** @type {Map<string, { err?: Error, result?: any, level: Set<string> }>} */
  152. this._data = new Map();
  153. /** @type {Set<string>[]} */
  154. this._levels = [];
  155. for (let i = 0; i < 10; i++) this._levels.push(new Set());
  156. for (let i = 5000; i < duration; i += 500) this._levels.push(new Set());
  157. this._currentLevel = 0;
  158. this._tickInterval = Math.floor(duration / this._levels.length);
  159. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC} */
  160. this._mode = STORAGE_MODE_IDLE;
  161. /** @type {NodeJS.Timeout | undefined} */
  162. this._timeout = undefined;
  163. /** @type {number | undefined} */
  164. this._nextDecay = undefined;
  165. // @ts-ignore
  166. this.provide = provider ? this.provide.bind(this) : null;
  167. // @ts-ignore
  168. this.provideSync = syncProvider ? this.provideSync.bind(this) : null;
  169. }
  170. /**
  171. * @param {string} path path
  172. * @param {any} options options
  173. * @param {FileSystemCallback<any>} callback callback
  174. * @returns {void}
  175. */
  176. provide(path, options, callback) {
  177. if (typeof options === "function") {
  178. callback = options;
  179. options = undefined;
  180. }
  181. if (typeof path !== "string") {
  182. callback(new TypeError("path must be a string"));
  183. return;
  184. }
  185. if (options) {
  186. return this._provider.call(
  187. this._providerContext,
  188. path,
  189. options,
  190. callback
  191. );
  192. }
  193. // When in sync mode we can move to async mode
  194. if (this._mode === STORAGE_MODE_SYNC) {
  195. this._enterAsyncMode();
  196. }
  197. // Check in cache
  198. let cacheEntry = this._data.get(path);
  199. if (cacheEntry !== undefined) {
  200. if (cacheEntry.err) return nextTick(callback, cacheEntry.err);
  201. return nextTick(callback, null, cacheEntry.result);
  202. }
  203. // Check if there is already the same operation running
  204. let callbacks = this._activeAsyncOperations.get(path);
  205. if (callbacks !== undefined) {
  206. callbacks.push(callback);
  207. return;
  208. }
  209. this._activeAsyncOperations.set(path, (callbacks = [callback]));
  210. // Run the operation
  211. this._provider.call(
  212. this._providerContext,
  213. path,
  214. /**
  215. * @param {Error} [err] error
  216. * @param {any} [result] result
  217. */
  218. (err, result) => {
  219. this._activeAsyncOperations.delete(path);
  220. this._storeResult(path, err, result);
  221. // Enter async mode if not yet done
  222. this._enterAsyncMode();
  223. runCallbacks(
  224. /** @type {FileSystemCallback<any>[]} */ (callbacks),
  225. err,
  226. result
  227. );
  228. }
  229. );
  230. }
  231. /**
  232. * @param {string} path path
  233. * @param {any} options options
  234. * @returns {any} result
  235. */
  236. provideSync(path, options) {
  237. if (typeof path !== "string") {
  238. throw new TypeError("path must be a string");
  239. }
  240. if (options) {
  241. return this._syncProvider.call(this._providerContext, path, options);
  242. }
  243. // In sync mode we may have to decay some cache items
  244. if (this._mode === STORAGE_MODE_SYNC) {
  245. this._runDecays();
  246. }
  247. // Check in cache
  248. let cacheEntry = this._data.get(path);
  249. if (cacheEntry !== undefined) {
  250. if (cacheEntry.err) throw cacheEntry.err;
  251. return cacheEntry.result;
  252. }
  253. // Get all active async operations
  254. // This sync operation will also complete them
  255. const callbacks = this._activeAsyncOperations.get(path);
  256. this._activeAsyncOperations.delete(path);
  257. // Run the operation
  258. // When in idle mode, we will enter sync mode
  259. let result;
  260. try {
  261. result = this._syncProvider.call(this._providerContext, path);
  262. } catch (err) {
  263. this._storeResult(path, /** @type {Error} */ (err), undefined);
  264. this._enterSyncModeWhenIdle();
  265. if (callbacks) {
  266. runCallbacks(callbacks, /** @type {Error} */ (err), undefined);
  267. }
  268. throw err;
  269. }
  270. this._storeResult(path, undefined, result);
  271. this._enterSyncModeWhenIdle();
  272. if (callbacks) {
  273. runCallbacks(callbacks, undefined, result);
  274. }
  275. return result;
  276. }
  277. /**
  278. * @param {string|string[]|Set<string>} [what] what to purge
  279. */
  280. purge(what) {
  281. if (!what) {
  282. if (this._mode !== STORAGE_MODE_IDLE) {
  283. this._data.clear();
  284. for (const level of this._levels) {
  285. level.clear();
  286. }
  287. this._enterIdleMode();
  288. }
  289. } else if (typeof what === "string") {
  290. for (let [key, data] of this._data) {
  291. if (key.startsWith(what)) {
  292. this._data.delete(key);
  293. data.level.delete(key);
  294. }
  295. }
  296. if (this._data.size === 0) {
  297. this._enterIdleMode();
  298. }
  299. } else {
  300. for (let [key, data] of this._data) {
  301. for (const item of what) {
  302. if (key.startsWith(item)) {
  303. this._data.delete(key);
  304. data.level.delete(key);
  305. break;
  306. }
  307. }
  308. }
  309. if (this._data.size === 0) {
  310. this._enterIdleMode();
  311. }
  312. }
  313. }
  314. /**
  315. * @param {string|string[]|Set<string>} [what] what to purge
  316. */
  317. purgeParent(what) {
  318. if (!what) {
  319. this.purge();
  320. } else if (typeof what === "string") {
  321. this.purge(dirname(what));
  322. } else {
  323. const set = new Set();
  324. for (const item of what) {
  325. set.add(dirname(item));
  326. }
  327. this.purge(set);
  328. }
  329. }
  330. /**
  331. * @param {string} path path
  332. * @param {undefined | Error} err error
  333. * @param {any} result result
  334. */
  335. _storeResult(path, err, result) {
  336. if (this._data.has(path)) return;
  337. const level = this._levels[this._currentLevel];
  338. this._data.set(path, { err, result, level });
  339. level.add(path);
  340. }
  341. _decayLevel() {
  342. const nextLevel = (this._currentLevel + 1) % this._levels.length;
  343. const decay = this._levels[nextLevel];
  344. this._currentLevel = nextLevel;
  345. for (let item of decay) {
  346. this._data.delete(item);
  347. }
  348. decay.clear();
  349. if (this._data.size === 0) {
  350. this._enterIdleMode();
  351. } else {
  352. /** @type {number} */
  353. (this._nextDecay) += this._tickInterval;
  354. }
  355. }
  356. _runDecays() {
  357. while (
  358. /** @type {number} */ (this._nextDecay) <= Date.now() &&
  359. this._mode !== STORAGE_MODE_IDLE
  360. ) {
  361. this._decayLevel();
  362. }
  363. }
  364. _enterAsyncMode() {
  365. let timeout = 0;
  366. switch (this._mode) {
  367. case STORAGE_MODE_ASYNC:
  368. return;
  369. case STORAGE_MODE_IDLE:
  370. this._nextDecay = Date.now() + this._tickInterval;
  371. timeout = this._tickInterval;
  372. break;
  373. case STORAGE_MODE_SYNC:
  374. this._runDecays();
  375. // _runDecays may change the mode
  376. if (
  377. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC}*/
  378. (this._mode) === STORAGE_MODE_IDLE
  379. )
  380. return;
  381. timeout = Math.max(
  382. 0,
  383. /** @type {number} */ (this._nextDecay) - Date.now()
  384. );
  385. break;
  386. }
  387. this._mode = STORAGE_MODE_ASYNC;
  388. const ref = setTimeout(() => {
  389. this._mode = STORAGE_MODE_SYNC;
  390. this._runDecays();
  391. }, timeout);
  392. if (ref.unref) ref.unref();
  393. this._timeout = ref;
  394. }
  395. _enterSyncModeWhenIdle() {
  396. if (this._mode === STORAGE_MODE_IDLE) {
  397. this._mode = STORAGE_MODE_SYNC;
  398. this._nextDecay = Date.now() + this._tickInterval;
  399. }
  400. }
  401. _enterIdleMode() {
  402. this._mode = STORAGE_MODE_IDLE;
  403. this._nextDecay = undefined;
  404. if (this._timeout) clearTimeout(this._timeout);
  405. }
  406. }
  407. /**
  408. * @template {function} Provider
  409. * @template {function} AsyncProvider
  410. * @template FileSystem
  411. * @param {number} duration duration in ms files are cached
  412. * @param {Provider} provider provider
  413. * @param {AsyncProvider} syncProvider sync provider
  414. * @param {FileSystem} providerContext provider context
  415. * @returns {OperationMergerBackend | CacheBackend} backend
  416. */
  417. const createBackend = (duration, provider, syncProvider, providerContext) => {
  418. if (duration > 0) {
  419. return new CacheBackend(duration, provider, syncProvider, providerContext);
  420. }
  421. return new OperationMergerBackend(provider, syncProvider, providerContext);
  422. };
  423. module.exports = class CachedInputFileSystem {
  424. /**
  425. * @param {BaseFileSystem} fileSystem file system
  426. * @param {number} duration duration in ms files are cached
  427. */
  428. constructor(fileSystem, duration) {
  429. this.fileSystem = fileSystem;
  430. this._lstatBackend = createBackend(
  431. duration,
  432. this.fileSystem.lstat,
  433. this.fileSystem.lstatSync,
  434. this.fileSystem
  435. );
  436. const lstat = this._lstatBackend.provide;
  437. this.lstat = /** @type {FileSystem["lstat"]} */ (lstat);
  438. const lstatSync = this._lstatBackend.provideSync;
  439. this.lstatSync = /** @type {SyncFileSystem["lstatSync"]} */ (lstatSync);
  440. this._statBackend = createBackend(
  441. duration,
  442. this.fileSystem.stat,
  443. this.fileSystem.statSync,
  444. this.fileSystem
  445. );
  446. const stat = this._statBackend.provide;
  447. this.stat = /** @type {FileSystem["stat"]} */ (stat);
  448. const statSync = this._statBackend.provideSync;
  449. this.statSync = /** @type {SyncFileSystem["statSync"]} */ (statSync);
  450. this._readdirBackend = createBackend(
  451. duration,
  452. this.fileSystem.readdir,
  453. this.fileSystem.readdirSync,
  454. this.fileSystem
  455. );
  456. const readdir = this._readdirBackend.provide;
  457. this.readdir = /** @type {FileSystem["readdir"]} */ (readdir);
  458. const readdirSync = this._readdirBackend.provideSync;
  459. this.readdirSync = /** @type {SyncFileSystem["readdirSync"]} */ (
  460. readdirSync
  461. );
  462. this._readFileBackend = createBackend(
  463. duration,
  464. this.fileSystem.readFile,
  465. this.fileSystem.readFileSync,
  466. this.fileSystem
  467. );
  468. const readFile = this._readFileBackend.provide;
  469. this.readFile = /** @type {FileSystem["readFile"]} */ (readFile);
  470. const readFileSync = this._readFileBackend.provideSync;
  471. this.readFileSync = /** @type {SyncFileSystem["readFileSync"]} */ (
  472. readFileSync
  473. );
  474. this._readJsonBackend = createBackend(
  475. duration,
  476. // prettier-ignore
  477. this.fileSystem.readJson ||
  478. (this.readFile &&
  479. (
  480. /**
  481. * @param {string} path path
  482. * @param {FileSystemCallback<any>} callback
  483. */
  484. (path, callback) => {
  485. this.readFile(path, (err, buffer) => {
  486. if (err) return callback(err);
  487. if (!buffer || buffer.length === 0)
  488. return callback(new Error("No file content"));
  489. let data;
  490. try {
  491. data = JSON.parse(buffer.toString("utf-8"));
  492. } catch (e) {
  493. return callback(/** @type {Error} */ (e));
  494. }
  495. callback(null, data);
  496. });
  497. })
  498. ),
  499. // prettier-ignore
  500. this.fileSystem.readJsonSync ||
  501. (this.readFileSync &&
  502. (
  503. /**
  504. * @param {string} path path
  505. * @returns {any} result
  506. */
  507. (path) => {
  508. const buffer = this.readFileSync(path);
  509. const data = JSON.parse(buffer.toString("utf-8"));
  510. return data;
  511. }
  512. )),
  513. this.fileSystem
  514. );
  515. const readJson = this._readJsonBackend.provide;
  516. this.readJson = /** @type {FileSystem["readJson"]} */ (readJson);
  517. const readJsonSync = this._readJsonBackend.provideSync;
  518. this.readJsonSync = /** @type {SyncFileSystem["readJsonSync"]} */ (
  519. readJsonSync
  520. );
  521. this._readlinkBackend = createBackend(
  522. duration,
  523. this.fileSystem.readlink,
  524. this.fileSystem.readlinkSync,
  525. this.fileSystem
  526. );
  527. const readlink = this._readlinkBackend.provide;
  528. this.readlink = /** @type {FileSystem["readlink"]} */ (readlink);
  529. const readlinkSync = this._readlinkBackend.provideSync;
  530. this.readlinkSync = /** @type {SyncFileSystem["readlinkSync"]} */ (
  531. readlinkSync
  532. );
  533. }
  534. /**
  535. * @param {string|string[]|Set<string>} [what] what to purge
  536. */
  537. purge(what) {
  538. this._statBackend.purge(what);
  539. this._lstatBackend.purge(what);
  540. this._readdirBackend.purgeParent(what);
  541. this._readFileBackend.purge(what);
  542. this._readlinkBackend.purge(what);
  543. this._readJsonBackend.purge(what);
  544. }
  545. };