index.js 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672
  1. var url = require("url");
  2. var URL = url.URL;
  3. var http = require("http");
  4. var https = require("https");
  5. var Writable = require("stream").Writable;
  6. var assert = require("assert");
  7. var debug = require("./debug");
  8. // Whether to use the native URL object or the legacy url module
  9. var useNativeURL = false;
  10. try {
  11. assert(new URL());
  12. }
  13. catch (error) {
  14. useNativeURL = error.code === "ERR_INVALID_URL";
  15. }
  16. // URL fields to preserve in copy operations
  17. var preservedUrlFields = [
  18. "auth",
  19. "host",
  20. "hostname",
  21. "href",
  22. "path",
  23. "pathname",
  24. "port",
  25. "protocol",
  26. "query",
  27. "search",
  28. "hash",
  29. ];
  30. // Create handlers that pass events from native requests
  31. var events = ["abort", "aborted", "connect", "error", "socket", "timeout"];
  32. var eventHandlers = Object.create(null);
  33. events.forEach(function (event) {
  34. eventHandlers[event] = function (arg1, arg2, arg3) {
  35. this._redirectable.emit(event, arg1, arg2, arg3);
  36. };
  37. });
  38. // Error types with codes
  39. var InvalidUrlError = createErrorType(
  40. "ERR_INVALID_URL",
  41. "Invalid URL",
  42. TypeError
  43. );
  44. var RedirectionError = createErrorType(
  45. "ERR_FR_REDIRECTION_FAILURE",
  46. "Redirected request failed"
  47. );
  48. var TooManyRedirectsError = createErrorType(
  49. "ERR_FR_TOO_MANY_REDIRECTS",
  50. "Maximum number of redirects exceeded",
  51. RedirectionError
  52. );
  53. var MaxBodyLengthExceededError = createErrorType(
  54. "ERR_FR_MAX_BODY_LENGTH_EXCEEDED",
  55. "Request body larger than maxBodyLength limit"
  56. );
  57. var WriteAfterEndError = createErrorType(
  58. "ERR_STREAM_WRITE_AFTER_END",
  59. "write after end"
  60. );
  61. // istanbul ignore next
  62. var destroy = Writable.prototype.destroy || noop;
  63. // An HTTP(S) request that can be redirected
  64. function RedirectableRequest(options, responseCallback) {
  65. // Initialize the request
  66. Writable.call(this);
  67. this._sanitizeOptions(options);
  68. this._options = options;
  69. this._ended = false;
  70. this._ending = false;
  71. this._redirectCount = 0;
  72. this._redirects = [];
  73. this._requestBodyLength = 0;
  74. this._requestBodyBuffers = [];
  75. // Attach a callback if passed
  76. if (responseCallback) {
  77. this.on("response", responseCallback);
  78. }
  79. // React to responses of native requests
  80. var self = this;
  81. this._onNativeResponse = function (response) {
  82. try {
  83. self._processResponse(response);
  84. }
  85. catch (cause) {
  86. self.emit("error", cause instanceof RedirectionError ?
  87. cause : new RedirectionError({ cause: cause }));
  88. }
  89. };
  90. // Perform the first request
  91. this._performRequest();
  92. }
  93. RedirectableRequest.prototype = Object.create(Writable.prototype);
  94. RedirectableRequest.prototype.abort = function () {
  95. destroyRequest(this._currentRequest);
  96. this._currentRequest.abort();
  97. this.emit("abort");
  98. };
  99. RedirectableRequest.prototype.destroy = function (error) {
  100. destroyRequest(this._currentRequest, error);
  101. destroy.call(this, error);
  102. return this;
  103. };
  104. // Writes buffered data to the current native request
  105. RedirectableRequest.prototype.write = function (data, encoding, callback) {
  106. // Writing is not allowed if end has been called
  107. if (this._ending) {
  108. throw new WriteAfterEndError();
  109. }
  110. // Validate input and shift parameters if necessary
  111. if (!isString(data) && !isBuffer(data)) {
  112. throw new TypeError("data should be a string, Buffer or Uint8Array");
  113. }
  114. if (isFunction(encoding)) {
  115. callback = encoding;
  116. encoding = null;
  117. }
  118. // Ignore empty buffers, since writing them doesn't invoke the callback
  119. // https://github.com/nodejs/node/issues/22066
  120. if (data.length === 0) {
  121. if (callback) {
  122. callback();
  123. }
  124. return;
  125. }
  126. // Only write when we don't exceed the maximum body length
  127. if (this._requestBodyLength + data.length <= this._options.maxBodyLength) {
  128. this._requestBodyLength += data.length;
  129. this._requestBodyBuffers.push({ data: data, encoding: encoding });
  130. this._currentRequest.write(data, encoding, callback);
  131. }
  132. // Error when we exceed the maximum body length
  133. else {
  134. this.emit("error", new MaxBodyLengthExceededError());
  135. this.abort();
  136. }
  137. };
  138. // Ends the current native request
  139. RedirectableRequest.prototype.end = function (data, encoding, callback) {
  140. // Shift parameters if necessary
  141. if (isFunction(data)) {
  142. callback = data;
  143. data = encoding = null;
  144. }
  145. else if (isFunction(encoding)) {
  146. callback = encoding;
  147. encoding = null;
  148. }
  149. // Write data if needed and end
  150. if (!data) {
  151. this._ended = this._ending = true;
  152. this._currentRequest.end(null, null, callback);
  153. }
  154. else {
  155. var self = this;
  156. var currentRequest = this._currentRequest;
  157. this.write(data, encoding, function () {
  158. self._ended = true;
  159. currentRequest.end(null, null, callback);
  160. });
  161. this._ending = true;
  162. }
  163. };
  164. // Sets a header value on the current native request
  165. RedirectableRequest.prototype.setHeader = function (name, value) {
  166. this._options.headers[name] = value;
  167. this._currentRequest.setHeader(name, value);
  168. };
  169. // Clears a header value on the current native request
  170. RedirectableRequest.prototype.removeHeader = function (name) {
  171. delete this._options.headers[name];
  172. this._currentRequest.removeHeader(name);
  173. };
  174. // Global timeout for all underlying requests
  175. RedirectableRequest.prototype.setTimeout = function (msecs, callback) {
  176. var self = this;
  177. // Destroys the socket on timeout
  178. function destroyOnTimeout(socket) {
  179. socket.setTimeout(msecs);
  180. socket.removeListener("timeout", socket.destroy);
  181. socket.addListener("timeout", socket.destroy);
  182. }
  183. // Sets up a timer to trigger a timeout event
  184. function startTimer(socket) {
  185. if (self._timeout) {
  186. clearTimeout(self._timeout);
  187. }
  188. self._timeout = setTimeout(function () {
  189. self.emit("timeout");
  190. clearTimer();
  191. }, msecs);
  192. destroyOnTimeout(socket);
  193. }
  194. // Stops a timeout from triggering
  195. function clearTimer() {
  196. // Clear the timeout
  197. if (self._timeout) {
  198. clearTimeout(self._timeout);
  199. self._timeout = null;
  200. }
  201. // Clean up all attached listeners
  202. self.removeListener("abort", clearTimer);
  203. self.removeListener("error", clearTimer);
  204. self.removeListener("response", clearTimer);
  205. self.removeListener("close", clearTimer);
  206. if (callback) {
  207. self.removeListener("timeout", callback);
  208. }
  209. if (!self.socket) {
  210. self._currentRequest.removeListener("socket", startTimer);
  211. }
  212. }
  213. // Attach callback if passed
  214. if (callback) {
  215. this.on("timeout", callback);
  216. }
  217. // Start the timer if or when the socket is opened
  218. if (this.socket) {
  219. startTimer(this.socket);
  220. }
  221. else {
  222. this._currentRequest.once("socket", startTimer);
  223. }
  224. // Clean up on events
  225. this.on("socket", destroyOnTimeout);
  226. this.on("abort", clearTimer);
  227. this.on("error", clearTimer);
  228. this.on("response", clearTimer);
  229. this.on("close", clearTimer);
  230. return this;
  231. };
  232. // Proxy all other public ClientRequest methods
  233. [
  234. "flushHeaders", "getHeader",
  235. "setNoDelay", "setSocketKeepAlive",
  236. ].forEach(function (method) {
  237. RedirectableRequest.prototype[method] = function (a, b) {
  238. return this._currentRequest[method](a, b);
  239. };
  240. });
  241. // Proxy all public ClientRequest properties
  242. ["aborted", "connection", "socket"].forEach(function (property) {
  243. Object.defineProperty(RedirectableRequest.prototype, property, {
  244. get: function () { return this._currentRequest[property]; },
  245. });
  246. });
  247. RedirectableRequest.prototype._sanitizeOptions = function (options) {
  248. // Ensure headers are always present
  249. if (!options.headers) {
  250. options.headers = {};
  251. }
  252. // Since http.request treats host as an alias of hostname,
  253. // but the url module interprets host as hostname plus port,
  254. // eliminate the host property to avoid confusion.
  255. if (options.host) {
  256. // Use hostname if set, because it has precedence
  257. if (!options.hostname) {
  258. options.hostname = options.host;
  259. }
  260. delete options.host;
  261. }
  262. // Complete the URL object when necessary
  263. if (!options.pathname && options.path) {
  264. var searchPos = options.path.indexOf("?");
  265. if (searchPos < 0) {
  266. options.pathname = options.path;
  267. }
  268. else {
  269. options.pathname = options.path.substring(0, searchPos);
  270. options.search = options.path.substring(searchPos);
  271. }
  272. }
  273. };
  274. // Executes the next native request (initial or redirect)
  275. RedirectableRequest.prototype._performRequest = function () {
  276. // Load the native protocol
  277. var protocol = this._options.protocol;
  278. var nativeProtocol = this._options.nativeProtocols[protocol];
  279. if (!nativeProtocol) {
  280. throw new TypeError("Unsupported protocol " + protocol);
  281. }
  282. // If specified, use the agent corresponding to the protocol
  283. // (HTTP and HTTPS use different types of agents)
  284. if (this._options.agents) {
  285. var scheme = protocol.slice(0, -1);
  286. this._options.agent = this._options.agents[scheme];
  287. }
  288. // Create the native request and set up its event handlers
  289. var request = this._currentRequest =
  290. nativeProtocol.request(this._options, this._onNativeResponse);
  291. request._redirectable = this;
  292. for (var event of events) {
  293. request.on(event, eventHandlers[event]);
  294. }
  295. // RFC7230§5.3.1: When making a request directly to an origin server, […]
  296. // a client MUST send only the absolute path […] as the request-target.
  297. this._currentUrl = /^\//.test(this._options.path) ?
  298. url.format(this._options) :
  299. // When making a request to a proxy, […]
  300. // a client MUST send the target URI in absolute-form […].
  301. this._options.path;
  302. // End a redirected request
  303. // (The first request must be ended explicitly with RedirectableRequest#end)
  304. if (this._isRedirect) {
  305. // Write the request entity and end
  306. var i = 0;
  307. var self = this;
  308. var buffers = this._requestBodyBuffers;
  309. (function writeNext(error) {
  310. // Only write if this request has not been redirected yet
  311. /* istanbul ignore else */
  312. if (request === self._currentRequest) {
  313. // Report any write errors
  314. /* istanbul ignore if */
  315. if (error) {
  316. self.emit("error", error);
  317. }
  318. // Write the next buffer if there are still left
  319. else if (i < buffers.length) {
  320. var buffer = buffers[i++];
  321. /* istanbul ignore else */
  322. if (!request.finished) {
  323. request.write(buffer.data, buffer.encoding, writeNext);
  324. }
  325. }
  326. // End the request if `end` has been called on us
  327. else if (self._ended) {
  328. request.end();
  329. }
  330. }
  331. }());
  332. }
  333. };
  334. // Processes a response from the current native request
  335. RedirectableRequest.prototype._processResponse = function (response) {
  336. // Store the redirected response
  337. var statusCode = response.statusCode;
  338. if (this._options.trackRedirects) {
  339. this._redirects.push({
  340. url: this._currentUrl,
  341. headers: response.headers,
  342. statusCode: statusCode,
  343. });
  344. }
  345. // RFC7231§6.4: The 3xx (Redirection) class of status code indicates
  346. // that further action needs to be taken by the user agent in order to
  347. // fulfill the request. If a Location header field is provided,
  348. // the user agent MAY automatically redirect its request to the URI
  349. // referenced by the Location field value,
  350. // even if the specific status code is not understood.
  351. // If the response is not a redirect; return it as-is
  352. var location = response.headers.location;
  353. if (!location || this._options.followRedirects === false ||
  354. statusCode < 300 || statusCode >= 400) {
  355. response.responseUrl = this._currentUrl;
  356. response.redirects = this._redirects;
  357. this.emit("response", response);
  358. // Clean up
  359. this._requestBodyBuffers = [];
  360. return;
  361. }
  362. // The response is a redirect, so abort the current request
  363. destroyRequest(this._currentRequest);
  364. // Discard the remainder of the response to avoid waiting for data
  365. response.destroy();
  366. // RFC7231§6.4: A client SHOULD detect and intervene
  367. // in cyclical redirections (i.e., "infinite" redirection loops).
  368. if (++this._redirectCount > this._options.maxRedirects) {
  369. throw new TooManyRedirectsError();
  370. }
  371. // Store the request headers if applicable
  372. var requestHeaders;
  373. var beforeRedirect = this._options.beforeRedirect;
  374. if (beforeRedirect) {
  375. requestHeaders = Object.assign({
  376. // The Host header was set by nativeProtocol.request
  377. Host: response.req.getHeader("host"),
  378. }, this._options.headers);
  379. }
  380. // RFC7231§6.4: Automatic redirection needs to done with
  381. // care for methods not known to be safe, […]
  382. // RFC7231§6.4.2–3: For historical reasons, a user agent MAY change
  383. // the request method from POST to GET for the subsequent request.
  384. var method = this._options.method;
  385. if ((statusCode === 301 || statusCode === 302) && this._options.method === "POST" ||
  386. // RFC7231§6.4.4: The 303 (See Other) status code indicates that
  387. // the server is redirecting the user agent to a different resource […]
  388. // A user agent can perform a retrieval request targeting that URI
  389. // (a GET or HEAD request if using HTTP) […]
  390. (statusCode === 303) && !/^(?:GET|HEAD)$/.test(this._options.method)) {
  391. this._options.method = "GET";
  392. // Drop a possible entity and headers related to it
  393. this._requestBodyBuffers = [];
  394. removeMatchingHeaders(/^content-/i, this._options.headers);
  395. }
  396. // Drop the Host header, as the redirect might lead to a different host
  397. var currentHostHeader = removeMatchingHeaders(/^host$/i, this._options.headers);
  398. // If the redirect is relative, carry over the host of the last request
  399. var currentUrlParts = parseUrl(this._currentUrl);
  400. var currentHost = currentHostHeader || currentUrlParts.host;
  401. var currentUrl = /^\w+:/.test(location) ? this._currentUrl :
  402. url.format(Object.assign(currentUrlParts, { host: currentHost }));
  403. // Create the redirected request
  404. var redirectUrl = resolveUrl(location, currentUrl);
  405. debug("redirecting to", redirectUrl.href);
  406. this._isRedirect = true;
  407. spreadUrlObject(redirectUrl, this._options);
  408. // Drop confidential headers when redirecting to a less secure protocol
  409. // or to a different domain that is not a superdomain
  410. if (redirectUrl.protocol !== currentUrlParts.protocol &&
  411. redirectUrl.protocol !== "https:" ||
  412. redirectUrl.host !== currentHost &&
  413. !isSubdomain(redirectUrl.host, currentHost)) {
  414. removeMatchingHeaders(/^(?:(?:proxy-)?authorization|cookie)$/i, this._options.headers);
  415. }
  416. // Evaluate the beforeRedirect callback
  417. if (isFunction(beforeRedirect)) {
  418. var responseDetails = {
  419. headers: response.headers,
  420. statusCode: statusCode,
  421. };
  422. var requestDetails = {
  423. url: currentUrl,
  424. method: method,
  425. headers: requestHeaders,
  426. };
  427. beforeRedirect(this._options, responseDetails, requestDetails);
  428. this._sanitizeOptions(this._options);
  429. }
  430. // Perform the redirected request
  431. this._performRequest();
  432. };
  433. // Wraps the key/value object of protocols with redirect functionality
  434. function wrap(protocols) {
  435. // Default settings
  436. var exports = {
  437. maxRedirects: 21,
  438. maxBodyLength: 10 * 1024 * 1024,
  439. };
  440. // Wrap each protocol
  441. var nativeProtocols = {};
  442. Object.keys(protocols).forEach(function (scheme) {
  443. var protocol = scheme + ":";
  444. var nativeProtocol = nativeProtocols[protocol] = protocols[scheme];
  445. var wrappedProtocol = exports[scheme] = Object.create(nativeProtocol);
  446. // Executes a request, following redirects
  447. function request(input, options, callback) {
  448. // Parse parameters, ensuring that input is an object
  449. if (isURL(input)) {
  450. input = spreadUrlObject(input);
  451. }
  452. else if (isString(input)) {
  453. input = spreadUrlObject(parseUrl(input));
  454. }
  455. else {
  456. callback = options;
  457. options = validateUrl(input);
  458. input = { protocol: protocol };
  459. }
  460. if (isFunction(options)) {
  461. callback = options;
  462. options = null;
  463. }
  464. // Set defaults
  465. options = Object.assign({
  466. maxRedirects: exports.maxRedirects,
  467. maxBodyLength: exports.maxBodyLength,
  468. }, input, options);
  469. options.nativeProtocols = nativeProtocols;
  470. if (!isString(options.host) && !isString(options.hostname)) {
  471. options.hostname = "::1";
  472. }
  473. assert.equal(options.protocol, protocol, "protocol mismatch");
  474. debug("options", options);
  475. return new RedirectableRequest(options, callback);
  476. }
  477. // Executes a GET request, following redirects
  478. function get(input, options, callback) {
  479. var wrappedRequest = wrappedProtocol.request(input, options, callback);
  480. wrappedRequest.end();
  481. return wrappedRequest;
  482. }
  483. // Expose the properties on the wrapped protocol
  484. Object.defineProperties(wrappedProtocol, {
  485. request: { value: request, configurable: true, enumerable: true, writable: true },
  486. get: { value: get, configurable: true, enumerable: true, writable: true },
  487. });
  488. });
  489. return exports;
  490. }
  491. function noop() { /* empty */ }
  492. function parseUrl(input) {
  493. var parsed;
  494. /* istanbul ignore else */
  495. if (useNativeURL) {
  496. parsed = new URL(input);
  497. }
  498. else {
  499. // Ensure the URL is valid and absolute
  500. parsed = validateUrl(url.parse(input));
  501. if (!isString(parsed.protocol)) {
  502. throw new InvalidUrlError({ input });
  503. }
  504. }
  505. return parsed;
  506. }
  507. function resolveUrl(relative, base) {
  508. /* istanbul ignore next */
  509. return useNativeURL ? new URL(relative, base) : parseUrl(url.resolve(base, relative));
  510. }
  511. function validateUrl(input) {
  512. if (/^\[/.test(input.hostname) && !/^\[[:0-9a-f]+\]$/i.test(input.hostname)) {
  513. throw new InvalidUrlError({ input: input.href || input });
  514. }
  515. if (/^\[/.test(input.host) && !/^\[[:0-9a-f]+\](:\d+)?$/i.test(input.host)) {
  516. throw new InvalidUrlError({ input: input.href || input });
  517. }
  518. return input;
  519. }
  520. function spreadUrlObject(urlObject, target) {
  521. var spread = target || {};
  522. for (var key of preservedUrlFields) {
  523. spread[key] = urlObject[key];
  524. }
  525. // Fix IPv6 hostname
  526. if (spread.hostname.startsWith("[")) {
  527. spread.hostname = spread.hostname.slice(1, -1);
  528. }
  529. // Ensure port is a number
  530. if (spread.port !== "") {
  531. spread.port = Number(spread.port);
  532. }
  533. // Concatenate path
  534. spread.path = spread.search ? spread.pathname + spread.search : spread.pathname;
  535. return spread;
  536. }
  537. function removeMatchingHeaders(regex, headers) {
  538. var lastValue;
  539. for (var header in headers) {
  540. if (regex.test(header)) {
  541. lastValue = headers[header];
  542. delete headers[header];
  543. }
  544. }
  545. return (lastValue === null || typeof lastValue === "undefined") ?
  546. undefined : String(lastValue).trim();
  547. }
  548. function createErrorType(code, message, baseClass) {
  549. // Create constructor
  550. function CustomError(properties) {
  551. Error.captureStackTrace(this, this.constructor);
  552. Object.assign(this, properties || {});
  553. this.code = code;
  554. this.message = this.cause ? message + ": " + this.cause.message : message;
  555. }
  556. // Attach constructor and set default properties
  557. CustomError.prototype = new (baseClass || Error)();
  558. Object.defineProperties(CustomError.prototype, {
  559. constructor: {
  560. value: CustomError,
  561. enumerable: false,
  562. },
  563. name: {
  564. value: "Error [" + code + "]",
  565. enumerable: false,
  566. },
  567. });
  568. return CustomError;
  569. }
  570. function destroyRequest(request, error) {
  571. for (var event of events) {
  572. request.removeListener(event, eventHandlers[event]);
  573. }
  574. request.on("error", noop);
  575. request.destroy(error);
  576. }
  577. function isSubdomain(subdomain, domain) {
  578. assert(isString(subdomain) && isString(domain));
  579. var dot = subdomain.length - domain.length - 1;
  580. return dot > 0 && subdomain[dot] === "." && subdomain.endsWith(domain);
  581. }
  582. function isString(value) {
  583. return typeof value === "string" || value instanceof String;
  584. }
  585. function isFunction(value) {
  586. return typeof value === "function";
  587. }
  588. function isBuffer(value) {
  589. return typeof value === "object" && ("length" in value);
  590. }
  591. function isURL(value) {
  592. return URL && value instanceof URL;
  593. }
  594. // Exports
  595. module.exports = wrap({ http: http, https: https });
  596. module.exports.wrap = wrap;