index.js 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568
  1. var url = require("url");
  2. var URL = url.URL;
  3. var http = require("http");
  4. var https = require("https");
  5. var Writable = require("stream").Writable;
  6. var assert = require("assert");
  7. var debug = require("./debug");
  8. // Create handlers that pass events from native requests
  9. var events = ["abort", "aborted", "connect", "error", "socket", "timeout"];
  10. var eventHandlers = Object.create(null);
  11. events.forEach(function (event) {
  12. eventHandlers[event] = function (arg1, arg2, arg3) {
  13. this._redirectable.emit(event, arg1, arg2, arg3);
  14. };
  15. });
  16. // Error types with codes
  17. var RedirectionError = createErrorType(
  18. "ERR_FR_REDIRECTION_FAILURE",
  19. "Redirected request failed"
  20. );
  21. var TooManyRedirectsError = createErrorType(
  22. "ERR_FR_TOO_MANY_REDIRECTS",
  23. "Maximum number of redirects exceeded"
  24. );
  25. var MaxBodyLengthExceededError = createErrorType(
  26. "ERR_FR_MAX_BODY_LENGTH_EXCEEDED",
  27. "Request body larger than maxBodyLength limit"
  28. );
  29. var WriteAfterEndError = createErrorType(
  30. "ERR_STREAM_WRITE_AFTER_END",
  31. "write after end"
  32. );
  33. // An HTTP(S) request that can be redirected
  34. function RedirectableRequest(options, responseCallback) {
  35. // Initialize the request
  36. Writable.call(this);
  37. this._sanitizeOptions(options);
  38. this._options = options;
  39. this._ended = false;
  40. this._ending = false;
  41. this._redirectCount = 0;
  42. this._redirects = [];
  43. this._requestBodyLength = 0;
  44. this._requestBodyBuffers = [];
  45. // Attach a callback if passed
  46. if (responseCallback) {
  47. this.on("response", responseCallback);
  48. }
  49. // React to responses of native requests
  50. var self = this;
  51. this._onNativeResponse = function (response) {
  52. self._processResponse(response);
  53. };
  54. // Perform the first request
  55. this._performRequest();
  56. }
  57. RedirectableRequest.prototype = Object.create(Writable.prototype);
  58. RedirectableRequest.prototype.abort = function () {
  59. abortRequest(this._currentRequest);
  60. this.emit("abort");
  61. };
  62. // Writes buffered data to the current native request
  63. RedirectableRequest.prototype.write = function (data, encoding, callback) {
  64. // Writing is not allowed if end has been called
  65. if (this._ending) {
  66. throw new WriteAfterEndError();
  67. }
  68. // Validate input and shift parameters if necessary
  69. if (!(typeof data === "string" || typeof data === "object" && ("length" in data))) {
  70. throw new TypeError("data should be a string, Buffer or Uint8Array");
  71. }
  72. if (typeof encoding === "function") {
  73. callback = encoding;
  74. encoding = null;
  75. }
  76. // Ignore empty buffers, since writing them doesn't invoke the callback
  77. // https://github.com/nodejs/node/issues/22066
  78. if (data.length === 0) {
  79. if (callback) {
  80. callback();
  81. }
  82. return;
  83. }
  84. // Only write when we don't exceed the maximum body length
  85. if (this._requestBodyLength + data.length <= this._options.maxBodyLength) {
  86. this._requestBodyLength += data.length;
  87. this._requestBodyBuffers.push({ data: data, encoding: encoding });
  88. this._currentRequest.write(data, encoding, callback);
  89. }
  90. // Error when we exceed the maximum body length
  91. else {
  92. this.emit("error", new MaxBodyLengthExceededError());
  93. this.abort();
  94. }
  95. };
  96. // Ends the current native request
  97. RedirectableRequest.prototype.end = function (data, encoding, callback) {
  98. // Shift parameters if necessary
  99. if (typeof data === "function") {
  100. callback = data;
  101. data = encoding = null;
  102. }
  103. else if (typeof encoding === "function") {
  104. callback = encoding;
  105. encoding = null;
  106. }
  107. // Write data if needed and end
  108. if (!data) {
  109. this._ended = this._ending = true;
  110. this._currentRequest.end(null, null, callback);
  111. }
  112. else {
  113. var self = this;
  114. var currentRequest = this._currentRequest;
  115. this.write(data, encoding, function () {
  116. self._ended = true;
  117. currentRequest.end(null, null, callback);
  118. });
  119. this._ending = true;
  120. }
  121. };
  122. // Sets a header value on the current native request
  123. RedirectableRequest.prototype.setHeader = function (name, value) {
  124. this._options.headers[name] = value;
  125. this._currentRequest.setHeader(name, value);
  126. };
  127. // Clears a header value on the current native request
  128. RedirectableRequest.prototype.removeHeader = function (name) {
  129. delete this._options.headers[name];
  130. this._currentRequest.removeHeader(name);
  131. };
  132. // Global timeout for all underlying requests
  133. RedirectableRequest.prototype.setTimeout = function (msecs, callback) {
  134. var self = this;
  135. // Destroys the socket on timeout
  136. function destroyOnTimeout(socket) {
  137. socket.setTimeout(msecs);
  138. socket.removeListener("timeout", socket.destroy);
  139. socket.addListener("timeout", socket.destroy);
  140. }
  141. // Sets up a timer to trigger a timeout event
  142. function startTimer(socket) {
  143. if (self._timeout) {
  144. clearTimeout(self._timeout);
  145. }
  146. self._timeout = setTimeout(function () {
  147. self.emit("timeout");
  148. clearTimer();
  149. }, msecs);
  150. destroyOnTimeout(socket);
  151. }
  152. // Stops a timeout from triggering
  153. function clearTimer() {
  154. // Clear the timeout
  155. if (self._timeout) {
  156. clearTimeout(self._timeout);
  157. self._timeout = null;
  158. }
  159. // Clean up all attached listeners
  160. self.removeListener("abort", clearTimer);
  161. self.removeListener("error", clearTimer);
  162. self.removeListener("response", clearTimer);
  163. if (callback) {
  164. self.removeListener("timeout", callback);
  165. }
  166. if (!self.socket) {
  167. self._currentRequest.removeListener("socket", startTimer);
  168. }
  169. }
  170. // Attach callback if passed
  171. if (callback) {
  172. this.on("timeout", callback);
  173. }
  174. // Start the timer if or when the socket is opened
  175. if (this.socket) {
  176. startTimer(this.socket);
  177. }
  178. else {
  179. this._currentRequest.once("socket", startTimer);
  180. }
  181. // Clean up on events
  182. this.on("socket", destroyOnTimeout);
  183. this.on("abort", clearTimer);
  184. this.on("error", clearTimer);
  185. this.on("response", clearTimer);
  186. return this;
  187. };
  188. // Proxy all other public ClientRequest methods
  189. [
  190. "flushHeaders", "getHeader",
  191. "setNoDelay", "setSocketKeepAlive",
  192. ].forEach(function (method) {
  193. RedirectableRequest.prototype[method] = function (a, b) {
  194. return this._currentRequest[method](a, b);
  195. };
  196. });
  197. // Proxy all public ClientRequest properties
  198. ["aborted", "connection", "socket"].forEach(function (property) {
  199. Object.defineProperty(RedirectableRequest.prototype, property, {
  200. get: function () { return this._currentRequest[property]; },
  201. });
  202. });
  203. RedirectableRequest.prototype._sanitizeOptions = function (options) {
  204. // Ensure headers are always present
  205. if (!options.headers) {
  206. options.headers = {};
  207. }
  208. // Since http.request treats host as an alias of hostname,
  209. // but the url module interprets host as hostname plus port,
  210. // eliminate the host property to avoid confusion.
  211. if (options.host) {
  212. // Use hostname if set, because it has precedence
  213. if (!options.hostname) {
  214. options.hostname = options.host;
  215. }
  216. delete options.host;
  217. }
  218. // Complete the URL object when necessary
  219. if (!options.pathname && options.path) {
  220. var searchPos = options.path.indexOf("?");
  221. if (searchPos < 0) {
  222. options.pathname = options.path;
  223. }
  224. else {
  225. options.pathname = options.path.substring(0, searchPos);
  226. options.search = options.path.substring(searchPos);
  227. }
  228. }
  229. };
  230. // Executes the next native request (initial or redirect)
  231. RedirectableRequest.prototype._performRequest = function () {
  232. // Load the native protocol
  233. var protocol = this._options.protocol;
  234. var nativeProtocol = this._options.nativeProtocols[protocol];
  235. if (!nativeProtocol) {
  236. this.emit("error", new TypeError("Unsupported protocol " + protocol));
  237. return;
  238. }
  239. // If specified, use the agent corresponding to the protocol
  240. // (HTTP and HTTPS use different types of agents)
  241. if (this._options.agents) {
  242. var scheme = protocol.substr(0, protocol.length - 1);
  243. this._options.agent = this._options.agents[scheme];
  244. }
  245. // Create the native request
  246. var request = this._currentRequest =
  247. nativeProtocol.request(this._options, this._onNativeResponse);
  248. this._currentUrl = url.format(this._options);
  249. // Set up event handlers
  250. request._redirectable = this;
  251. for (var e = 0; e < events.length; e++) {
  252. request.on(events[e], eventHandlers[events[e]]);
  253. }
  254. // End a redirected request
  255. // (The first request must be ended explicitly with RedirectableRequest#end)
  256. if (this._isRedirect) {
  257. // Write the request entity and end.
  258. var i = 0;
  259. var self = this;
  260. var buffers = this._requestBodyBuffers;
  261. (function writeNext(error) {
  262. // Only write if this request has not been redirected yet
  263. /* istanbul ignore else */
  264. if (request === self._currentRequest) {
  265. // Report any write errors
  266. /* istanbul ignore if */
  267. if (error) {
  268. self.emit("error", error);
  269. }
  270. // Write the next buffer if there are still left
  271. else if (i < buffers.length) {
  272. var buffer = buffers[i++];
  273. /* istanbul ignore else */
  274. if (!request.finished) {
  275. request.write(buffer.data, buffer.encoding, writeNext);
  276. }
  277. }
  278. // End the request if `end` has been called on us
  279. else if (self._ended) {
  280. request.end();
  281. }
  282. }
  283. }());
  284. }
  285. };
  286. // Processes a response from the current native request
  287. RedirectableRequest.prototype._processResponse = function (response) {
  288. // Store the redirected response
  289. var statusCode = response.statusCode;
  290. if (this._options.trackRedirects) {
  291. this._redirects.push({
  292. url: this._currentUrl,
  293. headers: response.headers,
  294. statusCode: statusCode,
  295. });
  296. }
  297. // RFC7231§6.4: The 3xx (Redirection) class of status code indicates
  298. // that further action needs to be taken by the user agent in order to
  299. // fulfill the request. If a Location header field is provided,
  300. // the user agent MAY automatically redirect its request to the URI
  301. // referenced by the Location field value,
  302. // even if the specific status code is not understood.
  303. var location = response.headers.location;
  304. if (location && this._options.followRedirects !== false &&
  305. statusCode >= 300 && statusCode < 400) {
  306. // Abort the current request
  307. abortRequest(this._currentRequest);
  308. // Discard the remainder of the response to avoid waiting for data
  309. response.destroy();
  310. // RFC7231§6.4: A client SHOULD detect and intervene
  311. // in cyclical redirections (i.e., "infinite" redirection loops).
  312. if (++this._redirectCount > this._options.maxRedirects) {
  313. this.emit("error", new TooManyRedirectsError());
  314. return;
  315. }
  316. // RFC7231§6.4: Automatic redirection needs to done with
  317. // care for methods not known to be safe, […]
  318. // RFC7231§6.4.2–3: For historical reasons, a user agent MAY change
  319. // the request method from POST to GET for the subsequent request.
  320. if ((statusCode === 301 || statusCode === 302) && this._options.method === "POST" ||
  321. // RFC7231§6.4.4: The 303 (See Other) status code indicates that
  322. // the server is redirecting the user agent to a different resource […]
  323. // A user agent can perform a retrieval request targeting that URI
  324. // (a GET or HEAD request if using HTTP) […]
  325. (statusCode === 303) && !/^(?:GET|HEAD)$/.test(this._options.method)) {
  326. this._options.method = "GET";
  327. // Drop a possible entity and headers related to it
  328. this._requestBodyBuffers = [];
  329. removeMatchingHeaders(/^content-/i, this._options.headers);
  330. }
  331. // Drop the Host header, as the redirect might lead to a different host
  332. var currentHostHeader = removeMatchingHeaders(/^host$/i, this._options.headers);
  333. // If the redirect is relative, carry over the host of the last request
  334. var currentUrlParts = url.parse(this._currentUrl);
  335. var currentHost = currentHostHeader || currentUrlParts.host;
  336. var currentUrl = /^\w+:/.test(location) ? this._currentUrl :
  337. url.format(Object.assign(currentUrlParts, { host: currentHost }));
  338. // Determine the URL of the redirection
  339. var redirectUrl;
  340. try {
  341. redirectUrl = url.resolve(currentUrl, location);
  342. }
  343. catch (cause) {
  344. this.emit("error", new RedirectionError(cause));
  345. return;
  346. }
  347. // Create the redirected request
  348. debug("redirecting to", redirectUrl);
  349. this._isRedirect = true;
  350. var redirectUrlParts = url.parse(redirectUrl);
  351. Object.assign(this._options, redirectUrlParts);
  352. // Drop the Authorization header if redirecting to another domain
  353. if (!(redirectUrlParts.host === currentHost || isSubdomainOf(redirectUrlParts.host, currentHost))) {
  354. removeMatchingHeaders(/^authorization$/i, this._options.headers);
  355. }
  356. // Evaluate the beforeRedirect callback
  357. if (typeof this._options.beforeRedirect === "function") {
  358. var responseDetails = { headers: response.headers };
  359. try {
  360. this._options.beforeRedirect.call(null, this._options, responseDetails);
  361. }
  362. catch (err) {
  363. this.emit("error", err);
  364. return;
  365. }
  366. this._sanitizeOptions(this._options);
  367. }
  368. // Perform the redirected request
  369. try {
  370. this._performRequest();
  371. }
  372. catch (cause) {
  373. this.emit("error", new RedirectionError(cause));
  374. }
  375. }
  376. else {
  377. // The response is not a redirect; return it as-is
  378. response.responseUrl = this._currentUrl;
  379. response.redirects = this._redirects;
  380. this.emit("response", response);
  381. // Clean up
  382. this._requestBodyBuffers = [];
  383. }
  384. };
  385. // Wraps the key/value object of protocols with redirect functionality
  386. function wrap(protocols) {
  387. // Default settings
  388. var exports = {
  389. maxRedirects: 21,
  390. maxBodyLength: 10 * 1024 * 1024,
  391. };
  392. // Wrap each protocol
  393. var nativeProtocols = {};
  394. Object.keys(protocols).forEach(function (scheme) {
  395. var protocol = scheme + ":";
  396. var nativeProtocol = nativeProtocols[protocol] = protocols[scheme];
  397. var wrappedProtocol = exports[scheme] = Object.create(nativeProtocol);
  398. // Executes a request, following redirects
  399. function request(input, options, callback) {
  400. // Parse parameters
  401. if (typeof input === "string") {
  402. var urlStr = input;
  403. try {
  404. input = urlToOptions(new URL(urlStr));
  405. }
  406. catch (err) {
  407. /* istanbul ignore next */
  408. input = url.parse(urlStr);
  409. }
  410. }
  411. else if (URL && (input instanceof URL)) {
  412. input = urlToOptions(input);
  413. }
  414. else {
  415. callback = options;
  416. options = input;
  417. input = { protocol: protocol };
  418. }
  419. if (typeof options === "function") {
  420. callback = options;
  421. options = null;
  422. }
  423. // Set defaults
  424. options = Object.assign({
  425. maxRedirects: exports.maxRedirects,
  426. maxBodyLength: exports.maxBodyLength,
  427. }, input, options);
  428. options.nativeProtocols = nativeProtocols;
  429. assert.equal(options.protocol, protocol, "protocol mismatch");
  430. debug("options", options);
  431. return new RedirectableRequest(options, callback);
  432. }
  433. // Executes a GET request, following redirects
  434. function get(input, options, callback) {
  435. var wrappedRequest = wrappedProtocol.request(input, options, callback);
  436. wrappedRequest.end();
  437. return wrappedRequest;
  438. }
  439. // Expose the properties on the wrapped protocol
  440. Object.defineProperties(wrappedProtocol, {
  441. request: { value: request, configurable: true, enumerable: true, writable: true },
  442. get: { value: get, configurable: true, enumerable: true, writable: true },
  443. });
  444. });
  445. return exports;
  446. }
  447. /* istanbul ignore next */
  448. function noop() { /* empty */ }
  449. // from https://github.com/nodejs/node/blob/master/lib/internal/url.js
  450. function urlToOptions(urlObject) {
  451. var options = {
  452. protocol: urlObject.protocol,
  453. hostname: urlObject.hostname.startsWith("[") ?
  454. /* istanbul ignore next */
  455. urlObject.hostname.slice(1, -1) :
  456. urlObject.hostname,
  457. hash: urlObject.hash,
  458. search: urlObject.search,
  459. pathname: urlObject.pathname,
  460. path: urlObject.pathname + urlObject.search,
  461. href: urlObject.href,
  462. };
  463. if (urlObject.port !== "") {
  464. options.port = Number(urlObject.port);
  465. }
  466. return options;
  467. }
  468. function removeMatchingHeaders(regex, headers) {
  469. var lastValue;
  470. for (var header in headers) {
  471. if (regex.test(header)) {
  472. lastValue = headers[header].toString().trim();
  473. delete headers[header];
  474. }
  475. }
  476. return lastValue;
  477. }
  478. function createErrorType(code, defaultMessage) {
  479. function CustomError(cause) {
  480. Error.captureStackTrace(this, this.constructor);
  481. if (!cause) {
  482. this.message = defaultMessage;
  483. }
  484. else {
  485. this.message = defaultMessage + ": " + cause.message;
  486. this.cause = cause;
  487. }
  488. }
  489. CustomError.prototype = new Error();
  490. CustomError.prototype.constructor = CustomError;
  491. CustomError.prototype.name = "Error [" + code + "]";
  492. CustomError.prototype.code = code;
  493. return CustomError;
  494. }
  495. function abortRequest(request) {
  496. for (var e = 0; e < events.length; e++) {
  497. request.removeListener(events[e], eventHandlers[events[e]]);
  498. }
  499. request.on("error", noop);
  500. request.abort();
  501. }
  502. function isSubdomainOf(subdomain, domain) {
  503. const dot = subdomain.length - domain.length - 1;
  504. return dot > 0 && subdomain[dot] === "." && subdomain.endsWith(domain);
  505. }
  506. // Exports
  507. module.exports = wrap({ http: http, https: https });
  508. module.exports.wrap = wrap;