diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 7802f2f7058..e73cc637d07 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v2 with: node-version: ${{ matrix.node-version }} @@ -28,17 +28,25 @@ jobs: run: | npm install - - name: Unit test (no coverage) + - name: Test Tap run: | - npm test + npm run test:tap - - name: Unit test (coverage) + - name: Test Jest run: | - npm run coverage:ci + npm run test:jest + + - name: Test node-fetch + run: | + npm run test:node-fetch - - name: Test types + - name: Test Types run: | npm run test:typescript - - name: Coverage report + - name: Coverage + run: | + npm run coverage:ci + + - name: Coverage Report uses: codecov/codecov-action@v1 diff --git a/README.md b/README.md index 002b50dd9b9..12d08482acc 100644 --- a/README.md +++ b/README.md @@ -155,55 +155,19 @@ Calls `options.dispatch.connect(options)`. See [Dispatcher.connect](docs/api/Dispatcher.md#dispatcherconnect) for more details. -https://fetch.spec.whatwg.org/ - -### `undici.fetch([url, options]): Promise` +### `undici.fetch(input[, init]): Promise` Implements [fetch](https://fetch.spec.whatwg.org/). +https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch +https://fetch.spec.whatwg.org/#fetch-method + Only supported on Node 16+. This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant the Fetch Standard. We plan to ship breaking changes to this feature until it is out of experimental. -Arguments: - -* **url** `string | URL | object` -* **options** `RequestInit` - Returns: `Promise` -#### Parameter: `RequestInit` - -https://fetch.spec.whatwg.org/#request-class - -* **method** `string` -* **headers** `HeadersInit` -* **body** `BodyInit?` -* **referrer** *not supported* -* **referrerPolicy** *not supported* -* **mode** *not supported* -* **credentials** *not supported* -* **cache** *not supported* -* **redirect** `RequestRedirect` -* **integrity** *not supported* -* **keepalive** *not supported* -* **signal** `AbortSignal?` -* **window** `null` - -#### Parameter: `Response` - -https://fetch.spec.whatwg.org/#response-class - -* **type** `ResponseType` -* **url** `string` -* **redirected** `boolean` -* **status** `number` -* **ok** `boolean` -* **statusText** `string` -* **headers** `Headers` - -See [Dispatcher.fetch](docs/api/Dispatcher.md#dispatcherfetchoptions-callback) for more details. - ### `undici.upgrade([url, options]): Promise` Upgrade to a different protocol. See [MDN - HTTP - Protocol upgrade mechanism](https://developer.mozilla.org/en-US/docs/Web/HTTP/Protocol_upgrade_mechanism) for more details. diff --git a/docs/api/Dispatcher.md b/docs/api/Dispatcher.md index 36a1c26796c..3ee3d05dc2f 100644 --- a/docs/api/Dispatcher.md +++ b/docs/api/Dispatcher.md @@ -305,14 +305,6 @@ client.dispatch({ }) ``` -### `Dispatcher.fetch(options)` - -Implements [fetch](https://fetch.spec.whatwg.org/). - -Only supported on Node 16+. - -This is [experimental](https://nodejs.org/api/documentation.html#documentation_stability_index) and is not yet fully compliant the Fetch Standard. We plan to ship breaking changes to this feature until it is out of experimental. - ### `Dispatcher.pipeline(options, handler)` For easy use with [stream.pipeline](https://nodejs.org/api/stream.html#stream_stream_pipeline_source_transforms_destination_callback). The `handler` argument should return a `Readable` from which the result will be read. Usually it should just return the `body` argument unless some kind of transformation needs to be performed based on e.g. `headers` or `statusCode`. The `handler` should validate the response and save any required state. If there is an error, it should be thrown. The function returns a `Duplex` which writes to the request and reads from the response. diff --git a/index.js b/index.js index b4b90b30033..f218bd282c3 100644 --- a/index.js +++ b/index.js @@ -14,6 +14,8 @@ const MockAgent = require('./lib/mock/mock-agent') const MockPool = require('./lib/mock/mock-pool') const mockErrors = require('./lib/mock/mock-errors') +const nodeMajor = Number(process.versions.node.split('.')[0]) + Object.assign(Dispatcher.prototype, api) module.exports.Dispatcher = Dispatcher @@ -84,7 +86,17 @@ function makeDispatcher (fn) { module.exports.setGlobalDispatcher = setGlobalDispatcher module.exports.getGlobalDispatcher = getGlobalDispatcher -module.exports.fetch = makeDispatcher(api.fetch) +if (nodeMajor >= 16) { + const fetchImpl = require('./lib/fetch') + module.exports.fetch = async function fetch (resource, init) { + const dispatcher = getGlobalDispatcher() + return fetchImpl.call(dispatcher, resource, init) + } + module.exports.Headers = require('./lib/fetch/headers').Headers + module.exports.Response = require('./lib/fetch/response').Response + module.exports.Request = require('./lib/fetch/request').Request +} + module.exports.request = makeDispatcher(api.request) module.exports.stream = makeDispatcher(api.stream) module.exports.pipeline = makeDispatcher(api.pipeline) diff --git a/lib/api/api-fetch/body.js b/lib/api/api-fetch/body.js deleted file mode 100644 index a012ded0496..00000000000 --- a/lib/api/api-fetch/body.js +++ /dev/null @@ -1,66 +0,0 @@ -'use strict' - -const util = require('../../core/util') -const { Readable } = require('stream') - -let TransformStream - -// https://fetch.spec.whatwg.org/#concept-bodyinit-extract -function extractBody (body) { - // TODO: FormBody - - if (body == null) { - return [null, null] - } else if (body instanceof URLSearchParams) { - // spec says to run application/x-www-form-urlencoded on body.list - // this is implemented in Node.js as apart of an URLSearchParams instance toString method - // See: https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L490 - // and https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L1100 - return [{ - source: body.toString() - }, 'application/x-www-form-urlencoded;charset=UTF-8'] - } else if (typeof body === 'string') { - return [{ - source: body - }, 'text/plain;charset=UTF-8'] - } else if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) { - return [{ - source: body - }, null] - } else if (util.isBlob(body)) { - return [{ - source: body, - length: body.size - }, body.type || null] - } else if (util.isStream(body) || typeof body.pipeThrough === 'function') { - if (util.isDisturbed(body)) { - throw new TypeError('disturbed') - } - - let stream - if (util.isStream(body)) { - stream = Readable.toWeb(body) - } else { - if (body.locked) { - throw new TypeError('locked') - } - - if (!TransformStream) { - TransformStream = require('stream/web').TransformStream - } - - // https://streams.spec.whatwg.org/#readablestream-create-a-proxy - const identityTransform = new TransformStream() - body.pipeThrough(identityTransform) - stream = identityTransform - } - - return [{ - stream - }, null] - } else { - throw Error('Cannot extract Body from input: ', body) - } -} - -module.exports = { extractBody } diff --git a/lib/api/api-fetch/headers.js b/lib/api/api-fetch/headers.js deleted file mode 100644 index 967df35260b..00000000000 --- a/lib/api/api-fetch/headers.js +++ /dev/null @@ -1,240 +0,0 @@ -// https://github.com/Ethan-Arrowood/undici-fetch - -'use strict' - -const { types } = require('util') -const { validateHeaderName, validateHeaderValue } = require('http') -const { kHeadersList } = require('../../core/symbols') -const { InvalidHTTPTokenError, HTTPInvalidHeaderValueError, InvalidArgumentError, InvalidThisError } = require('../../core/errors') - -function binarySearch (arr, val) { - let low = 0 - let high = Math.floor(arr.length / 2) - - while (high > low) { - const mid = (high + low) >>> 1 - - if (val.localeCompare(arr[mid * 2]) > 0) { - low = mid + 1 - } else { - high = mid - } - } - - return low * 2 -} - -function normalizeAndValidateHeaderName (name) { - if (name === undefined) { - throw new InvalidHTTPTokenError(`Header name ${name}`) - } - const normalizedHeaderName = name.toLocaleLowerCase() - validateHeaderName(normalizedHeaderName) - return normalizedHeaderName -} - -function normalizeAndValidateHeaderValue (name, value) { - if (value === undefined) { - throw new HTTPInvalidHeaderValueError(value, name) - } - const normalizedHeaderValue = `${value}`.replace(/^[\n\t\r\x20]+|[\n\t\r\x20]+$/g, '') - validateHeaderValue(name, normalizedHeaderValue) - return normalizedHeaderValue -} - -function isHeaders (object) { - return kHeadersList in object -} - -function fill (headers, object) { - if (isHeaders(object)) { - // Object is instance of Headers - headers[kHeadersList] = Array.splice(object[kHeadersList]) - } else if (Array.isArray(object)) { - // Support both 1D and 2D arrays of header entries - if (Array.isArray(object[0])) { - // Array of arrays - for (let i = 0; i < object.length; i++) { - if (object[i].length !== 2) { - throw new InvalidArgumentError(`The argument 'init' is not of length 2. Received ${object[i]}`) - } - headers.append(object[i][0], object[i][1]) - } - } else if (typeof object[0] === 'string' || Buffer.isBuffer(object[0])) { - // Flat array of strings or Buffers - if (object.length % 2 !== 0) { - throw new InvalidArgumentError(`The argument 'init' is not even in length. Received ${object}`) - } - for (let i = 0; i < object.length; i += 2) { - headers.append( - object[i].toString('utf-8'), - object[i + 1].toString('utf-8') - ) - } - } else { - // All other array based entries - throw new InvalidArgumentError(`The argument 'init' is not a valid array entry. Received ${object}`) - } - } else if (!types.isBoxedPrimitive(object)) { - // Object of key/value entries - const entries = Object.entries(object) - for (let i = 0; i < entries.length; i++) { - headers.append(entries[i][0], entries[i][1]) - } - } -} - -function validateArgumentLength (found, expected) { - if (found !== expected) { - throw new TypeError(`${expected} ${expected > 1 ? 'arguments' : 'argument'} required, but only ${found} present`) - } -} - -class Headers { - constructor (init = {}) { - // validateObject allowArray = true - if (!Array.isArray(init) && typeof init !== 'object') { - throw new InvalidArgumentError('The argument \'init\' must be one of type Object or Array') - } - this[kHeadersList] = [] - fill(this, init) - } - - append (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - - validateArgumentLength(args.length, 2) - - const [name, value] = args - const normalizedName = normalizeAndValidateHeaderName(name) - const normalizedValue = normalizeAndValidateHeaderValue(name, value) - - const index = binarySearch(this[kHeadersList], normalizedName) - - if (this[kHeadersList][index] === normalizedName) { - this[kHeadersList][index + 1] += `, ${normalizedValue}` - } else { - this[kHeadersList].splice(index, 0, normalizedName, normalizedValue) - } - } - - delete (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - - validateArgumentLength(args.length, 1) - - const [name] = args - - const normalizedName = normalizeAndValidateHeaderName(name) - - const index = binarySearch(this[kHeadersList], normalizedName) - - if (this[kHeadersList][index] === normalizedName) { - this[kHeadersList].splice(index, 2) - } - } - - get (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - - validateArgumentLength(args.length, 1) - - const [name] = args - - const normalizedName = normalizeAndValidateHeaderName(name) - - const index = binarySearch(this[kHeadersList], normalizedName) - - if (this[kHeadersList][index] === normalizedName) { - return this[kHeadersList][index + 1] - } - - return null - } - - has (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - - validateArgumentLength(args.length, 1) - - const [name] = args - - const normalizedName = normalizeAndValidateHeaderName(name) - - const index = binarySearch(this[kHeadersList], normalizedName) - - return this[kHeadersList][index] === normalizedName - } - - set (...args) { - if (!isHeaders(this)) { - throw new InvalidThisError('Header') - } - - validateArgumentLength(args.length, 2) - - const [name, value] = args - - const normalizedName = normalizeAndValidateHeaderName(name) - const normalizedValue = normalizeAndValidateHeaderValue(name, value) - - const index = binarySearch(this[kHeadersList], normalizedName) - if (this[kHeadersList][index] === normalizedName) { - this[kHeadersList][index + 1] = normalizedValue - } else { - this[kHeadersList].splice(index, 0, normalizedName, normalizedValue) - } - } - - * keys () { - if (!isHeaders(this)) { - throw new InvalidThisError('Headers') - } - - for (let index = 0; index < this[kHeadersList].length; index += 2) { - yield this[kHeadersList][index] - } - } - - * values () { - if (!isHeaders(this)) { - throw new InvalidThisError('Headers') - } - - for (let index = 1; index < this[kHeadersList].length; index += 2) { - yield this[kHeadersList][index] - } - } - - * entries () { - if (!isHeaders(this)) { - throw new InvalidThisError('Headers') - } - - for (let index = 0; index < this[kHeadersList].length; index += 2) { - yield [this[kHeadersList][index], this[kHeadersList][index + 1]] - } - } - - forEach (callback, thisArg) { - if (!isHeaders(this)) { - throw new InvalidThisError('Headers') - } - - for (let index = 0; index < this[kHeadersList].length; index += 2) { - callback.call(thisArg, this[kHeadersList][index + 1], this[kHeadersList][index], this) - } - } -} - -Headers.prototype[Symbol.iterator] = Headers.prototype.entries - -module.exports = Headers diff --git a/lib/api/api-fetch/index.js b/lib/api/api-fetch/index.js deleted file mode 100644 index 77426ef957b..00000000000 --- a/lib/api/api-fetch/index.js +++ /dev/null @@ -1,251 +0,0 @@ -// https://github.com/Ethan-Arrowood/undici-fetch - -'use strict' - -const Headers = require('./headers') -const { kHeadersList } = require('../../core/symbols') -const { METHODS } = require('http') -const Response = require('./response') -const { - InvalidArgumentError, - NotSupportedError, - RequestAbortedError -} = require('../../core/errors') -const { addSignal, removeSignal } = require('../abort-signal') -const { extractBody } = require('./body') - -let ReadableStream - -class FetchHandler { - constructor (opts, callback) { - if (!opts || typeof opts !== 'object') { - throw new InvalidArgumentError('invalid opts') - } - - const { signal, method, opaque } = opts - - if (typeof callback !== 'function') { - throw new InvalidArgumentError('invalid callback') - } - - if (signal && typeof signal.on !== 'function' && typeof signal.addEventListener !== 'function') { - throw new InvalidArgumentError('signal must be an EventEmitter or EventTarget') - } - - if (method === 'CONNECT') { - throw new InvalidArgumentError('invalid method') - } - - this.opaque = opaque || null - this.callback = callback - this.controller = null - - this.abort = null - this.context = null - this.redirect = opts.redirect || 'follow' - this.url = new URL(opts.path, opts.origin) - - addSignal(this, signal) - } - - onConnect (abort, context) { - if (!this.callback) { - throw new RequestAbortedError() - } - - this.abort = abort - this.context = context - } - - onHeaders (statusCode, headers, resume) { - const { callback, abort, context } = this - - if (statusCode < 200) { - return - } - - headers = new Headers(headers) - - let response - if (headers.has('location')) { - if (this.redirect === 'manual') { - response = new Response({ - type: 'opaqueredirect', - url: this.url - }) - } else { - response = new Response({ - type: 'error', - url: this.url - }) - } - } else { - const self = this - if (!ReadableStream) { - ReadableStream = require('stream/web').ReadableStream - } - response = new Response({ - type: 'default', - url: this.url, - body: new ReadableStream({ - async start (controller) { - self.controller = controller - }, - async pull () { - resume() - }, - async cancel (reason) { - let err - if (reason instanceof Error) { - err = reason - } else if (typeof reason === 'string') { - err = new Error(reason) - } else { - err = new RequestAbortedError() - } - abort(err) - } - }, { highWaterMark: 16384 }), - statusCode, - headers, - context - }) - } - - this.callback = null - callback(null, response) - - return false - } - - onData (chunk) { - const { controller } = this - - // Copy the Buffer to detach it from Buffer pool. - // TODO: Is this required? - chunk = new Uint8Array(chunk) - - controller.enqueue(chunk) - - return controller.desiredSize > 0 - } - - onComplete () { - const { controller } = this - - removeSignal(this) - - controller.close() - } - - onError (err) { - const { controller, callback } = this - - removeSignal(this) - - if (callback) { - this.callback = null - callback(err) - } - - if (controller) { - this.controller = null - controller.error(err) - } - } -} - -async function fetch (opts) { - if (opts.referrer != null) { - // TODO: Implement? - throw new NotSupportedError() - } - - if (opts.referrerPolicy != null) { - // TODO: Implement? - throw new NotSupportedError() - } - - if (opts.mode != null) { - // TODO: Implement? - throw new NotSupportedError() - } - - if (opts.credentials != null) { - // TODO: Implement? - throw new NotSupportedError() - } - - if (opts.cache != null) { - // TODO: Implement? - throw new NotSupportedError() - } - - if (opts.redirect != null) { - // TODO: Validate - } else { - opts.redirect = 'follow' - } - - if (opts.method != null) { - opts.method = normalizeAndValidateRequestMethod(opts.method) - } else { - opts.method = 'GET' - } - - if (opts.integrity != null) { - // TODO: Implement? - throw new NotSupportedError() - } - - if (opts.keepalive != null) { - // TODO: Validate - } - - const headers = new Headers(opts.headers) - - if (!headers.has('accept')) { - headers.set('accept', '*/*') - } - - if (!headers.has('accept-language')) { - headers.set('accept-language', '*') - } - - const [body, contentType] = extractBody(opts.body) - - if (contentType && !headers.has('content-type')) { - headers.set('content-type', contentType) - } - - return new Promise((resolve, reject) => this.dispatch({ - path: opts.path, - origin: opts.origin, - method: opts.method, - body: body ? (body.stream || body.source) : null, - headers: headers[kHeadersList], - maxRedirections: opts.redirect === 'follow' ? 20 : 0 // https://fetch.spec.whatwg.org/#concept-http-redirect-fetch - }, new FetchHandler(opts, (err, res) => { - if (err) { - reject(err) - } else { - resolve(res) - } - }))) -} - -function normalizeAndValidateRequestMethod (method) { - if (typeof method !== 'string') { - throw TypeError(`Request method: ${method} must be type 'string'`) - } - - const normalizedMethod = method.toUpperCase() - - if (METHODS.indexOf(normalizedMethod) === -1) { - throw Error(`Normalized request method: ${normalizedMethod} must be one of ${METHODS.join(', ')}`) - } - - return normalizedMethod -} - -module.exports = fetch diff --git a/lib/api/api-fetch/response.js b/lib/api/api-fetch/response.js deleted file mode 100644 index db36102d658..00000000000 --- a/lib/api/api-fetch/response.js +++ /dev/null @@ -1,141 +0,0 @@ -'use strict' - -const Headers = require('./headers') -const { Blob } = require('buffer') -const { STATUS_CODES } = require('http') -const { - NotSupportedError -} = require('../../core/errors') -const util = require('../../core/util') - -const { - kType, - kStatus, - kStatusText, - kUrlList, - kHeaders, - kBody -} = require('./symbols') - -class Response { - constructor ({ - type, - url, - body, - statusCode, - headers, - context - }) { - this[kType] = type || 'default' - this[kStatus] = statusCode || 0 - this[kStatusText] = STATUS_CODES[statusCode] || '' - this[kUrlList] = Array.isArray(url) ? url : (url ? [url] : []) - this[kHeaders] = headers || new Headers() - this[kBody] = body || null - - if (context && context.history) { - this[kUrlList].push(...context.history) - } - } - - get type () { - return this[kType] - } - - get url () { - const length = this[kUrlList].length - return length === 0 ? '' : this[kUrlList][length - 1].toString() - } - - get redirected () { - return this[kUrlList].length > 1 - } - - get status () { - return this[kStatus] - } - - get ok () { - return this[kStatus] >= 200 && this[kStatus] <= 299 - } - - get statusText () { - return this[kStatusText] - } - - get headers () { - return this[kHeaders] - } - - async blob () { - const chunks = [] - if (this.body) { - if (this.bodyUsed || this.body.locked) { - throw new TypeError('unusable') - } - - for await (const chunk of this.body) { - chunks.push(chunk) - } - } - - return new Blob(chunks, { type: this.headers.get('Content-Type') || '' }) - } - - async arrayBuffer () { - const blob = await this.blob() - return await blob.arrayBuffer() - } - - async text () { - const blob = await this.blob() - return await blob.text() - } - - async json () { - return JSON.parse(await this.text()) - } - - async formData () { - // TODO: Implement. - throw new NotSupportedError('formData') - } - - get body () { - return this[kBody] - } - - get bodyUsed () { - return util.isDisturbed(this.body) - } - - clone () { - let body = null - - if (this[kBody]) { - if (util.isDisturbed(this[kBody])) { - throw new TypeError('disturbed') - } - - if (this[kBody].locked) { - throw new TypeError('locked') - } - - // https://fetch.spec.whatwg.org/#concept-body-clone - const [out1, out2] = this[kBody].tee() - - this[kBody] = out1 - body = out2 - } - - return new Response({ - type: this[kType], - statusCode: this[kStatus], - url: this[kUrlList], - headers: this[kHeaders], - body - }) - } -} - -module.exports = Response diff --git a/lib/api/api-fetch/symbols.js b/lib/api/api-fetch/symbols.js deleted file mode 100644 index 7b45f11a9e0..00000000000 --- a/lib/api/api-fetch/symbols.js +++ /dev/null @@ -1,11 +0,0 @@ -'use strict' - -module.exports = { - kType: Symbol('type'), - kStatus: Symbol('status'), - kStatusText: Symbol('status text'), - kUrlList: Symbol('url list'), - kHeaders: Symbol('headers'), - kBody: Symbol('body'), - kBodyUsed: Symbol('body used') -} diff --git a/lib/api/index.js b/lib/api/index.js index 6e957a8bf73..8983a5e746f 100644 --- a/lib/api/index.js +++ b/lib/api/index.js @@ -1,13 +1,7 @@ 'use strict' -const nodeMajor = Number(process.versions.node.split('.')[0]) - module.exports.request = require('./api-request') module.exports.stream = require('./api-stream') module.exports.pipeline = require('./api-pipeline') module.exports.upgrade = require('./api-upgrade') module.exports.connect = require('./api-connect') - -if (nodeMajor >= 16) { - module.exports.fetch = require('./api-fetch') -} diff --git a/lib/api/readable.js b/lib/api/readable.js index cd97b1e5b43..d4efeec95b9 100644 --- a/lib/api/readable.js +++ b/lib/api/readable.js @@ -130,7 +130,7 @@ module.exports = class BodyReadable extends Readable { // https://fetch.spec.whatwg.org/#dom-body-body get body () { if (!this[kBody]) { - this[kBody] = Readable.toWeb(this) + this[kBody] = util.toWeb(this) if (this[kConsume]) { // TODO: Is this the best way to force a lock? this[kBody].getReader() // Ensure stream is locked. diff --git a/lib/client.js b/lib/client.js index d042784969b..8d3ce51be39 100644 --- a/lib/client.js +++ b/lib/client.js @@ -406,7 +406,10 @@ const llhttp = new WebAssembly.Instance(mod, { return 0 }, wasm_on_status: (p, at, len) => { - return 0 + assert.strictEqual(currentParser.ptr, p) + const start = at - currentBufferPtr + const end = start + len + return currentParser.onStatus(currentBufferRef.slice(start, end)) || 0 }, wasm_on_message_begin: (p) => { assert.strictEqual(currentParser.ptr, p) @@ -463,6 +466,7 @@ class Parser { this.timeoutValue = null this.timeoutType = null this.statusCode = null + this.statusText = '' this.upgrade = false this.headers = [] this.headersSize = 0 @@ -653,6 +657,10 @@ class Parser { this.paused = false } + onStatus (buf) { + this.statusText = buf.toString() + } + onMessageBegin () { const { socket, client } = this @@ -722,6 +730,7 @@ class Parser { assert(request.upgrade || request.method === 'CONNECT') this.statusCode = null + this.statusText = '' this.shouldKeepAlive = null assert(this.headers.length % 2 === 0) @@ -755,7 +764,7 @@ class Parser { } onHeadersComplete (statusCode, upgrade, shouldKeepAlive) { - const { client, socket, headers } = this + const { client, socket, headers, statusText } = this /* istanbul ignore next: difficult to make a test case for */ if (socket.destroyed) { @@ -842,7 +851,7 @@ class Parser { } try { - if (request.onHeaders(statusCode, headers, this.resume) === false) { + if (request.onHeaders(statusCode, headers, this.resume, statusText) === false) { return constants.ERROR.PAUSED } } catch (err) { @@ -909,6 +918,7 @@ class Parser { assert(statusCode >= 100) this.statusCode = null + this.statusText = '' this.bytesRead = 0 this.contentLength = '' this.trailer = '' diff --git a/lib/core/errors.js b/lib/core/errors.js index 8e9b13b46c5..7f7ac4a958a 100644 --- a/lib/core/errors.js +++ b/lib/core/errors.js @@ -1,5 +1,13 @@ 'use strict' +class AbortError extends Error { + constructor () { + super('The operation was aborted') + this.code = 'ABORT_ERR' + this.name = 'AbortError' + } +} + class UndiciError extends Error { constructor (message) { super(message) @@ -196,6 +204,7 @@ class InvalidThisError extends TypeError { } module.exports = { + AbortError, HTTPParserError, UndiciError, HeadersTimeoutError, diff --git a/lib/core/request.js b/lib/core/request.js index 33552ae83dd..5e3278c60ac 100644 --- a/lib/core/request.js +++ b/lib/core/request.js @@ -52,6 +52,9 @@ class Request { this.body = null } else if (util.isStream(body)) { this.body = body + } else if (body instanceof DataView) { + // TODO: Why is DataView special? + this.body = body.buffer.byteLength ? Buffer.from(body.buffer) : null } else if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) { this.body = body.byteLength ? Buffer.from(body) : null } else if (util.isBuffer(body)) { @@ -130,11 +133,11 @@ class Request { return this[kHandler].onConnect(abort) } - onHeaders (statusCode, headers, resume) { + onHeaders (statusCode, headers, resume, statusText) { assert(!this.aborted) assert(!this.completed) - return this[kHandler].onHeaders(statusCode, headers, resume) + return this[kHandler].onHeaders(statusCode, headers, resume, statusText) } onData (chunk) { diff --git a/lib/core/util.js b/lib/core/util.js index 845247c321b..57d614f9457 100644 --- a/lib/core/util.js +++ b/lib/core/util.js @@ -239,7 +239,11 @@ function isDisturbed (body) { )) } +const kEnumerableProperty = Object.create(null) +kEnumerableProperty.enumerable = true + module.exports = { + kEnumerableProperty, nop, isDisturbed, isAborted, diff --git a/lib/fetch/LICENSE b/lib/fetch/LICENSE new file mode 100644 index 00000000000..294350045bb --- /dev/null +++ b/lib/fetch/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Ethan Arrowood + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/fetch/body.js b/lib/fetch/body.js new file mode 100644 index 00000000000..7e26a801f98 --- /dev/null +++ b/lib/fetch/body.js @@ -0,0 +1,277 @@ +'use strict' + +const util = require('../core/util') +const { toWebReadable } = require('./util') +const { kState } = require('./symbols') +const { Blob } = require('buffer') +const { Readable } = require('stream') +const { NotSupportedError } = require('../core/errors') +const { ReadableStream } = require('stream/web') +const { kBodyUsed } = require('../core/symbols') +const assert = require('assert') +const nodeUtil = require('util') + +// https://fetch.spec.whatwg.org/#concept-bodyinit-extract +function extractBody (object, keepalive = false) { + // 1. Let stream be object if object is a ReadableStream object. + // Otherwise, let stream be a new ReadableStream, and set up stream. + let stream = object + let controller + if (!stream || !(stream instanceof ReadableStream)) { + stream = new ReadableStream({ + async start (c) { + controller = c + }, + async pull () {}, + async cancel (reason) {} + }) + } + + // 2. Let action be null. + let action = null + + // 3. Let source be null. + let source = null + + // 4. Let length be null. + let length = null + + // 5. Let Content-Type be null. + let contentType = null + + // 6. Switch on object: + if (object == null) { + // Note: The IDL processor cannot handle this situation. See + // https://crbug.com/335871. + } else if (object instanceof URLSearchParams) { + // URLSearchParams + + // spec says to run application/x-www-form-urlencoded on body.list + // this is implemented in Node.js as apart of an URLSearchParams instance toString method + // See: https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L490 + // and https://github.com/nodejs/node/blob/e46c680bf2b211bbd52cf959ca17ee98c7f657f5/lib/internal/url.js#L1100 + + // Set source to the result of running the application/x-www-form-urlencoded serializer with object’s list. + source = object.toString() + + // Set Content-Type to `application/x-www-form-urlencoded;charset=UTF-8`. + contentType = 'application/x-www-form-urlencoded;charset=UTF-8' + } else if (object instanceof ArrayBuffer || ArrayBuffer.isView(object)) { + // BufferSource + + if (object instanceof DataView) { + // TODO: Blob doesn't seem to work with DataView? + object = object.buffer + } + + // Set source to a copy of the bytes held by object. + source = new Uint8Array(object) + } else if (object instanceof Blob) { + // Blob + + // Set action to this step: read object. + action = async (onNext, onError, onComplete) => { + try { + onNext(await object.arrayBuffer()) + onComplete() + } catch (err) { + onError(err) + } + } + + // Set source to object. + source = object + + // Set length to object’s size. + length = object.size + + // If object’s type attribute is not the empty byte sequence, set Content-Type to its value. + if (object.type) { + contentType = object.type + } + } else if (object instanceof ReadableStream || object instanceof Readable) { + // ReadableStream + + // If keepalive is true, then throw a TypeError. + if (keepalive) { + throw new TypeError('keepalive') + } + + // If object is disturbed or locked, then throw a TypeError. + if (util.isDisturbed(stream) || stream.locked) { + throw new TypeError( + 'Response body object should not be disturbed or locked' + ) + } + + if (util.isStream(object)) { + stream = toWebReadable(object) + } + } else if (typeof object === 'string') { + // scalar value string + // TODO: How to check for "scalar value string"? + source = object + contentType = 'text/plain;charset=UTF-8' + } else { + // TODO: byte sequence? + // TODO: FormData? + // TODO: else? + source = String(object) + contentType = 'text/plain;charset=UTF-8' + } + + // 7. If source is a byte sequence, then set action to a + // step that returns source and length to source’s length. + // TODO: What is a "byte sequence?" + if (typeof source === 'string' || util.isBuffer(source)) { + length = Buffer.byteLength(source) + } + + // 8. If action is non-null, then run these steps in in parallel: + if (action !== null) { + // Run action. + action( + (bytes) => { + // Whenever one or more bytes are available and stream is not errored, + // enqueue a Uint8Array wrapping an ArrayBuffer containing the available + // bytes into stream. + if (!/state: 'errored'/.test(nodeUtil.inspect(stream))) { + controller.enqueue(new Uint8Array(bytes)) + } + }, + (err) => { + // TODO: Spec doesn't say anything about this? + controller.error(err) + }, + () => { + // When running action is done, close stream. + controller.close() + } + ) + } else if (controller) { + // TODO: Spec doesn't say anything about this? + controller.enqueue(source) + controller.close() + } + + // 9. Let body be a body whose stream is stream, source is source, + // and length is length. + const body = { stream, source, length } + + // 10. Return body and Content-Type. + return [body, contentType] +} + +// https://fetch.spec.whatwg.org/#bodyinit-safely-extract +function safelyExtractBody (object, keepalive = false) { + // To safely extract a body and a `Content-Type` value from + // a byte sequence or BodyInit object object, run these steps: + + // 1. If object is a ReadableStream object, then: + if (object instanceof ReadableStream) { + // Assert: object is neither disturbed nor locked. + assert(!util.isDisturbed(object), 'disturbed') + assert(!object.locked, 'locked') + } + + // 2. Return the results of extracting object. + return extractBody(object, keepalive) +} + +function cloneBody (body) { + // To clone a body body, run these steps: + + // https://fetch.spec.whatwg.org/#concept-body-clone + + // 1. Let « out1, out2 » be the result of teeing body’s stream. + const [out1, out2] = body.stream.tee() + + // 2. Set body’s stream to out1. + body.stream = out1 + + // 3. Return a body whose stream is out2 and other members are copied from body. + return { + stream: out2, + length: body.length, + source: body.source + } +} + +const methods = { + async blob () { + const chunks = [] + + if (this[kState].body) { + const stream = this[kState].body.stream + + if (util.isDisturbed(stream)) { + throw new TypeError('disturbed') + } + + if (stream.locked) { + throw new TypeError('locked') + } + + // NOTE: stream.isDisturbed hasn't landed on Node 16.x yet. + stream[kBodyUsed] = true + + for await (const chunk of stream) { + chunks.push(chunk) + } + } + + return new Blob(chunks, { type: this.headers.get('Content-Type') || '' }) + }, + + async arrayBuffer () { + const blob = await this.blob() + return await blob.arrayBuffer() + }, + + async text () { + const blob = await this.blob() + return await blob.text() + }, + + async json () { + return JSON.parse(await this.text()) + }, + + async formData () { + // TODO: Implement. + throw new NotSupportedError('formData') + } +} + +const properties = { + body: { + enumerable: true, + get () { + return this[kState].body ? this[kState].body.stream : null + } + }, + bodyUsed: { + enumerable: true, + get () { + return this[kState].body && util.isDisturbed(this[kState].body.stream) + } + } +} + +function mixinBody (prototype) { + Object.assign(prototype, methods) + Object.defineProperties(prototype, properties) +} + +function isBodyReadable (body) { + // This is a hack! + return body && /state: 'readable'/.test(nodeUtil.inspect(body.stream)) +} + +module.exports = { + extractBody, + isBodyReadable, + safelyExtractBody, + cloneBody, + mixinBody +} diff --git a/lib/fetch/constants.js b/lib/fetch/constants.js new file mode 100644 index 00000000000..f40145ab978 --- /dev/null +++ b/lib/fetch/constants.js @@ -0,0 +1,87 @@ +'use strict' + +const forbiddenHeaderNames = [ + 'accept-charset', + 'accept-encoding', + 'access-control-request-headers', + 'access-control-request-method', + 'connection', + 'content-length', + 'cookie', + 'cookie2', + 'date', + 'dnt', + 'expect', + 'host', + 'keep-alive', + 'origin', + 'referer', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', + 'via' +] + +const corsSafeListedMethods = ['GET', 'HEAD', 'POST'] + +const nullBodyStatus = [101, 204, 205, 304] + +const redirectStatus = [301, 302, 303, 307, 308] + +const referrerPolicy = [ + '', + 'no-referrer', + 'no-referrer-when-downgrade', + 'same-origin', + 'origin', + 'strict-origin', + 'origin-when-cross-origin', + 'strict-origin-when-cross-origin', + 'unsafe-url' +] + +const requestRedirect = ['follow', 'manual', 'error'] + +const safeMethods = ['GET', 'HEAD', 'OPTIONS', 'TRACE'] + +const requestMode = ['navigate', 'same-origin', 'no-cors', 'cors'] + +const requestCredentials = ['omit', 'same-origin', 'include'] + +const requestCache = [ + 'default', + 'no-store', + 'reload', + 'no-cache', + 'force-cache', + 'only-if-cached' +] + +const forbiddenResponseHeaderNames = ['set-cookie', 'set-cookie2'] + +const requestBodyHeader = [ + 'content-encoding', + 'content-language', + 'content-location', + 'content-type' +] + +// http://fetch.spec.whatwg.org/#forbidden-method +const forbiddenMethods = ['CONNECT', 'TRACE', 'TRACK'] + +module.exports = { + forbiddenResponseHeaderNames, + forbiddenMethods, + requestBodyHeader, + referrerPolicy, + requestRedirect, + requestMode, + requestCredentials, + requestCache, + forbiddenHeaderNames, + redirectStatus, + corsSafeListedMethods, + nullBodyStatus, + safeMethods +} diff --git a/lib/fetch/headers.js b/lib/fetch/headers.js new file mode 100644 index 00000000000..834473b9069 --- /dev/null +++ b/lib/fetch/headers.js @@ -0,0 +1,368 @@ +// https://github.com/Ethan-Arrowood/undici-fetch + +'use strict' + +const { validateHeaderName, validateHeaderValue } = require('http') +const { kHeadersList } = require('../core/symbols') +const { kGuard } = require('./symbols') +const { kEnumerableProperty } = require('../core/util') +const { + InvalidHTTPTokenError, + HTTPInvalidHeaderValueError, + InvalidThisError +} = require('../core/errors') +const { + forbiddenHeaderNames, + forbiddenResponseHeaderNames +} = require('./constants') + +function binarySearch (arr, val) { + let low = 0 + let high = Math.floor(arr.length / 2) + + while (high > low) { + const mid = (high + low) >>> 1 + + if (val.localeCompare(arr[mid * 2]) > 0) { + low = mid + 1 + } else { + high = mid + } + } + + return low * 2 +} + +function normalizeAndValidateHeaderName (name) { + if (name === undefined) { + throw new InvalidHTTPTokenError(`Header name ${name}`) + } + const normalizedHeaderName = name.toLocaleLowerCase() + validateHeaderName(normalizedHeaderName) + return normalizedHeaderName +} + +function normalizeAndValidateHeaderValue (name, value) { + if (value === undefined) { + throw new HTTPInvalidHeaderValueError(value, name) + } + const normalizedHeaderValue = `${value}`.replace( + /^[\n\t\r\x20]+|[\n\t\r\x20]+$/g, + '' + ) + validateHeaderValue(name, normalizedHeaderValue) + return normalizedHeaderValue +} + +function isHeaders (object) { + return kHeadersList in object +} + +function fill (headers, object) { + // To fill a Headers object headers with a given object object, run these steps: + + if (object[Symbol.iterator]) { + // 1. If object is a sequence, then for each header in object: + // TODO: How to check if sequence? + for (let header of object) { + // 1. If header does not contain exactly two items, then throw a TypeError. + if (!header[Symbol.iterator]) { + // TODO: Spec doesn't define what to do here? + throw new TypeError() + } + + if (typeof header === 'string') { + // TODO: Spec doesn't define what to do here? + throw new TypeError() + } + + if (!Array.isArray(header)) { + header = [...header] + } + + if (header.length !== 2) { + throw new TypeError() + } + + // 2. Append header’s first item/header’s second item to headers. + headers.append(header[0], header[1]) + } + } else if (object && typeof object === 'object') { + // Otherwise, object is a record, then for each key → value in object, + // append key/value to headers. + // TODO: How to check if record? + for (const header of Object.entries(object)) { + headers.append(header[0], header[1]) + } + } else { + // TODO: Spec doesn't define what to do here? + throw TypeError() + } +} + +function validateArgumentLength (found, expected) { + if (found !== expected) { + throw new TypeError( + `${expected} ${ + expected > 1 ? 'arguments' : 'argument' + } required, but only ${found} present` + ) + } +} + +class HeadersList extends Array { + append (...args) { + validateArgumentLength(args.length, 2) + + const [name, value] = args + const normalizedName = normalizeAndValidateHeaderName(name) + const normalizedValue = normalizeAndValidateHeaderValue(name, value) + + const index = binarySearch(this, normalizedName) + + if (this[index] === normalizedName) { + this[index + 1] += `, ${normalizedValue}` + } else { + this.splice(index, 0, normalizedName, normalizedValue) + } + } + + delete (...args) { + validateArgumentLength(args.length, 1) + + const [name] = args + + const normalizedName = normalizeAndValidateHeaderName(name) + + const index = binarySearch(this, normalizedName) + + if (this[index] === normalizedName) { + this.splice(index, 2) + } + } + + get (...args) { + validateArgumentLength(args.length, 1) + + const [name] = args + + const normalizedName = normalizeAndValidateHeaderName(name) + + const index = binarySearch(this, normalizedName) + + if (this[index] === normalizedName) { + return this[index + 1] + } + + return null + } + + has (...args) { + validateArgumentLength(args.length, 1) + + const [name] = args + + const normalizedName = normalizeAndValidateHeaderName(name) + + const index = binarySearch(this, normalizedName) + + return this[index] === normalizedName + } + + set (...args) { + validateArgumentLength(args.length, 2) + + const [name, value] = args + + const normalizedName = normalizeAndValidateHeaderName(name) + const normalizedValue = normalizeAndValidateHeaderValue(name, value) + + const index = binarySearch(this, normalizedName) + if (this[index] === normalizedName) { + this[index + 1] = normalizedValue + } else { + this.splice(index, 0, normalizedName, normalizedValue) + } + } +} + +class Headers { + constructor (init = {}) { + this[kHeadersList] = new HeadersList() + + // The new Headers(init) constructor steps are: + + // 1. Set this’s guard to "none". + this[kGuard] = 'none' + + // 2. If init is given, then fill this with init. + fill(this[kHeadersList], init) + } + + get [Symbol.toStringTag] () { + return this.constructor.name + } + + toString () { + return Object.prototype.toString.call(this) + } + + append (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + const normalizedName = normalizeAndValidateHeaderName(args[0]) + + if (this[kGuard] === 'immutable') { + throw new TypeError('immutable') + } else if ( + this[kGuard] === 'request' && + forbiddenHeaderNames.includes(normalizedName) + ) { + return + } else if (this[kGuard] === 'request-no-cors') { + // TODO + } else if ( + this[kGuard] === 'response' && + forbiddenResponseHeaderNames.includes(normalizedName) + ) { + return + } + + return this[kHeadersList].append(...args) + } + + delete (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + const normalizedName = normalizeAndValidateHeaderName(args[0]) + + if (this[kGuard] === 'immutable') { + throw new TypeError('immutable') + } else if ( + this[kGuard] === 'request' && + forbiddenHeaderNames.includes(normalizedName) + ) { + return + } else if (this[kGuard] === 'request-no-cors') { + // TODO + } else if ( + this[kGuard] === 'response' && + forbiddenResponseHeaderNames.includes(normalizedName) + ) { + return + } + + return this[kHeadersList].delete(...args) + } + + get (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + return this[kHeadersList].get(...args) + } + + has (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + return this[kHeadersList].has(...args) + } + + set (...args) { + if (!isHeaders(this)) { + throw new InvalidThisError('Header') + } + + const normalizedName = normalizeAndValidateHeaderName(args[0]) + + if (this[kGuard] === 'immutable') { + throw new TypeError('immutable') + } else if ( + this[kGuard] === 'request' && + forbiddenHeaderNames.includes(normalizedName) + ) { + return + } else if (this[kGuard] === 'request-no-cors') { + // TODO + } else if ( + this[kGuard] === 'response' && + forbiddenResponseHeaderNames.includes(normalizedName) + ) { + return + } + + return this[kHeadersList].set(...args) + } + + * keys () { + if (!isHeaders(this)) { + throw new InvalidThisError('Headers') + } + + for (let index = 0; index < this[kHeadersList].length; index += 2) { + yield this[kHeadersList][index] + } + } + + * values () { + if (!isHeaders(this)) { + throw new InvalidThisError('Headers') + } + + for (let index = 1; index < this[kHeadersList].length; index += 2) { + yield this[kHeadersList][index] + } + } + + * entries () { + if (!isHeaders(this)) { + throw new InvalidThisError('Headers') + } + + for (let index = 0; index < this[kHeadersList].length; index += 2) { + yield [this[kHeadersList][index], this[kHeadersList][index + 1]] + } + } + + forEach (callback, thisArg) { + if (!isHeaders(this)) { + throw new InvalidThisError('Headers') + } + + for (let index = 0; index < this[kHeadersList].length; index += 2) { + callback.call( + thisArg, + this[kHeadersList][index + 1], + this[kHeadersList][index], + this + ) + } + } + + [Symbol.for('nodejs.util.inspect.custom')] () { + return this[kHeadersList] + } +} + +Headers.prototype[Symbol.iterator] = Headers.prototype.entries + +Object.defineProperties(Headers.prototype, { + append: kEnumerableProperty, + delete: kEnumerableProperty, + get: kEnumerableProperty, + has: kEnumerableProperty, + set: kEnumerableProperty, + keys: kEnumerableProperty, + values: kEnumerableProperty, + entries: kEnumerableProperty, + forEach: kEnumerableProperty +}) + +module.exports = { fill, Headers, HeadersList } diff --git a/lib/fetch/index.js b/lib/fetch/index.js new file mode 100644 index 00000000000..895b6206af3 --- /dev/null +++ b/lib/fetch/index.js @@ -0,0 +1,1445 @@ +// https://github.com/Ethan-Arrowood/undici-fetch + +'use strict' + +const { + Response, + makeNetworkError, + filterResponse, + makeResponse +} = require('./response') +const { Headers } = require('./headers') +const { Request, makeRequest } = require('./request') +const { + requestBadPort, + responseLocationURL, + requestCurrentURL +} = require('./util') +const { kState, kHeaders, kGuard } = require('./symbols') +const { AbortError } = require('../core/errors') +const assert = require('assert') +const { safelyExtractBody, isBodyReadable } = require('./body') +const { + redirectStatus, + nullBodyStatus, + safeMethods, + requestBodyHeader +} = require('./constants') +const { kHeadersList } = require('../core/symbols') +const { ReadableStream } = require('stream/web') +const { performance } = require('perf_hooks') +const EE = require('events') + +// https://fetch.spec.whatwg.org/#fetch-method +async function fetch (resource, init) { + const context = Object.assign(new EE(), { + dispatcher: this, + controller: null, + terminated: false, + abort: null, // "connection" abort + terminate ({ aborted } = {}) { + if (this.terminated) { + return + } + + if (context.abort) { + context.abort() + } + + this.terminated = { aborted } + + this.emit('terminated') + } + }) + + // 1. Let p be a new promise. + const p = createDeferredPromise() + + // 2. Let requestObject be the result of invoking the initial value of + // Request as constructor with input and init as arguments. If this throws + // an exception, reject p with it and return p. + const requestObject = new Request(resource, init) + + // 3. Let request be requestObject’s request. + const request = requestObject[kState] + + // 4. If requestObject’s signal’s aborted flag is set, then: + if (requestObject.signal.aborted) { + // 1. Abort fetch with p, request, and null. + abortFetch.call(context, p, request, null) + + // 2. Return p. + return p.promise + } + + // 5. Let globalObject be request’s client’s global object. + // TODO + + // 6. If globalObject is a ServiceWorkerGlobalScope object, then set + // request’s service-workers mode to "none". + // TODO + + // 7. Let responseObject be null. + let responseObject = null + + // 8. Let relevantRealm be this’s relevant Realm. + // TODO + + // 9. Let locallyAborted be false. + let locallyAborted = false + + // 10. Add the following abort steps to requestObject’s signal: + requestObject.signal.addEventListener( + 'abort', + () => { + // 1. Set locallyAborted to true. + locallyAborted = true + + // 2. Abort fetch with p, request, and responseObject. + abortFetch.call(context, p, request, responseObject) + + // 3. Terminate the ongoing fetch with the aborted flag set. + context.terminate({ aborted: true }) + }, + { once: true } + ) + + // 11. Let handleFetchDone given response response be to finalize and + // report timing with response, globalObject, and "fetch". + const handleFetchDone = (response) => + finalizeAndReportTiming(response, 'fetch') + + // 12. Fetch request with processResponseDone set to handleFetchDone, + // and processResponse given response being these substeps: + const processResponse = (response) => { + // 1. If locallyAborted is true, terminate these substeps. + if (locallyAborted) { + return + } + + // 2. If response’s aborted flag is set, then abort fetch with p, + // request, and responseObject, and terminate these substeps. + if (response.aborted) { + abortFetch.call(context, p, request, responseObject) + return + } + + // 3. If response is a network error, then reject p with a TypeError + // and terminate these substeps. + if (response.status === 0) { + const error = new TypeError('fetch failed') + error.cause = response.error + p.reject(error) + return + } + + // 4. Set responseObject to the result of creating a Response object, + // given response, "immutable", and relevantRealm. + // TODO: relevantRealm + responseObject = new Response() + responseObject[kState] = response + responseObject[kHeaders][kHeadersList] = response.headersList + responseObject[kHeaders][kGuard] = 'immutable' + + // 5. Resolve p with responseObject. + p.resolve(responseObject) + } + + fetching.call(context, { + request, + processResponseDone: handleFetchDone, + processResponse + }) + + // 13. Return p. + return p.promise +} + +function finalizeAndReportTiming (response, initiatorType = 'other') { + // TODO +} + +// https://fetch.spec.whatwg.org/#abort-fetch +function abortFetch (p, request, responseObject) { + const context = this + + // 1. Let error be an "AbortError" DOMException. + const error = new AbortError() + + // 2. Reject promise with error. + p.reject(error) + + // 3. If request’s body is not null and is readable, then cancel request’s + // body with error. + if (request.body !== null && isBodyReadable(request.body)) { + request.body.stream.cancel(error) + } + + // 4. If responseObject is null, then return. + if (responseObject == null) { + return + } + + // 5. Let response be responseObject’s response. + const response = responseObject[kState] + + // 6. If response’s body is not null and is readable, then error response’s + // body with error. + if (response.body != null && isBodyReadable(response.body)) { + context.controller.error(error) + } +} + +// https://fetch.spec.whatwg.org/#fetching +function fetching ({ request, processResponse, processResponseDone }) { + // 1. Let taskDestination be null. + // TODO + + // 2. Let crossOriginIsolatedCapability be false. + // TODO + + // 3. If request’s client is non-null, then: + // TODO + + // 4. If useParallelQueue is true, then set taskDestination to the result of + // starting a new parallel queue. + // TODO + + // 5. Let timingInfo be a new fetch timing info whose start time and + // post-redirect start time are the coarsened shared current time given + // crossOriginIsolatedCapability. + // TODO: Coarsened shared current time given crossOriginIsolatedCapability? + const currenTime = performance.now() + const timingInfo = { + startTime: currenTime, + redirectStartTime: 0, + redirectEndTime: 0, + postRedirectStartTime: currenTime, + finalServiceWorkerStartTime: 0, + finalNetworkResponseStartTime: 0, + finalNetworkRequestStartTime: 0, + endTime: 0, + encodedBodySize: 0, + decodedBodySize: 0, + finalConnectionTimingInfo: null + } + + // 6. Let fetchParams be a new fetch params whose request is request, timing + // info is timingInfo, process request body is processRequestBody, + // process request end-of-body is processRequestEndOfBody, process response + // is processResponse, process response end-of-body is + // processResponseEndOfBody, process response done is processResponseDone, + // task destination is taskDestination, and cross-origin isolated capability + // is crossOriginIsolatedCapability. + const fetchParams = { + request, + timingInfo, + processRequestBody: null, + processRequestEndOfBody: null, + processResponse, + processResponseEndOfBody: null, + processResponseDone + } + + // 7. If request’s body is a byte sequence, then set request’s body to the + // first return value of safely extracting request’s body. + // TODO + + // 8. If request’s window is "client", then set request’s window to request’s + // client, if request’s client’s global object is a Window object; otherwise + // "no-window". + // TODO + + // 9. If request’s origin is "client", then set request’s origin to request’s + // client’s origin. + if (request.origin === 'client') { + // TODO: What is correct here? + request.origin = requestCurrentURL(request).origin + } + + // 10. If request’s policy container is "client", then: + if (request.policyContainer === 'client') { + // TODO + } + + // 11. If request’s header list does not contain `Accept`, then: + if (!request.headersList.has('accept')) { + // 1. Let value be `*/*`. + const value = '*/*' + + // 2. A user agent should set value to the first matching statement, if + // any, switching on request’s destination: + // "document" + // "frame" + // "iframe" + // `text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8` + // "image" + // `image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5` + // "style" + // `text/css,*/*;q=0.1` + // TODO + + // 3. Append `Accept`/value to request’s header list. + request.headersList.append('accept', value) + + // 12. If request’s header list does not contain `Accept-Language`, then + // user agents should append `Accept-Language`/an appropriate value to + // request’s header list. + if (!request.headersList.has('accept-language')) { + request.headersList.append('accept-language', '*') + } + } + + // 13. If request’s priority is null, then use request’s initiator and + // destination appropriately in setting request’s priority to a + // user-agent-defined object. + // TODO + + // 14. If request is a subresource request, then: + // TODO + + // 15. Run main fetch given fetchParams. + mainFetch.call(this, fetchParams) +} + +// https://fetch.spec.whatwg.org/#concept-main-fetch +async function mainFetch (fetchParams, recursive = false) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let response be null. + let response = null + + // 3. If request’s local-URLs-only flag is set and request’s current URL is + // not local, then set response to a network error. + if ( + request.localURLsOnly && + !/^(about|blob|data):/.test(requestCurrentURL(request).protocol) + ) { + return makeNetworkError('local URLs only') + } + + // 4. Run report Content Security Policy violations for request. + // TODO + + // 5. Upgrade request to a potentially trustworthy URL, if appropriate. + // TODO + + // 6. If should request be blocked due to a bad port, should fetching request + // be blocked as mixed content, or should request be blocked by Content + // Security Policy returns blocked, then set response to a network error. + if (requestBadPort(request) === 'blocked') { + return makeNetworkError('bad port') + } + // TODO: should fetching request be blocked as mixed content? + // TODO: should request be blocked by Content Security Policy? + + // 7. If request’s referrer policy is the empty string, then set request’s + // referrer policy to request’s policy container’s referrer policy. + if (request.referrerPolicy === '') { + // TODO + } + + // 8. If request’s referrer is not "no-referrer", then set request’s + // referrer to the result of invoking determine request’s referrer. + if (request.referrer !== 'no-referrer') { + // TODO + } + + // 9. Set request’s current URL’s scheme to "https" if all of the following + // conditions are true: + // - request’s current URL’s scheme is "http" + // - request’s current URL’s host is a domain + // - Matching request’s current URL’s host per Known HSTS Host Domain Name + // Matching results in either a superdomain match with an asserted + // includeSubDomains directive or a congruent match (with or without an + // asserted includeSubDomains directive). [HSTS] + // TODO + + // 10. If recursive is false, then run the remaining steps in parallel. + // TODO + + // 11. If response is null, then set response to the result of running + // the steps corresponding to the first matching statement: + // TODO + response = await (async () => { + // - request’s current URL’s origin is same origin with request’s origin, + // and request’s response tainting is "basic" + // - request’s current URL’s scheme is "data" + // - request’s mode is "navigate" or "websocket" + // 1. Set request’s response tainting to "basic". + // 2. Return the result of running scheme fetch given fetchParams. + // TODO + + // request’s mode is "same-origin" + if (request.mode === 'same-origin') { + // 1. Return a network error. + return makeNetworkError('request mode cannot be "same-origin"') + } + + // request’s mode is "no-cors" + if (request.mode === 'no-cors') { + // 1. If request’s redirect mode is not "follow", then return a network + // error. + if (request.redirect !== 'follow') { + return makeNetworkError( + 'redirect cmode cannot be "follow" for "no-cors" request' + ) + } + + // 2. Set request’s response tainting to "opaque". + request.responseTainting = 'opaque' + + // 3. Let noCorsResponse be the result of running scheme fetch given + // fetchParams. + // TODO + + // 4. If noCorsResponse is a filtered response or the CORB check with + // request and noCorsResponse returns allowed, then return noCorsResponse. + // TODO + + // 5. Return a new response whose status is noCorsResponse’s status. + // TODO + } + + // request’s current URL’s scheme is not an HTTP(S) scheme + if (!/^https?:/.test(requestCurrentURL(request).protocol)) { + // Return a network error. + return makeNetworkError('URL scheme must be a HTTP(S) scheme') + } + + // - request’s use-CORS-preflight flag is set + // - request’s unsafe-request flag is set and either request’s method is + // not a CORS-safelisted method or CORS-unsafe request-header names with + // request’s header list is not empty + // 1. Set request’s response tainting to "cors". + // 2. Let corsWithPreflightResponse be the result of running HTTP fetch + // given fetchParams and true. + // 3. If corsWithPreflightResponse is a network error, then clear cache + // entries using request. + // 4. Return corsWithPreflightResponse. + // TODO + + // Otherwise + // 1. Set request’s response tainting to "cors". + request.responseTainting = 'cors' + + // 2. Return the result of running HTTP fetch given fetchParams. + return await httpFetch + .call(this, fetchParams) + .catch((err) => makeNetworkError(err)) + })() + + // 12. If recursive is true, then return response. + if (recursive) { + return response + } + + // 13. If response is not a network error and response is not a filtered + // response, then: + if (response.status !== 0 && !response.internalResponse) { + // If request’s response tainting is "cors", then: + if (request.responseTainting === 'cors') { + // 1. Let headerNames be the result of extracting header list values + // given `Access-Control-Expose-Headers` and response’s header list. + // TODO + // 2. If request’s credentials mode is not "include" and headerNames + // contains `*`, then set response’s CORS-exposed header-name list to + // all unique header names in response’s header list. + // TODO + // 3. Otherwise, if headerNames is not null or failure, then set + // response’s CORS-exposed header-name list to headerNames. + // TODO + } + + // Set response to the following filtered response with response as its + // internal response, depending on request’s response tainting: + if (request.responseTainting === 'basic') { + response = filterResponse(response, 'basic') + } else if (request.responseTainting === 'cors') { + response = filterResponse(response, 'cors') + } else if (request.responseTainting === 'opaque') { + response = filterResponse(response, 'opaque') + } else { + assert(false) + } + } + + // 14. Let internalResponse be response, if response is a network error, + // and response’s internal response otherwise. + let internalResponse = + response.status === 0 ? response : response.internalResponse + + // 15. If internalResponse’s URL list is empty, then set it to a clone of + // request’s URL list. + if (internalResponse.urlList.length === 0) { + internalResponse.urlList.push(...request.urlList) + } + + // 16. If request’s timing allow failed flag is unset, then set + // internalResponse’s timing allow passed flag. + if (!request.timingAllowFailed) { + response.timingAllowPassed = true + } + + // 17. If response is not a network error and any of the following returns + // blocked + // - should internalResponse to request be blocked as mixed content + // - should internalResponse to request be blocked by Content Security Policy + // - should internalResponse to request be blocked due to its MIME type + // - should internalResponse to request be blocked due to nosniff + // TODO + + // 18. If response’s type is "opaque", internalResponse’s status is 206, + // internalResponse’s range-requested flag is set, and request’s header + // list does not contain `Range`, then set response and internalResponse + // to a network error. + if ( + response.type === 'opaque' && + internalResponse.status === 206 && + internalResponse.rangeRequested && + !request.headers.has('range') + ) { + response = internalResponse = makeNetworkError() + } + + // 19. If response is not a network error and either request’s method is + // `HEAD` or `CONNECT`, or internalResponse’s status is a null body status, + // set internalResponse’s body to null and disregard any enqueuing toward + // it (if any). + if ( + response.status !== 0 && + (request.method === 'HEAD' || + request.method === 'CONNECT' || + nullBodyStatus.includes(internalResponse.status)) + ) { + internalResponse.body = null + if (context.controller) { + context.controller.error(new AbortError()) + } + } + + // 20. If request’s integrity metadata is not the empty string, then: + if (request.integrity) { + // 1. Let processBodyError be this step: run fetch finale given fetchParams + // and a network error. + const processBodyError = (reason) => + fetchFinale(fetchParams, makeNetworkError(reason)) + + // 2. If request’s response tainting is "opaque", response is a network + // error, or response’s body is null, then run processBodyError and abort + // these steps. + if (request.responseTainting === 'opaque' && response.status === 0) { + processBodyError(response.error) + return + } + + // 3. Let processBody given bytes be these steps: + const processBody = (bytes) => { + // 1. If bytes do not match request’s integrity metadata, + // then run processBodyError and abort these steps. [SRI] + // TODO + + // 2. Set response’s body to the first return value of safely + // extracting bytes. + response.body = safelyExtractBody(bytes)[0] + + // 3. Run fetch finale given fetchParams and response. + fetchFinale(fetchParams, response) + } + + // 4. Fully read response’s body given processBody and processBodyError. + try { + processBody(await response.arrayBuffer()) + } catch (err) { + processBodyError(err) + } + } else { + // 21. Otherwise, run fetch finale given fetchParams and response. + fetchFinale(fetchParams, response) + } +} + +// https://fetch.spec.whatwg.org/#finalize-response +function finalizeResponse (fetchParams, response) { + // 1. Set fetchParams’s request’s done flag. + fetchParams.request.done = true + + // 2, If fetchParams’s process response done is not null, then queue a fetch + // task to run fetchParams’s process response done given response, with + // fetchParams’s task destination. + if (fetchParams.processResponseDone) { + fetchParams.processResponseDone(response) + } +} + +// https://fetch.spec.whatwg.org/#fetch-finale +function fetchFinale (fetchParams, response) { + // 1. If fetchParams’s process response is non-null, + // then queue a fetch task to run fetchParams’s process response + // given response, with fetchParams’s task destination. + if (fetchParams.processResponse !== null) { + fetchParams.processResponse(response) + } + + // 2. If fetchParams’s process response end-of-body is non-null, then:. + // TODO + // 1. Let processBody given nullOrBytes be this step: run fetchParams’s + // process response end-of-body given response and nullOrBytes.on. + // TODO + // 2. Let processBodyError be this step: run fetchParams’s process + // response end-of-body given response and failure.on. + // TODO + // 3. If response’s body is null, then queue a fetch task to run + // processBody given null, with fetchParams’s task destination.on. + // TODO + // 4. Otherwise, fully read response’s body given processBody, + // processBodyError, and fetchParams’s task destination.on. + // TODO +} + +// https://fetch.spec.whatwg.org/#http-fetch +async function httpFetch (fetchParams) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let response be null. + let response = null + + // 3. Let actualResponse be null. + let actualResponse = null + + // 4. Let timingInfo be fetchParams’s timing info. + const timingInfo = fetchParams.timingInfo + + // 5. If request’s service-workers mode is "all", then: + // TODO + + // 6. If response is null, then: + if (response === null) { + // 1. If makeCORSPreflight is true and one of these conditions is true: + // TODO + + // 2. If request’s redirect mode is "follow", then set request’s + // service-workers mode to "none". + if (request.redirect === 'follow') { + request.serviceWorkers = 'none' + } + + // 3. Set response and actualResponse to the result of running + // HTTP-network-or-cache fetch given fetchParams. + actualResponse = response = await httpNetworkOrCacheFetch.call( + this, + fetchParams + ) + + // 4. If request’s response tainting is "cors" and a CORS check + // for request and response returns failure, then return a network error. + // TODO + + // 5. If the TAO check for request and response returns failure, then set + // request’s timing allow failed flag. + // TODO + } + + // 7. If either request’s response tainting or response’s type + // is "opaque", and the cross-origin resource policy check with + // request’s origin, request’s client, request’s destination, + // and actualResponse returns blocked, then return a network error. + // TODO + + // 8. If actualResponse’s status is a redirect status, then: + if (redirectStatus.includes(actualResponse.status)) { + // 1. If actualResponse’s status is not 303, request’s body is not null, + // and the connection uses HTTP/2, then user agents may, and are even + // encouraged to, transmit an RST_STREAM frame. + // See, https://github.com/whatwg/fetch/issues/1288 + if (context.abort) { + context.abort() + } + + // 2. Switch on request’s redirect mode: + if (request.redirect === 'error') { + // Set response to a network error. + response = makeNetworkError() + } else if (request.redirect === 'manual') { + // Set response to an opaque-redirect filtered response whose internal + // response is actualResponse. + response = filterResponse(response, 'opaqueredirect') + } else if (request.redirect === 'follow') { + // Set response to the result of running HTTP-redirect fetch given + // fetchParams and response. + response = await httpRedirectFetch.call(this, fetchParams, response) + } else { + assert(false) + } + } + + // 9. Set response’s timing info to timingInfo. + response.timingInfo = timingInfo + + // 10. Return response. + return response +} + +// https://fetch.spec.whatwg.org/#http-redirect-fetch +async function httpRedirectFetch (fetchParams, response) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let actualResponse be response, if response is not a filtered response, + // and response’s internal response otherwise. + const actualResponse = response.internalResponse + ? response.internalResponse + : response + + // 3. Let locationURL be actualResponse’s location URL given request’s current + // URL’s fragment. + let locationURL + + try { + locationURL = responseLocationURL( + actualResponse, + requestCurrentURL(request).hash + ) + + // 4. If locationURL is null, then return response. + if (locationURL == null) { + return response + } + } catch (err) { + // 5. If locationURL is failure, then return a network error. + return makeNetworkError(err) + } + + // 6. If locationURL’s scheme is not an HTTP(S) scheme, then return a network + // error. + if (!/^https?:/.test(locationURL)) { + return makeNetworkError('URL scheme must be a HTTP(S) scheme') + } + + // 7. If request’s redirect count is twenty, return a network error. + if (request.redirectCount === 20) { + return makeNetworkError('redirect count exceeded') + } + + // 8. Increase request’s redirect count by one. + request.redirectCount += 1 + + // 9. If request’s mode is "cors", locationURL includes credentials, and + // request’s origin is not same origin with locationURL’s origin, then return + // a network error. + if (request.mode === 'cors' && request.origin !== locationURL.origin) { + return makeNetworkError('cross origin not allowed for request mode "cors"') + } + + // 10. If request’s response tainting is "cors" and locationURL includes + // credentials, then return a network error. + if ( + request.responseTainting === 'cors' && + (locationURL.username || locationURL.password) + ) { + return makeNetworkError( + 'URL cannot contain credentials for request mode "cors"' + ) + } + + // 11. If actualResponse’s status is not 303, request’s body is non-null, + // and request’s body’s source is null, then return a network error. + if ( + actualResponse.status !== 303 && + request.body !== null && + request.body.source == null + ) { + return makeNetworkError() + } + + // 12. If locationURL’s origin is not same origin with request’s current URL’s + // origin and request’s origin is not same origin with request’s current + // URL’s origin, then set request’s tainted origin flag. + if ( + locationURL.origin !== requestCurrentURL(request).origin && + request.origin !== locationURL.origin + ) { + request.taintedOrigin = true + } + + // 13. If one of the following is true + // - actualResponse’s status is 301 or 302 and request’s method is `POST` + // - actualResponse’s status is 303 and request’s method is not `GET` or `HEAD` + if ( + ([301, 302].includes(actualResponse.status) && request.method === 'POST') || + (actualResponse.status === 303 && + !['GET', 'HEADER'].includes(request.method)) + ) { + // then: + // 1. Set request’s method to `GET` and request’s body to null. + request.method = 'GET' + request.body = null + + // 2. For each headerName of request-body-header name, delete headerName from + // request’s header list. + for (const headerName of requestBodyHeader) { + request.headersList.delete(headerName) + } + } + + // 14. If request’s body is non-null, then set request’s body to the first return + // value of safely extracting request’s body’s source. + if (request.body !== null) { + assert(request.body.source) + request.body = safelyExtractBody(request.body.source)[0] + } + + // 15. Let timingInfo be fetchParams’s timing info. + const timingInfo = fetchParams.timingInfo + + // 16. Set timingInfo’s redirect end time and post-redirect start time to the + // coarsened shared current time given fetchParams’s cross-origin isolated capability. + // TODO: given fetchParams’s cross-origin isolated capability? + timingInfo.redirectEndTime = timingInfo.postRedirectStartTime = + performance.now() + + // 17. If timingInfo’s redirect start time is 0, then set timingInfo’s redirect start + // time to timingInfo’s start time. + if (timingInfo.redirectStartTime === 0) { + timingInfo.redirectStartTime = timingInfo.startTime + } + + // 18. Append locationURL to request’s URL list. + request.urlList.push(locationURL) + + // 19. Invoke set request’s referrer policy on redirect on request and actualResponse. + // TODO + + // 20. Return the result of running main fetch given fetchParams and true. + return mainFetch.call(this, fetchParams, true) +} + +// https://fetch.spec.whatwg.org/#http-network-or-cache-fetch +async function httpNetworkOrCacheFetch ( + fetchParams, + isAuthenticationFetch = false, + isNewConnectionFetch = false +) { + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let httpFetchParams be null. + let httpFetchParams = null + + // 3. Let httpRequest be null. + let httpRequest = null + + // 4. Let response be null. + let response = null + + // 5. Let storedResponse be null. + // TODO + + // 6. Let httpCache be null. + const httpCache = null + + // 7. Let the revalidatingFlag be unset. + const revalidatingFlag = false + + // 8. Run these steps, but abort when the ongoing fetch is terminated: + + // 1. If request’s window is "no-window" and request’s redirect mode is + // "error", then set httpFetchParams to fetchParams and httpRequest to + // request. + if (request.window === 'no-window' && request.redirect === 'error') { + httpFetchParams = fetchParams + httpRequest = request + } else { + // Otherwise: + + // 1. Set httpRequest to a clone of request. + httpRequest = makeRequest(request) + + // 2. Set httpFetchParams to a copy of fetchParams. + httpFetchParams = { ...fetchParams } + + // 3. Set httpFetchParams’s request to httpRequest. + httpFetchParams.request = httpRequest + } + + // 3. Let includeCredentials be true if one of + const includeCredentials = + request.credentials === 'include' || + (request.credentials === 'same-origin' && + request.responseTainting === 'basic') + + // 4. Let contentLength be httpRequest’s body’s length, if httpRequest’s + // body is non-null; otherwise null. + const contentLength = httpRequest.body ? httpRequest.body.length : null + + // 5. Let contentLengthHeaderValue be null. + let contentLengthHeaderValue = null + + // 6. If httpRequest’s body is null and httpRequest’s method is `POST` or + // `PUT`, then set contentLengthHeaderValue to `0`. + if ( + httpRequest.body == null && + ['POST', 'PUT'].includes(httpRequest.method) + ) { + contentLengthHeaderValue = '0' + } + + // 7. If contentLength is non-null, then set contentLengthHeaderValue to + // contentLength, serialized and isomorphic encoded. + if (contentLength !== null) { + // TODO: isomorphic encoded + contentLengthHeaderValue = String(contentLength) + } + + // 8. If contentLengthHeaderValue is non-null, then append + // `Content-Length`/contentLengthHeaderValue to httpRequest’s header + // list. + if (contentLengthHeaderValue !== null) { + httpRequest.headersList.append('content-length', contentLengthHeaderValue) + } + + // 9. If contentLength is non-null and httpRequest’s keepalive is true, + // then: + if (contentLength !== null && httpRequest.keepalive) { + // NOTE: keepalive is a noop outside of browser context. + } + + // 10 .If httpRequest’s referrer is a URL, then append + // `Referer`/httpRequest’s referrer, serialized and isomorphic encoded, + // to httpRequest’s header list. + if (httpRequest.referrer instanceof URL) { + // TODO: isomorphic encoded + httpRequest.headersList.append('referer', httpRequest.referrer.href) + } + + // 11. Append a request `Origin` header for httpRequest. + // TODO + + // 12. Append the Fetch metadata headers for httpRequest. [FETCH-METADATA] + + // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-dest-header + // TODO + + // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-mode-header + { + // 1. Assert: r’s url is a potentially trustworthy URL. + // TODO + + // 2. Let header be a Structured Header whose value is a token. + let header = null + + // 3. Set header’s value to r’s mode. + header = request.mode + + // 4. Set a structured field value `Sec-Fetch-Mode`/header in r’s header list. + httpRequest.headersList.append('sec-fetch-mode', header) + } + + // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-site-header + // TODO + + // https://w3c.github.io/webappsec-fetch-metadata/#sec-fetch-user-header + // TODO + + // 13. If httpRequest’s header list does not contain `User-Agent`, then + // user agents should append `User-Agent`/default `User-Agent` value to + // httpRequest’s header list. + if (!httpRequest.headersList.has('user-agent')) { + httpRequest.headersList.append('user-agent', 'undici') + } + + // 14. If httpRequest’s cache mode is "default" and httpRequest’s header + // list contains `If-Modified-Since`, `If-None-Match`, + // `If-Unmodified-Since`, `If-Match`, or `If-Range`, then set + // httpRequest’s cache mode to "no-store". + if ( + httpRequest.cache === 'default' && + (httpRequest.headersList.has('if-modified-since') || + httpRequest.headersList.has('if-none-match') || + httpRequest.headersList.has('if-unmodified-since') || + httpRequest.headersList.has('if-match') || + httpRequest.headersList.has('if-range')) + ) { + httpRequest.cache = 'no-store' + } + + // 15. If httpRequest’s cache mode is "no-cache", httpRequest’s prevent + // no-cache cache-control header modification flag is unset, and + // httpRequest’s header list does not contain `Cache-Control`, then append + // `Cache-Control`/`max-age=0` to httpRequest’s header list. + if ( + httpRequest.cache === 'no-cache' && + !httpRequest.preventNoCacheCacheControlHeaderModification && + !httpRequest.headersList.has('cache-control') + ) { + httpRequest.headersList.append('cache-control', 'max-age=0') + } + + // 16. If httpRequest’s cache mode is "no-store" or "reload", then: + if (httpRequest.cache === 'no-store' || httpRequest.cache === 'reload') { + // 1. If httpRequest’s header list does not contain `Pragma`, then append + // `Pragma`/`no-cache` to httpRequest’s header list. + if (!httpRequest.headersList.has('pragma')) { + httpRequest.headersList.append('pragma', 'no-cache') + } + + // 2. If httpRequest’s header list does not contain `Cache-Control`, + // then append `Cache-Control`/`no-cache` to httpRequest’s header list. + if (!httpRequest.headersList.has('cache-control')) { + httpRequest.headersList.append('cache-control', 'no-cache') + } + } + + // 17. If httpRequest’s header list contains `Range`, then append + // `Accept-Encoding`/`identity` to httpRequest’s header list. + if (httpRequest.headersList.has('range')) { + httpRequest.headersList.append('accept-encoding', 'identity') + } + + // 18. Modify httpRequest’s header list per HTTP. Do not append a given + // header if httpRequest’s header list contains that header’s name. + // TODO: https://github.com/whatwg/fetch/issues/1285#issuecomment-896560129 + + // 19. If includeCredentials is true, then: + if (includeCredentials) { + // 1. If the user agent is not configured to block cookies for httpRequest + // (see section 7 of [COOKIES]), then: + // TODO + // 2. If httpRequest’s header list does not contain `Authorization`, then: + // TODO + } + + // 20. If there’s a proxy-authentication entry, use it as appropriate. + // TODO + + // 21. Set httpCache to the result of determining the HTTP cache + // partition, given httpRequest. + // TODO + + // 22. If httpCache is null, then set httpRequest’s cache mode to + // "no-store". + if (httpCache == null) { + httpRequest.cache = 'no-store' + } + + // 23. If httpRequest’s cache mode is neither "no-store" nor "reload", + // then: + if (httpRequest.mode !== 'no-store' && httpRequest.mode !== 'reload') { + // TODO + } + + // 9. If aborted, then: + // TODO + + // 10. If response is null, then: + if (response == null) { + // 1. If httpRequest’s cache mode is "only-if-cached", then return a + // network error. + if (httpRequest.mode === 'only-if-cached') { + return makeNetworkError('only if cached') + } + + // 2. Let forwardResponse be the result of running HTTP-network fetch + // given httpFetchParams, includeCredentials, and isNewConnectionFetch. + const forwardResponse = await httpNetworkFetch.call( + this, + httpFetchParams, + includeCredentials, + isNewConnectionFetch + ) + + // 3. If httpRequest’s method is unsafe and forwardResponse’s status is + // in the range 200 to 399, inclusive, invalidate appropriate stored + // responses in httpCache, as per the "Invalidation" chapter of HTTP + // Caching, and set storedResponse to null. [HTTP-CACHING] + if ( + !safeMethods.includes(httpRequest.method) && + forwardResponse.status >= 200 && + forwardResponse.status <= 399 + ) { + // TODO + } + + // 4. If the revalidatingFlag is set and forwardResponse’s status is 304, + // then: + if (revalidatingFlag && forwardResponse.status === 304) { + // TODO + } + + // 5. If response is null, then: + if (response == null) { + // 1. Set response to forwardResponse. + response = forwardResponse + + // 2. Store httpRequest and forwardResponse in httpCache, as per the + // "Storing Responses in Caches" chapter of HTTP Caching. [HTTP-CACHING] + // TODO + } + } + + // 11. Set response’s URL list to a clone of httpRequest’s URL list. + response.urlList = [...httpRequest.urlList] + + // 12. If httpRequest’s header list contains `Range`, then set response’s + // range-requested flag. + if (httpRequest.headersList.has('range')) { + response.rangeRequested = true + } + + // 13. If response’s status is 401, httpRequest’s response tainting is not + // "cors", includeCredentials is true, and request’s window is an environment + // settings object, then: + // TODO + + // 14. If response’s status is 407, then: + if (response.status === 407) { + // 1. If request’s window is "no-window", then return a network error. + if (request.window === 'no-window') { + return makeNetworkError() + } + + // 2. ??? + + // 3. If the ongoing fetch is terminated, then: + if (context.terminated) { + // 1. Let aborted be the termination’s aborted flag. + // 2. If aborted is set, then return an aborted network error. + // 3. Return a network error. + return makeNetworkError( + context.terminated.aborted ? new AbortError() : null + ) + } + + // 4. Prompt the end user as appropriate in request’s window and store + // the result as a proxy-authentication entry. [HTTP-AUTH] + // TODO: Invoke some kind of callback? + + // 5. Set response to the result of running HTTP-network-or-cache fetch given + // fetchParams. + // TODO + return makeNetworkError('proxy authentication required') + } + + // 15. If all of the following are true + if ( + // response’s status is 421 + response.status === 421 && + // isNewConnectionFetch is false + !isNewConnectionFetch && + // request’s body is null, or request’s body is non-null and request’s body’s source is non-null + (request.body == null || request.body.source !== null) + ) { + // 1. If the ongoing fetch is terminated, then: + if (context.terminated) { + // 1. Let aborted be the termination’s aborted flag. + const aborted = context.terminated.aborted + + // 2. If aborted is set, then return an aborted network error. + const reason = aborted ? new AbortError() : new Error('terminated') + + // 3. Return a network error. + return makeNetworkError(reason) + } + + response = await httpNetworkOrCacheFetch.call( + this, + fetchParams, + isAuthenticationFetch, + true + ) + } + + // 16. If isAuthenticationFetch is true, then create an authentication entry + // for request and the given realm. + if (isAuthenticationFetch) { + // TODO + } + + // 17. Return response. + return response +} + +// https://fetch.spec.whatwg.org/#http-network-fetch +function httpNetworkFetch ( + fetchParams, + includeCredentials = false, + forceNewConnection = false +) { + return new Promise((resolve) => { + const context = this + + // 1. Let request be fetchParams’s request. + const request = fetchParams.request + + // 2. Let response be null. + let response = null + + // 3. Let timingInfo be fetchParams’s timing info. + // TODO + + // 4. Let httpCache be the result of determining the HTTP cache partition, + // given request. + // TODO + const httpCache = null + + // 5. If httpCache is null, then set request’s cache mode to "no-store". + if (httpCache == null) { + request.cache = 'no-store' + } + + // 6. Let networkPartitionKey be the result of determining the network + // partition key given request. + // TODO + + // 7. Switch on request’s mode: + // TODO + + // 8. Run these steps, but abort when the ongoing fetch is terminated: + // TODO: When do we cleanup this listener? + context.on('terminated', onRequestAborted) + + // 5. Set response to the result of making an HTTP request over connection + // using request with the following caveats: + + // Follow the relevant requirements from HTTP. [HTTP] [HTTP-SEMANTICS] + // [HTTP-COND] [HTTP-CACHING] [HTTP-AUTH] + + // If request’s body is non-null, and request’s body’s source is null, + // then the user agent may have a buffer of up to 64 kibibytes and store + // a part of request’s body in that buffer. If the user agent reads from + // request’s body beyond that buffer’s size and the user agent needs to + // resend request, then instead return a network error. + // TODO + + // Set timingInfo’s final network-response start time to the coarsened + // shared current time given fetchParams’s cross-origin isolated capability, + // immediately after the user agent’s HTTP parser receives the first byte + // of the response (e.g., frame header bytes for HTTP/2 or response status + // line for HTTP/1.x). + // TODO + + // Wait until all the headers are transmitted. + + // Any responses whose status is in the range 100 to 199, inclusive, + // and is not 101, are to be ignored, except for the purposes of setting + // timingInfo’s final network-response start time above. + + // If request’s header list contains `Transfer-Encoding`/`chunked` and + // response is transferred via HTTP/1.0 or older, then return a network + // error. + + // If the HTTP request results in a TLS client certificate dialog, then: + + // 1. If request’s window is an environment settings object, make the + // dialog available in request’s window. + + // 2. Otherwise, return a network error. + + // To transmit request’s body body, run these steps: + const body = (async function * () { + try { + // 1. If body is null and fetchParams’s process request end-of-body is + // non-null, then queue a fetch task given fetchParams’s process request + // end-of-body and fetchParams’s task destination. + if (request.body === null) { + fetchParams.processEndOfBody?.() + return + } + + // 2. Otherwise, if body is non-null: + + // 1. Let processBodyChunk given bytes be these steps: + for await (const bytes of request.body.stream) { + // 1. If the ongoing fetch is terminated, then abort these steps. + if (context.terminated) { + return + } + + // 2. Run this step in parallel: transmit bytes. + yield bytes + + // 3. If fetchParams’s process request body is non-null, then run + // fetchParams’s process request body given bytes’s length. + fetchParams.processRequestBody?.(bytes.byteLength) + } + + // 2. Let processEndOfBody be these steps: + + // 1. If the ongoing fetch is terminated, then abort these steps. + if (context.terminated) { + return + } + + // 2. If fetchParams’s process request end-of-body is non-null, + // then run fetchParams’s process request end-of-body. + fetchParams.processRequestEndOfBody?.() + } catch (e) { + // 3. Let processBodyError given e be these steps: + + // 1. If the ongoing fetch is terminated, then abort these steps. + if (context.terminated) { + return + } + + // 2. If e is an "AbortError" DOMException, then terminate the ongoing fetch with the aborted flag set. + if (e.name === 'AbortError') { + context.terminate({ aborted: true }) + return + } + + // 3. Otherwise, terminate the ongoing fetch. + context.terminate() + } + })() + + function onRequestAborted () { + // 1. Let aborted be the termination’s aborted flag. + const aborted = context.terminated.aborted + + // 2. If connection uses HTTP/2, then transmit an RST_STREAM frame. + if (context.abort) { + context.abort() + } + + // 3. If aborted is set, then return an aborted network error. + if (aborted) { + resolve(makeNetworkError(new AbortError())) + } + + // 4. Return a network error. + resolve(makeNetworkError()) + } + + // TODO... + + // TODO: forceNewConnection + + // NOTE: This is just a hack. + if (forceNewConnection && context.controller) { + context.abort() + } + + assert(!context.controller) + + const url = requestCurrentURL(request) + context.dispatcher.dispatch( + { + path: url.pathname + url.search, + origin: url.origin, + method: request.method, + body, + headers: request.headersList, + maxRedirections: 0 + }, + { + onConnect (abort) { + if (context.terminated) { + abort(new AbortError()) + } else { + context.abort = (err) => abort(err ?? new AbortError()) + } + }, + + onHeaders (status, headersList, resume, statusText) { + if (status < 200) { + return + } + + const headers = new Headers() + for (let n = 0; n < headersList.length; n += 2) { + headers.append( + headersList[n + 0].toString(), + headersList[n + 1].toString() + ) + } + + const stream = + status === 204 + ? null + : new ReadableStream( + { + async start (controller) { + context.controller = controller + }, + async pull () { + resume() + }, + async cancel (reason) { + let err + if (reason instanceof Error) { + err = reason + } else if (typeof reason === 'string') { + err = new Error(reason) + } else { + err = new AbortError() + } + + context.abort(err) + } + }, + { highWaterMark: 16384 } + ) + + response = makeResponse({ + status, + statusText, + headersList: headers[kHeadersList], + body: stream ? { stream } : null + }) + + resolve(response) + + return false + }, + + onData (chunk) { + assert(context.controller) + + // Copy the Buffer to detach it from Buffer pool. + // TODO: Is this required? + chunk = new Uint8Array(chunk) + + context.controller.enqueue(chunk) + + return context.controller.desiredSize > 0 + }, + + onComplete () { + assert(context.controller) + + context.controller.close() + context.controller = null + + finalizeResponse(fetchParams, response) + }, + + onError (err) { + context.terminate({ aborted: err.name === 'AbortError' }) + + if (context.controller) { + context.controller.error(err) + context.controller = null + } else { + // TODO: What if 204? + resolve(makeNetworkError(err)) + } + } + } + ) + }) +} + +function createDeferredPromise () { + let res + let rej + const promise = new Promise((resolve, reject) => { + res = resolve + rej = reject + }) + + return { promise, resolve: res, reject: rej } +} + +module.exports = fetch diff --git a/lib/fetch/request.js b/lib/fetch/request.js new file mode 100644 index 00000000000..4d0d869fec5 --- /dev/null +++ b/lib/fetch/request.js @@ -0,0 +1,690 @@ +/* globals AbortController */ + +'use strict' + +const { extractBody, mixinBody, cloneBody } = require('./body') +const { Headers, fill: fillHeaders, HeadersList } = require('./headers') +const util = require('../core/util') +const { isValidHTTPToken } = require('./util') +const { + forbiddenMethods, + corsSafeListedMethods, + referrerPolicy, + requestRedirect, + requestMode, + requestCredentials, + requestCache +} = require('./constants') +const { kEnumerableProperty } = util +const { kHeaders, kSignal, kState, kGuard } = require('./symbols') +const { kHeadersList } = require('../core/symbols') +const assert = require('assert') + +let TransformStream + +// https://fetch.spec.whatwg.org/#request-class +class Request { + // https://fetch.spec.whatwg.org/#dom-request + constructor (input, init = {}) { + if (!(input instanceof Request)) { + input = String(input) + } + + // 1. Let request be null. + let request = null + + // 2. Let fallbackMode be null. + let fallbackMode = null + + // 3. Let baseURL be this’s relevant settings object’s API base URL. + // TODO: this’s relevant settings object’s API base URL? + const baseUrl = undefined + + // 4. Let signal be null. + let signal = null + + // 5. If input is a string, then: + if (typeof input === 'string') { + // 1. Let parsedURL be the result of parsing input with baseURL. + // 2. If parsedURL is failure, then throw a TypeError. + let parsedURL + try { + parsedURL = new URL(input, baseUrl) + } catch (err) { + const error = new TypeError('Failed to parse URL from ' + input) + error.cause = err + throw error + } + + // 3. If parsedURL includes credentials, then throw a TypeError. + if (parsedURL.username || parsedURL.password) { + throw new TypeError( + 'Request cannot be constructed from a URL that includes credentials: ' + + input + ) + } + + // 4. Set request to a new request whose URL is parsedURL. + request = makeRequest({ urlList: [parsedURL] }) + + // 5. Set fallbackMode to "cors". + fallbackMode = 'cors' + } else { + // 6. Otherwise: + + // 7. Assert: input is a Request object. + assert(input instanceof Request) + + // 8. Set request to input’s request. + request = input[kState] + + // 9. Set signal to input’s signal. + signal = input[kSignal] + } + + // 7. Let origin be this’s relevant settings object’s origin. + // TODO + + // 8. Let window be "client". + let window = 'client' + + // 9. If request’s window is an environment settings object and its origin + // is same origin with origin, then set window to request’s window. + // TODO + + // 10. If init["window"] exists and is non-null, then throw a TypeError. + if ('window' in init && window !== null) { + throw new TypeError(`'window' option '${window}' must be null`) + } + + // 11. If init["window"] exists, then set window to "no-window". + if ('window' in init) { + window = 'no-window' + } + + // 12. Set request to a new request with the following properties: + request = makeRequest({ ...request, window }) + + // 13. If init is not empty, then: + if (Object.keys(init).length > 0) { + // 1. If request’s mode is "navigate", then set it to "same-origin". + if (request.mode === 'navigate') { + request.mode = 'same-origin' + } + + // 2. Unset request’s reload-navigation flag. + request.reloadNavigation = false + + // 3. Unset request’s history-navigation flag. + request.historyNavigation = false + + // 4. Set request’s referrer to "client" + request.referrer = 'client' + + // 5. Set request’s referrer policy to the empty string. + request.referrerPolicy = '' + } + + // 14. If init["referrer"] exists, then: + if ('referrer' in init) { + // 1. Let referrer be init["referrer"]. + const referrer = init.referrer + + // 2. If referrer is the empty string, then set request’s referrer to "no-referrer". + if (!referrer === '') { + request.referrer = 'no-referrer' + } else { + // 1. Let parsedReferrer be the result of parsing referrer with + // baseURL. + // 2. If parsedReferrer is failure, then throw a TypeError. + let parsedReferrer + try { + parsedReferrer = new URL(referrer, baseUrl) + } catch (err) { + const error = new TypeError( + `Referrer "${referrer}" is not a valid URL.` + ) + error.cause = err + throw error + } + + // 3. If one of the following is true + // parsedReferrer’s cannot-be-a-base-URL is true, scheme is "about", + // and path contains a single string "client" + // parsedReferrer’s origin is not same origin with origin + // then set request’s referrer to "client". + // TODO + + // 4. Otherwise, set request’s referrer to parsedReferrer. + request.referrer = parsedReferrer + } + } + + // 15. If init["referrerPolicy"] exists, then set request’s referrer policy + // to it. + if ('referrerPolicy' in init) { + request.referrerPolicy = init.referrerPolicy + if (!referrerPolicy.includes(request.referrerPolicy)) { + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.referrerPolicy}' is not a valid enum value of type ReferrerPolicy.` + ) + } + } + + // 16. Let mode be init["mode"] if it exists, and fallbackMode otherwise. + let mode + if ('mode' in init) { + mode = init.mode + if (!requestMode.includes(mode)) { + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.mode}' is not a valid enum value of type RequestMode.` + ) + } + } else { + mode = fallbackMode + } + + // 17. If mode is "navigate", then throw a TypeError. + if (mode === 'navigate') { + throw new TypeError() + } + + // 18. If mode is non-null, set request’s mode to mode. + if (mode !== null) { + request.mode = mode + } + + // 19. If init["credentials"] exists, then set request’s credentials mode + // to it. + if ('credentials' in init) { + request.credentials = init.credentials + if (!requestCredentials.includes(request.credentials)) { + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.credentials}' is not a valid enum value of type RequestCredentials.` + ) + } + } + + // 18. If init["cache"] exists, then set request’s cache mode to it. + if ('cache' in init) { + request.cache = init.cache + if (!requestCache.includes(request.cache)) { + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.cache}' is not a valid enum value of type RequestCache.` + ) + } + } + + // 21. If request’s cache mode is "only-if-cached" and request’s mode is + // not "same-origin", then throw a TypeError. + if (request.cache === 'only-if-cached' && request.mode !== 'same-origin') { + throw new TypeError( + "'only-if-cached' can be set only with 'same-origin' mode" + ) + } + + // 22. If init["redirect"] exists, then set request’s redirect mode to it. + if ('redirect' in init) { + request.redirect = init.redirect + if (!requestRedirect.includes(request.redirect)) { + throw new TypeError( + `Failed to construct 'Request': The provided value '${request.redirect}' is not a valid enum value of type RequestRedirect.` + ) + } + } + + // 23. If init["integrity"] exists, then set request’s integrity metadata to it. + if ('integrity' in init) { + request.integrity = String(init.integrity) + } + + // 24. If init["keepalive"] exists, then set request’s keepalive to it. + if ('keepalive' in init) { + request.keepalive = Boolean(init.keepalive) + } + + // 25. If init["method"] exists, then: + if ('method' in init) { + // 1. Let method be init["method"]. + let method = init.method + + // 2. If method is not a method or method is a forbidden method, then + // throw a TypeError. + if (!isValidHTTPToken(init.method)) { + throw TypeError(`'${init.method}' is not a valid HTTP method.`) + } + + if (forbiddenMethods.indexOf(method.toUpperCase()) !== -1) { + throw Error(`'${init.method}' HTTP method is unsupported.`) + } + + // 3. Normalize method. + // https://fetch.spec.whatwg.org/#concept-method-normalize + method = init.method.toUpperCase() + + // 4. Set request’s method to method. + request.method = method + } + + // 26. If init["signal"] exists, then set signal to it. + if ('signal' in init) { + signal = init.signal + } + + // 27. Set this’s request to request. + this[kState] = request + + // 28. Set this’s signal to a new AbortSignal object with this’s relevant + // Realm. + // TODO: relevant Realm? + const ac = new AbortController() + this[kSignal] = ac.signal + + // 29. If signal is not null, then make this’s signal follow signal. + if (signal !== null) { + if ( + !signal || + typeof signal.aborted !== 'boolean' || + typeof signal.addEventListener !== 'function' + ) { + throw new TypeError( + "Failed to construct 'Request': member signal is not of type AbortSignal." + ) + } + + if (signal.aborted) { + ac.abort() + } else { + // TODO: Remove this listener on failure/success. + signal.addEventListener( + 'abort', + function () { + ac.abort() + }, + { once: true } + ) + } + } + + // 30. Set this’s headers to a new Headers object with this’s relevant + // Realm, whose header list is request’s header list and guard is + // "request". + // TODO: relevant Realm? + this[kHeaders] = new Headers() + this[kHeaders][kGuard] = 'request' + this[kHeaders][kHeadersList] = request.headersList + + // 31. If this’s request’s mode is "no-cors", then: + if (mode === 'no-cors') { + // 1. If this’s request’s method is not a CORS-safelisted method, + // then throw a TypeError. + if (!corsSafeListedMethods.includes(request.method)) { + throw new TypeError( + `'${request.method} is unsupported in no-cors mode.` + ) + } + + // 2. Set this’s headers’s guard to "request-no-cors". + this[kHeaders][kGuard] = 'request-no-cors' + } + + // 32. If init is not empty, then: + if (Object.keys(init).length !== 0) { + // 1. Let headers be a copy of this’s headers and its associated header + // list. + let headers = new Headers(this.headers) + + // 2. If init["headers"] exists, then set headers to init["headers"]. + if ('headers' in init) { + headers = init.headers + } + + // 3. Empty this’s headers’s header list. + this[kState].headersList = new HeadersList() + this[kHeaders][kHeadersList] = this[kState].headersList + + // 4. If headers is a Headers object, then for each header in its header + // list, append header’s name/header’s value to this’s headers. + if (headers instanceof Headers) { + this[kState].headersList.push(...headers[kHeadersList]) + } else { + // 5. Otherwise, fill this’s headers with headers. + fillHeaders(this[kState].headersList, headers) + } + } + + // 33. Let inputBody be input’s request’s body if input is a Request + // object; otherwise null. + const inputBody = input instanceof Request ? input[kState].body : null + + // 34. If either init["body"] exists and is non-null or inputBody is + // non-null, and request’s method is `GET` or `HEAD`, then throw a + // TypeError. + if ( + (('body' in init && init !== null) || inputBody !== null) && + (request.method === 'GET' || request.method === 'HEAD') + ) { + throw new TypeError('Request with GET/HEAD method cannot have body.') + } + + // 35. Let initBody be null. + let initBody = null + + // 36. If init["body"] exists and is non-null, then: + if ('body' in init && init !== null) { + // 1. Let Content-Type be null. + // 2. Set initBody and Content-Type to the result of extracting + // init["body"], with keepalive set to request’s keepalive. + const [extractedBody, contentType] = extractBody( + init.body, + request.keepalive + ) + initBody = extractedBody + + // 3, If Content-Type is non-null and this’s headers’s header list does + // not contain `Content-Type`, then append `Content-Type`/Content-Type to + // this’s headers. + if (contentType && !this[kHeaders].has('content-type')) { + this[kHeaders].append('content-type', contentType) + } + } + + // 37. Let inputOrInitBody be initBody if it is non-null; otherwise + // inputBody. + const inputOrInitBody = initBody ?? inputBody + + // 38. If inputOrInitBody is non-null and inputOrInitBody’s source is + // null, then: + if (inputOrInitBody !== null && inputOrInitBody.source == null) { + // 1. If this’s request’s mode is neither "same-origin" nor "cors", + // then throw a TypeError. + if (request.mode !== 'same-origin' && request.mode !== 'cors') { + throw new TypeError( + 'If request is made from ReadableStream, mode should be "same-origin" or "cors"' + ) + } + + // 2. Set this’s request’s use-CORS-preflight flag. + request.useCORSPreflightFlag = true + } + + // 39. Let finalBody be inputOrInitBody. + let finalBody = inputOrInitBody + + // 40. If initBody is null and inputBody is non-null, then: + if (initBody == null && inputBody !== null) { + // 1. If input is unusable, then throw a TypeError. + if (util.isDisturbed(inputBody.stream) || inputBody.stream.locked) { + throw new TypeError( + 'Cannot construct a Request with a Request object that has already been used.' + ) + } + + // 2. Set finalBody to the result of creating a proxy for inputBody. + if (!TransformStream) { + TransformStream = require('stream/web').TransformStream + } + + // https://streams.spec.whatwg.org/#readablestream-create-a-proxy + const identityTransform = new TransformStream() + inputBody.stream.pipeThrough(identityTransform) + finalBody = { + source: inputBody.source, + length: inputBody.length, + stream: identityTransform.readable + } + } + + // 41. Set this’s request’s body to finalBody. + this[kState].body = finalBody + } + + get [Symbol.toStringTag] () { + return this.constructor.name + } + + // Returns request’s HTTP method, which is "GET" by default. + get method () { + // The method getter steps are to return this’s request’s method. + return this[kState].method + } + + // Returns the URL of request as a string. + get url () { + // The url getter steps are to return this’s request’s URL, serialized. + return this[kState].url.toString() + } + + // Returns a Headers object consisting of the headers associated with request. + // Note that headers added in the network layer by the user agent will not + // be accounted for in this object, e.g., the "Host" header. + get headers () { + // The headers getter steps are to return this’s headers. + return this[kHeaders] + } + + // Returns the kind of resource requested by request, e.g., "document" + // or "script". + get destination () { + // The destination getter are to return this’s request’s destination. + return '' + } + + // Returns the referrer of request. Its value can be a same-origin URL if + // explicitly set in init, the empty string to indicate no referrer, and + // "about:client" when defaulting to the global’s default. This is used + // during fetching to determine the value of the `Referer` header of the + // request being made. + get referrer () { + // 1. If this’s request’s referrer is "no-referrer", then return the + // empty string. + if (this[kState].referrer === 'no-referrer') { + return '' + } + + // 2. If this’s request’s referrer is "client", then return + // "about:client". + if (this[kState].referrer === 'client') { + return 'about:client' + } + + // Return this’s request’s referrer, serialized. + return this[kState].referrer.toString() + } + + // Returns the referrer policy associated with request. + // This is used during fetching to compute the value of the request’s + // referrer. + get referrerPolicy () { + // The referrerPolicy getter steps are to return this’s request’s referrer policy. + return this[kState].referrerPolicy + } + + // Returns the mode associated with request, which is a string indicating + // whether the request will use CORS, or will be restricted to same-origin + // URLs. + get mode () { + // The mode getter steps are to return this’s request’s mode. + return this[kState].mode + } + + // Returns the credentials mode associated with request, + // which is a string indicating whether credentials will be sent with the + // request always, never, or only when sent to a same-origin URL. + get credentials () { + // The credentials getter steps are to return this’s request’s credentials mode. + return this[kState].credentials + } + + // Returns the cache mode associated with request, + // which is a string indicating how the request will + // interact with the browser’s cache when fetching. + get cache () { + // The cache getter steps are to return this’s request’s cache mode. + return this[kState].cache + } + + // Returns the redirect mode associated with request, + // which is a string indicating how redirects for the + // request will be handled during fetching. A request + // will follow redirects by default. + get redirect () { + // The redirect getter steps are to return this’s request’s redirect mode. + return this[kState].redirect + } + + // Returns request’s subresource integrity metadata, which is a + // cryptographic hash of the resource being fetched. Its value + // consists of multiple hashes separated by whitespace. [SRI] + get integrity () { + // The integrity getter steps are to return this’s request’s integrity + // metadata. + return this[kState].integrity + } + + // Returns a boolean indicating whether or not request can outlive the + // global in which it was created. + get keepalive () { + // The keepalive getter steps are to return this’s request’s keepalive. + return this[kState].keepalive + } + + // Returns a boolean indicating whether or not request is for a reload + // navigation. + get isReloadNavigation () { + // The isReloadNavigation getter steps are to return true if this’s + // request’s reload-navigation flag is set; otherwise false. + return this[kState].reloadNavigation + } + + // Returns a boolean indicating whether or not request is for a history + // navigation (a.k.a. back-foward navigation). + get isHistoryNavigation () { + // The isHistoryNavigation getter steps are to return true if this’s request’s + // history-navigation flag is set; otherwise false. + return this[kState].historyNavigation + } + + // Returns the signal associated with request, which is an AbortSignal + // object indicating whether or not request has been aborted, and its + // abort event handler. + get signal () { + // The signal getter steps are to return this’s signal. + return this[kSignal] + } + + // Returns a clone of request. + clone () { + // 1. If this is unusable, then throw a TypeError. + if (this.bodyUsed || (this.body && this.body.locked)) { + throw new TypeError() + } + + // 2. Let clonedRequest be the result of cloning this’s request. + const clonedRequest = cloneRequest(this[kState]) + + // 3. Let clonedRequestObject be the result of creating a Request object, + // given clonedRequest, this’s headers’s guard, and this’s relevant Realm. + // TODO: relevant Realm? + const clonedRequestObject = new Request() + clonedRequestObject[kState] = clonedRequest + clonedRequestObject[kHeaders][kHeadersList] = clonedRequest.headersList + clonedRequestObject[kHeaders][kGuard] = this[kHeaders][kGuard] + + // 4. Make clonedRequestObject’s signal follow this’s signal. + const ac = new AbortController() + if (this.signal.aborted) { + ac.abort() + } else { + this.signal.addEventListener( + 'abort', + function () { + ac.abort() + }, + { once: true } + ) + } + clonedRequestObject[kSignal] = ac.signal + + // 4. Return clonedRequestObject. + return clonedRequestObject + } +} + +mixinBody(Request.prototype) + +function makeRequest (init) { + // https://fetch.spec.whatwg.org/#requests + const request = { + method: 'GET', + localURLsOnly: false, + unsafeRequest: false, + body: null, + client: null, + reservedClient: null, + replacesClientId: '', + window: 'client', + keepalive: false, + serviceWorkers: 'all', + initiator: '', + destination: '', + priority: null, + origin: 'client', + policyContainer: 'client', + referrer: 'client', + referrerPolicy: '', + mode: 'no-cors', + useCORSPreflightFlag: false, + credentials: 'same-origin', + useCredentials: false, + cache: 'default', + redirect: 'follow', + integrity: '', + cryptoGraphicsNonceMetadata: '', + parserMetadata: '', + reloadNavigation: false, + historyNavigation: false, + userActivation: false, + taintedOrigin: false, + redirectCount: 0, + responseTainting: 'basic', + preventNoCacheCacheControlHeaderModification: false, + done: false, + timingAllowFailed: false, + ...init, + headersList: init.headersList + ? new HeadersList(...init.headersList) + : new HeadersList(), + urlList: init.urlList ? [...init.urlList.map((url) => new URL(url))] : [] + } + request.url = request.urlList[0] + return request +} + +// https://fetch.spec.whatwg.org/#concept-request-clone +function cloneRequest (request) { + // To clone a request request, run these steps: + + // 1. Let newRequest be a copy of request, except for its body. + const newRequest = makeRequest({ ...request, body: null }) + + // 2. If request’s body is non-null, set newRequest’s body to the + // result of cloning request’s body. + if (request.body !== null) { + newRequest.body = cloneBody(request.body) + } + + // 3. Return newRequest. + return newRequest +} + +Object.defineProperties(Request.prototype, { + method: kEnumerableProperty, + url: kEnumerableProperty, + headers: kEnumerableProperty, + redirect: kEnumerableProperty, + clone: kEnumerableProperty, + signal: kEnumerableProperty +}) + +module.exports = { Request, makeRequest } diff --git a/lib/fetch/response.js b/lib/fetch/response.js new file mode 100644 index 00000000000..0912d0ad28c --- /dev/null +++ b/lib/fetch/response.js @@ -0,0 +1,365 @@ +'use strict' + +const { Headers, HeadersList, fill } = require('./headers') +const { extractBody, cloneBody, mixinBody } = require('./body') +const util = require('../core/util') +const { kEnumerableProperty } = util +const { responseURL, isValidReasonPhrase } = require('./util') +const { + redirectStatus, + nullBodyStatus, + forbiddenHeaderNames +} = require('./constants') +const { kState, kHeaders, kGuard } = require('./symbols') +const { kHeadersList } = require('../core/symbols') +const assert = require('assert') + +// https://fetch.spec.whatwg.org/#response-class +class Response { + // Creates network error Response. + static error () { + // The static error() method steps are to return the result of creating a + // Response object, given a new network error, "immutable", and this’s + // relevant Realm. + // TODO: relevant Realm? + const responseObject = new Response() + responseObject[kState] = makeNetworkError() + responseObject[kHeaders][kHeadersList] = responseObject[kState].headersList + responseObject[kHeaders][kGuard] = 'immutable' + return responseObject + } + + // Creates a redirect Response that redirects to url with status status. + static redirect (url, status = 302) { + // 1. Let parsedURL be the result of parsing url with current settings + // object’s API base URL. + // 2. If parsedURL is failure, then throw a TypeError. + // TODO: base-URL? + let parsedURL + try { + parsedURL = new URL(url) + } catch (err) { + const error = new TypeError('Failed to parse URL from ' + url) + error.cause = err + throw error + } + + // 3. If status is not a redirect status, then throw a RangeError. + if (!redirectStatus.includes(status)) { + throw new RangeError('Invalid status code') + } + + // 4. Let responseObject be the result of creating a Response object, + // given a new response, "immutable", and this’s relevant Realm. + // TODO: relevant Realm? + const responseObject = new Response() + responseObject[kHeaders][kGuard] = 'immutable' + + // 5. Set responseObject’s response’s status to status. + responseObject[kState].status = status + + // 6. Let value be parsedURL, serialized and isomorphic encoded. + // TODO: isomorphic encoded? + const value = parsedURL.toString() + + // 7. Append `Location`/value to responseObject’s response’s header list. + responseObject[kState].headersList.push('location', value) + + // 8. Return responseObject. + return responseObject + } + + // https://fetch.spec.whatwg.org/#dom-response + constructor (body = null, init = {}) { + // 1. If init["status"] is not in the range 200 to 599, inclusive, then + // throw a RangeError. + if ('status' in init) { + if (!Number.isFinite(init.status)) { + throw new TypeError() + } + + if (init.status < 200 || init.status > 599) { + throw new RangeError( + `Failed to construct 'Response': The status provided (${init.status}) is outside the range [200, 599].` + ) + } + } + + if ('statusText' in init) { + // 2. If init["statusText"] does not match the reason-phrase token + // production, then throw a TypeError. + // See, https://datatracker.ietf.org/doc/html/rfc7230#section-3.1.2: + // reason-phrase = *( HTAB / SP / VCHAR / obs-text ) + if (!isValidReasonPhrase(String(init.statusText))) { + throw new TypeError('Invalid statusText') + } + } + + // 3. Set this’s response to a new response. + this[kState] = makeResponse({}) + + // 4. Set this’s headers to a new Headers object with this’s relevant + // Realm, whose header list is this’s response’s header list and guard + // is "response". + // TODO: relevant Realm? + this[kHeaders] = new Headers() + this[kHeaders][kGuard] = 'response' + this[kHeaders][kHeadersList] = this[kState].headersList + + // 5. Set this’s response’s status to init["status"]. + if ('status' in init) { + this[kState].status = init.status + } + + // 6. Set this’s response’s status message to init["statusText"]. + if ('statusText' in init) { + this[kState].statusText = String(init.statusText) + } + + // 7. If init["headers"] exists, then fill this’s headers with init["headers"]. + if ('headers' in init) { + fill(this[kState].headersList, init.headers) + } + + // 8. If body is non-null, then: + if (body !== null) { + // 1. If init["status"] is a null body status, then throw a TypeError. + if (nullBodyStatus.includes(init.status)) { + throw new TypeError('Response with null body status cannot have body') + } + + // 2. Let Content-Type be null. + // 3. Set this’s response’s body and Content-Type to the result of + // extracting body. + const [extractedBody, contentType] = extractBody(body) + this[kState].body = extractedBody + + // 4. If Content-Type is non-null and this’s response’s header list does + // not contain `Content-Type`, then append `Content-Type`/Content-Type + // to this’s response’s header list. + if (contentType && !this.headers.has('content-type')) { + this.headers.set('content-type', contentType) + } + } + } + + get [Symbol.toStringTag] () { + return this.constructor.name + } + + // Returns response’s type, e.g., "cors". + get type () { + // The type getter steps are to return this’s response’s type. + return this[kState].type + } + + // Returns response’s URL, if it has one; otherwise the empty string. + get url () { + // The url getter steps are to return the empty string if this’s + // response’s URL is null; otherwise this’s response’s URL, + // serialized with exclude fragment set to true. + let url = responseURL(this[kState]) + + if (url == null) { + return '' + } + + if (url.hash) { + url = new URL(url) + url.hash = '' + } + + return url.toString() + } + + // Returns whether response was obtained through a redirect. + get redirected () { + // The redirected getter steps are to return true if this’s response’s URL + // list has more than one item; otherwise false. + return this[kState].urlList.length > 1 + } + + // Returns response’s status. + get status () { + // The status getter steps are to return this’s response’s status. + return this[kState].status + } + + // Returns whether response’s status is an ok status. + get ok () { + // The ok getter steps are to return true if this’s response’s status is an + // ok status; otherwise false. + return this[kState].status >= 200 && this[kState].status <= 299 + } + + // Returns response’s status message. + get statusText () { + // The statusText getter steps are to return this’s response’s status + // message. + return this[kState].statusText + } + + // Returns response’s headers as Headers. + get headers () { + // The headers getter steps are to return this’s headers. + return this[kHeaders] + } + + // Returns a clone of response. + clone () { + // 1. If this is unusable, then throw a TypeError. + if (this.bodyUsed || (this.body && this.body.locked)) { + throw new TypeError() + } + + // 2. Let clonedResponse be the result of cloning this’s response. + const clonedResponse = cloneResponse(this[kState]) + + // 3. Return the result of creating a Response object, given + // clonedResponse, this’s headers’s guard, and this’s relevant Realm. + // TODO: relevant Realm? + const clonedResponseObject = new Response() + clonedResponseObject[kState] = clonedResponse + clonedResponseObject[kHeaders][kHeadersList] = clonedResponse.headersList + clonedResponseObject[kHeaders][kGuard] = this[kHeaders][kGuard] + + return clonedResponseObject + } +} +mixinBody(Response.prototype) + +Object.defineProperties(Response.prototype, { + type: kEnumerableProperty, + url: kEnumerableProperty, + status: kEnumerableProperty, + ok: kEnumerableProperty, + redirected: kEnumerableProperty, + statusText: kEnumerableProperty, + headers: kEnumerableProperty, + clone: kEnumerableProperty +}) + +// https://fetch.spec.whatwg.org/#concept-response-clone +function cloneResponse (response) { + // To clone a response response, run these steps: + + // 1. If response is a filtered response, then return a new identical + // filtered response whose internal response is a clone of response’s + // internal response. + if (response.internalResponse) { + return filterResponse( + cloneResponse(response.internalResponse), + response.type + ) + } + + // 2. Let newResponse be a copy of response, except for its body. + const newResponse = makeResponse({ ...response, body: null }) + + // 3. If response’s body is non-null, then set newResponse’s body to the + // result of cloning response’s body. + if (response.body !== null) { + newResponse.body = cloneBody(response.body) + } + + // 4. Return newResponse. + return newResponse +} + +function makeResponse (init) { + return { + internalResponse: null, + aborted: false, + rangeRequested: false, + timingAllowPassed: false, + type: 'default', + status: 200, + timingInfo: null, + statusText: '', + ...init, + headersList: init.headersList + ? new HeadersList(...init.headersList) + : new HeadersList(), + urlList: init.urlList ? [...init.urlList] : [] + } +} + +function makeNetworkError (reason) { + return makeResponse({ + type: 'error', + status: 0, + error: + reason instanceof Error + ? reason + : new Error(reason ? String(reason) : reason), + aborted: reason && reason.name === 'AbortError' + }) +} + +function filterResponse (response, type) { + // Set response to the following filtered response with response as its + // internal response, depending on request’s response tainting: + if (type === 'basic') { + // A basic filtered response is a filtered response whose type is "basic" + // and header list excludes any headers in internal response’s header list + // whose name is a forbidden response-header name. + + const headers = [] + for (let n = 0; n < response.headersList.length; n += 2) { + if (!forbiddenHeaderNames.includes(response.headersList[n])) { + headers.push(response.headersList[n + 0], response.headersList[n + 1]) + } + } + + return makeResponse({ + ...response, + internalResponse: response, + headersList: new HeadersList(...headers), + type: 'basic' + }) + } else if (type === 'cors') { + // A CORS filtered response is a filtered response whose type is "cors" + // and header list excludes any headers in internal response’s header + // list whose name is not a CORS-safelisted response-header name, given + // internal response’s CORS-exposed header-name list. + + // TODO: This is not correct... + return makeResponse({ + ...response, + internalResponse: response, + type: 'cors' + }) + } else if (type === 'opaque') { + // An opaque filtered response is a filtered response whose type is + // "opaque", URL list is the empty list, status is 0, status message + // is the empty byte sequence, header list is empty, and body is null. + + return makeResponse({ + ...response, + internalResponse: response, + type: 'opaque', + urlList: [], + status: 0, + statusText: '', + body: null + }) + } else if (type === 'opaqueredirect') { + // An opaque-redirect filtered response is a filtered response whose type + // is "opaqueredirect", status is 0, status message is the empty byte + // sequence, header list is empty, and body is null. + + return makeResponse({ + ...response, + internalResponse: response, + type: 'opaqueredirect', + status: 0, + statusText: '', + headersList: new HeadersList(), + body: null + }) + } else { + assert(false) + } +} + +module.exports = { makeNetworkError, makeResponse, filterResponse, Response } diff --git a/lib/fetch/symbols.js b/lib/fetch/symbols.js new file mode 100644 index 00000000000..23580d99b1d --- /dev/null +++ b/lib/fetch/symbols.js @@ -0,0 +1,9 @@ +'use strict' + +module.exports = { + kUrl: Symbol('url'), + kHeaders: Symbol('headers'), + kSignal: Symbol('signal'), + kState: Symbol('state'), + kGuard: Symbol('guard') +} diff --git a/lib/fetch/util.js b/lib/fetch/util.js new file mode 100644 index 00000000000..3863830f9b9 --- /dev/null +++ b/lib/fetch/util.js @@ -0,0 +1,213 @@ +'use strict' + +const { redirectStatus } = require('./constants') +const { destroy, isDestroyed } = require('../../lib/core/util') +const { AbortError } = require('../../lib/core/errors') +const { finished } = require('stream') + +let ReadableStream +let CountQueuingStrategy + +// https://fetch.spec.whatwg.org/#block-bad-port +const badPorts = [ + 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, 43, 53, 69, 77, 79, + 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, 115, 117, 119, 123, 135, 137, + 139, 143, 161, 179, 389, 427, 465, 512, 513, 514, 515, 526, 530, 531, 532, + 540, 548, 554, 556, 563, 587, 601, 636, 989, 990, 993, 995, 1719, 1720, 1723, + 2049, 3659, 4045, 5060, 5061, 6000, 6566, 6665, 6666, 6667, 6668, 6669, 6697, + 10080 +] + +function responseURL (response) { + // https://fetch.spec.whatwg.org/#responses + // A response has an associated URL. It is a pointer to the last URL + // in response’s URL list and null if response’s URL list is empty. + const urlList = response.urlList + const length = urlList.length + return length === 0 ? null : urlList[length - 1].toString() +} + +// https://fetch.spec.whatwg.org/#concept-response-location-url +function responseLocationURL (response, requestFragment) { + // 1. If response’s status is not a redirect status, then return null. + if (!redirectStatus.includes(response.status)) { + return null + } + + // 2. Let location be the result of extracting header list values given + // `Location` and response’s header list. + let location = response.headersList.get('location') + + // 3. If location is a value, then set location to the result of parsing + // location with response’s URL. + location = location ? new URL(location, responseURL(response)) : null + + // 4. If location is a URL whose fragment is null, then set location’s + // fragment to requestFragment. + if (location && !location.hash) { + location.hash = requestFragment + } + + // 5. Return location. + return location +} + +function requestCurrentURL (request) { + return request.urlList[request.urlList.length - 1] +} + +function requestBadPort (request) { + // 1. Let url be request’s current URL. + const url = requestCurrentURL(request) + + // 2. If url’s scheme is an HTTP(S) scheme and url’s port is a bad port, + // then return blocked. + if (/^http?s/.test(url.protocol) && badPorts.includes(url.port)) { + return 'blocked' + } + + // 3. Return allowed. + return 'allowed' +} + +// Check whether |statusText| is a ByteString and +// matches the Reason-Phrase token production. +// RFC 2616: https://tools.ietf.org/html/rfc2616 +// RFC 7230: https://tools.ietf.org/html/rfc7230 +// "reason-phrase = *( HTAB / SP / VCHAR / obs-text )" +// https://github.com/chromium/chromium/blob/94.0.4604.1/third_party/blink/renderer/core/fetch/response.cc#L116 +function isValidReasonPhrase (statusText) { + for (let i = 0; i < statusText.length; ++i) { + const c = statusText.charCodeAt(i) + if ( + !( + ( + c === 0x09 || // HTAB + (c >= 0x20 && c <= 0x7e) || // SP / VCHAR + (c >= 0x80 && c <= 0xff) + ) // obs-text + ) + ) { + return false + } + } + return true +} + +function isTokenChar (c) { + return !( + c >= 0x7f || + c <= 0x20 || + c === '(' || + c === ')' || + c === '<' || + c === '>' || + c === '@' || + c === ',' || + c === ';' || + c === ':' || + c === '\\' || + c === '"' || + c === '/' || + c === '[' || + c === ']' || + c === '?' || + c === '=' || + c === '{' || + c === '}' + ) +} + +// See RFC 7230, Section 3.2.6. +// https://github.com/chromium/chromium/blob/d7da0240cae77824d1eda25745c4022757499131/third_party/blink/renderer/platform/network/http_parsers.cc#L321 +function isValidHTTPToken (characters) { + if (!characters || typeof characters !== 'string') { + return false + } + for (let i = 0; i < characters.length; ++i) { + const c = characters.charCodeAt(i) + if (c > 0x7f || !isTokenChar(c)) { + return false + } + } + return true +} + +function toWebReadable (streamReadable) { + if (!ReadableStream) { + ReadableStream = require('stream/web').ReadableStream + } + if (!CountQueuingStrategy) { + CountQueuingStrategy = require('stream/web').CountQueuingStrategy + } + + if (isDestroyed(streamReadable)) { + const readable = new ReadableStream() + readable.cancel() + return readable + } + + const objectMode = streamReadable.readableObjectMode + const highWaterMark = streamReadable.readableHighWaterMark + const strategy = objectMode + ? new CountQueuingStrategy({ highWaterMark }) + : { highWaterMark } + + let controller + + function onData (chunk) { + // Copy the Buffer to detach it from the pool. + if (Buffer.isBuffer(chunk) && !objectMode) { + chunk = new Uint8Array(chunk) + } + controller.enqueue(chunk) + if (controller.desiredSize <= 0) { + streamReadable.pause() + } + } + + streamReadable.pause() + + finished(streamReadable, (err) => { + if (err && err.code === 'ERR_STREAM_PREMATURE_CLOSE') { + const er = new AbortError() + er.cause = er + err = er + } + + if (err) { + controller.error(err) + } else { + controller.close() + } + }) + + streamReadable.on('data', onData) + + return new ReadableStream( + { + start (c) { + controller = c + }, + + pull () { + streamReadable.resume() + }, + + cancel (reason) { + destroy(streamReadable, reason) + } + }, + strategy + ) +} + +module.exports = { + toWebReadable, + isValidHTTPToken, + requestBadPort, + requestCurrentURL, + responseURL, + responseLocationURL, + isValidReasonPhrase +} diff --git a/lib/handler/redirect.js b/lib/handler/redirect.js index 46b0ea8900a..776fd082a48 100644 --- a/lib/handler/redirect.js +++ b/lib/handler/redirect.js @@ -86,13 +86,13 @@ class RedirectHandler { this.handler.onError(error) } - onHeaders (statusCode, headers, resume) { + onHeaders (statusCode, headers, resume, statusText) { this.location = this.history.length >= this.maxRedirections || util.isDisturbed(this.opts.body) ? null : parseLocation(statusCode, headers) if (!this.location) { - return this.handler.onHeaders(statusCode, headers, resume) + return this.handler.onHeaders(statusCode, headers, resume, statusText) } this.history.push(new URL(this.opts.path, this.opts.origin)) diff --git a/package.json b/package.json index b89098adc78..268bd61ef0f 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,10 @@ "build:wasm": "node build/wasm.js --docker", "lint": "standard | snazzy", "lint:fix": "standard --fix | snazzy", - "test": "tap test/*.js --no-coverage && jest test/jest/test", + "test": "tap test/*.js --no-coverage && mocha test/node-fetch && jest test/jest/test", + "test:node-fetch": "node scripts/test-node-fetch.js 16 && mocha test/node-fetch || echo Skipping", + "test:jest": "jest test/jest/test", + "test:tap": "tap test/*.js --no-coverage ", "test:tdd": "tap test/*.js -w --no-coverage-report", "test:typescript": "tsd", "coverage": "standard | snazzy && tap test/*.js", @@ -50,13 +53,21 @@ "@sinonjs/fake-timers": "^7.0.5", "@types/node": "^15.0.2", "abort-controller": "^3.0.0", + "busboy": "^0.3.1", + "chai": "^4.3.4", + "chai-as-promised": "^7.1.1", + "chai-iterator": "^3.0.2", + "chai-string": "^1.5.0", "concurrently": "^6.1.0", "cronometro": "^0.8.0", + "delay": "^5.0.0", "docsify-cli": "^4.4.2", "https-pem": "^2.0.0", "husky": "^6.0.0", "jest": "^27.0.5", "jsfuzz": "^1.0.15", + "mocha": "^9.0.3", + "p-timeout": "^3.2.0", "pre-commit": "^1.2.2", "proxy": "^1.0.2", "proxyquire": "^2.1.3", @@ -72,6 +83,9 @@ "node": ">=12.18" }, "standard": { + "env": [ + "mocha" + ], "ignore": [ "lib/llhttp/constants.js", "lib/llhttp/utils.js" diff --git a/scripts/test-node-fetch.js b/scripts/test-node-fetch.js new file mode 100644 index 00000000000..fbccb87cfc4 --- /dev/null +++ b/scripts/test-node-fetch.js @@ -0,0 +1,11 @@ +const [major, minor, patch] = process.versions.node.split('.').map(v => Number(v)) +const required = process.argv.pop().split('.').map(v => Number(v)) + +const badMajor = major < required[0] +const badMinor = major === required[0] && minor < required[1] +const badPatch = major === required[0] && minor === required[1] && patch < required[2] + +if (badMajor || badMinor || badPatch) { + console.log(`Required Node.js >=${required.join('.')}, got ${process.versions.node}`) + process.exit(1) +} diff --git a/test/client-fetch.js b/test/client-fetch.js index d15c9e0d849..702d0560064 100644 --- a/test/client-fetch.js +++ b/test/client-fetch.js @@ -1,24 +1,32 @@ 'use strict' const { test } = require('tap') -const { fetch } = require('..') -const { createServer } = require('http') +const { createServer } = require('https') const nodeMajor = Number(process.versions.node.split('.')[0]) +const pem = require('https-pem') test('fetch', { skip: nodeMajor < 16 }, t => { + const { fetch, setGlobalDispatcher, Agent } = require('..') + + setGlobalDispatcher(new Agent({ + connect: { + rejectUnauthorized: false + } + })) + t.test('request json', (t) => { t.plan(1) const obj = { asd: true } - const server = createServer((req, res) => { + const server = createServer(pem, (req, res) => { res.end(JSON.stringify(obj)) }) t.teardown(server.close.bind(server)) server.listen(0, async () => { - const body = await fetch(`http://localhost:${server.address().port}`) + const body = await fetch(`https://localhost:${server.address().port}`) t.strictSame(obj, await body.json()) }) }) @@ -27,13 +35,13 @@ test('fetch', { t.plan(1) const obj = { asd: true } - const server = createServer((req, res) => { + const server = createServer(pem, (req, res) => { res.end(JSON.stringify(obj)) }) t.teardown(server.close.bind(server)) server.listen(0, async () => { - const body = await fetch(`http://localhost:${server.address().port}`) + const body = await fetch(`https://localhost:${server.address().port}`) t.strictSame(JSON.stringify(obj), await body.text()) }) }) @@ -42,13 +50,13 @@ test('fetch', { t.plan(1) const obj = { asd: true } - const server = createServer((req, res) => { + const server = createServer(pem, (req, res) => { res.end(JSON.stringify(obj)) }) t.teardown(server.close.bind(server)) server.listen(0, async () => { - const body = await fetch(`http://localhost:${server.address().port}`) + const body = await fetch(`https://localhost:${server.address().port}`) t.strictSame(Buffer.from(JSON.stringify(obj)), Buffer.from(await body.arrayBuffer())) }) }) @@ -57,14 +65,14 @@ test('fetch', { t.plan(1) const obj = { asd: true } - const server = createServer((req, res) => { + const server = createServer(pem, (req, res) => { res.setHeader('Content-Type', 'application/json') res.end(JSON.stringify(obj)) }) t.teardown(server.close.bind(server)) server.listen(0, async () => { - const response = await fetch(`http://localhost:${server.address().port}`) + const response = await fetch(`https://localhost:${server.address().port}`) t.equal('application/json', (await response.blob()).type) }) }) diff --git a/test/node-fetch/LICENSE b/test/node-fetch/LICENSE new file mode 100644 index 00000000000..41ca1b6eb4d --- /dev/null +++ b/test/node-fetch/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2016 - 2020 Node Fetch Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/test/node-fetch/headers.js b/test/node-fetch/headers.js new file mode 100644 index 00000000000..e509fd8d893 --- /dev/null +++ b/test/node-fetch/headers.js @@ -0,0 +1,282 @@ +/* eslint no-unused-expressions: "off" */ + +const { format } = require('util') +const chai = require('chai') +const chaiIterator = require('chai-iterator') +const { Headers } = require('../../lib/fetch/headers.js') + +chai.use(chaiIterator) + +const { expect } = chai + +describe('Headers', () => { + it('should have attributes conforming to Web IDL', () => { + const headers = new Headers() + expect(Object.getOwnPropertyNames(headers)).to.be.empty + const enumerableProperties = [] + + for (const property in headers) { + enumerableProperties.push(property) + } + + for (const toCheck of [ + 'append', + 'delete', + 'entries', + 'forEach', + 'get', + 'has', + 'keys', + 'set', + 'values' + ]) { + expect(enumerableProperties).to.contain(toCheck) + } + }) + + it('should allow iterating through all headers with forEach', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['b', '3'], + ['a', '1'] + ]) + expect(headers).to.have.property('forEach') + + const result = [] + for (const [key, value] of headers.entries()) { + result.push([key, value]) + } + + expect(result).to.deep.equal([ + ['a', '1'], + ['b', '2, 3'], + ['c', '4'] + ]) + }) + + it('should be iterable with forEach', () => { + const headers = new Headers() + headers.append('Accept', 'application/json') + headers.append('Accept', 'text/plain') + headers.append('Content-Type', 'text/html') + + const results = [] + headers.forEach((value, key, object) => { + results.push({ value, key, object }) + }) + + expect(results.length).to.equal(2) + expect({ key: 'accept', value: 'application/json, text/plain', object: headers }).to.deep.equal(results[0]) + expect({ key: 'content-type', value: 'text/html', object: headers }).to.deep.equal(results[1]) + }) + + xit('should set "this" to undefined by default on forEach', () => { + const headers = new Headers({ Accept: 'application/json' }) + headers.forEach(function () { + expect(this).to.be.undefined + }) + }) + + it('should accept thisArg as a second argument for forEach', () => { + const headers = new Headers({ Accept: 'application/json' }) + const thisArg = {} + headers.forEach(function () { + expect(this).to.equal(thisArg) + }, thisArg) + }) + + it('should allow iterating through all headers with for-of loop', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]) + headers.append('b', '3') + expect(headers).to.be.iterable + + const result = [] + for (const pair of headers) { + result.push(pair) + } + + expect(result).to.deep.equal([ + ['a', '1'], + ['b', '2, 3'], + ['c', '4'] + ]) + }) + + it('should allow iterating through all headers with entries()', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]) + headers.append('b', '3') + + expect(headers.entries()).to.be.iterable + .and.to.deep.iterate.over([ + ['a', '1'], + ['b', '2, 3'], + ['c', '4'] + ]) + }) + + it('should allow iterating through all headers with keys()', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]) + headers.append('b', '3') + + expect(headers.keys()).to.be.iterable + .and.to.iterate.over(['a', 'b', 'c']) + }) + + it('should allow iterating through all headers with values()', () => { + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]) + headers.append('b', '3') + + expect(headers.values()).to.be.iterable + .and.to.iterate.over(['1', '2, 3', '4']) + }) + + it('should reject illegal header', () => { + const headers = new Headers() + expect(() => new Headers({ 'He y': 'ok' })).to.throw(TypeError) + expect(() => new Headers({ 'Hé-y': 'ok' })).to.throw(TypeError) + expect(() => new Headers({ 'He-y': 'ăk' })).to.throw(TypeError) + expect(() => headers.append('Hé-y', 'ok')).to.throw(TypeError) + expect(() => headers.delete('Hé-y')).to.throw(TypeError) + expect(() => headers.get('Hé-y')).to.throw(TypeError) + expect(() => headers.has('Hé-y')).to.throw(TypeError) + expect(() => headers.set('Hé-y', 'ok')).to.throw(TypeError) + // Should reject empty header + expect(() => headers.append('', 'ok')).to.throw(TypeError) + }) + + xit('should ignore unsupported attributes while reading headers', () => { + const FakeHeader = function () {} + // Prototypes are currently ignored + // This might change in the future: #181 + FakeHeader.prototype.z = 'fake' + + const res = new FakeHeader() + res.a = 'string' + res.b = ['1', '2'] + res.c = '' + res.d = [] + res.e = 1 + res.f = [1, 2] + res.g = { a: 1 } + res.h = undefined + res.i = null + res.j = Number.NaN + res.k = true + res.l = false + res.m = Buffer.from('test') + + const h1 = new Headers(res) + h1.set('n', [1, 2]) + h1.append('n', ['3', 4]) + + const h1Raw = h1.raw() + + expect(h1Raw.a).to.include('string') + expect(h1Raw.b).to.include('1,2') + expect(h1Raw.c).to.include('') + expect(h1Raw.d).to.include('') + expect(h1Raw.e).to.include('1') + expect(h1Raw.f).to.include('1,2') + expect(h1Raw.g).to.include('[object Object]') + expect(h1Raw.h).to.include('undefined') + expect(h1Raw.i).to.include('null') + expect(h1Raw.j).to.include('NaN') + expect(h1Raw.k).to.include('true') + expect(h1Raw.l).to.include('false') + expect(h1Raw.m).to.include('test') + expect(h1Raw.n).to.include('1,2') + expect(h1Raw.n).to.include('3,4') + + expect(h1Raw.z).to.be.undefined + }) + + xit('should wrap headers', () => { + const h1 = new Headers({ + a: '1' + }) + const h1Raw = h1.raw() + + const h2 = new Headers(h1) + h2.set('b', '1') + const h2Raw = h2.raw() + + const h3 = new Headers(h2) + h3.append('a', '2') + const h3Raw = h3.raw() + + expect(h1Raw.a).to.include('1') + expect(h1Raw.a).to.not.include('2') + + expect(h2Raw.a).to.include('1') + expect(h2Raw.a).to.not.include('2') + expect(h2Raw.b).to.include('1') + + expect(h3Raw.a).to.include('1') + expect(h3Raw.a).to.include('2') + expect(h3Raw.b).to.include('1') + }) + + it('should accept headers as an iterable of tuples', () => { + let headers + + headers = new Headers([ + ['a', '1'], + ['b', '2'], + ['a', '3'] + ]) + expect(headers.get('a')).to.equal('1, 3') + expect(headers.get('b')).to.equal('2') + + headers = new Headers([ + new Set(['a', '1']), + ['b', '2'], + new Map([['a', null], ['3', null]]).keys() + ]) + expect(headers.get('a')).to.equal('1, 3') + expect(headers.get('b')).to.equal('2') + + headers = new Headers(new Map([ + ['a', '1'], + ['b', '2'] + ])) + expect(headers.get('a')).to.equal('1') + expect(headers.get('b')).to.equal('2') + }) + + it('should throw a TypeError if non-tuple exists in a headers initializer', () => { + expect(() => new Headers([['b', '2', 'huh?']])).to.throw(TypeError) + expect(() => new Headers(['b2'])).to.throw(TypeError) + expect(() => new Headers('b2')).to.throw(TypeError) + expect(() => new Headers({ [Symbol.iterator]: 42 })).to.throw(TypeError) + }) + + xit('should use a custom inspect function', () => { + const headers = new Headers([ + ['Host', 'thehost'], + ['Host', 'notthehost'], + ['a', '1'], + ['b', '2'], + ['a', '3'] + ]) + + // eslint-disable-next-line quotes + expect(format(headers)).to.equal("{ a: [ '1', '3' ], b: '2', host: 'thehost' }") + }) +}) diff --git a/test/node-fetch/main.js b/test/node-fetch/main.js new file mode 100644 index 00000000000..e7ab9188e6e --- /dev/null +++ b/test/node-fetch/main.js @@ -0,0 +1,1618 @@ +/* eslint no-unused-expressions: "off" */ +/* globals AbortController */ + +// Test tools +const zlib = require('zlib') +const stream = require('stream') +const vm = require('vm') +const chai = require('chai') +const crypto = require('crypto') +const chaiPromised = require('chai-as-promised') +const chaiIterator = require('chai-iterator') +const chaiString = require('chai-string') +const delay = require('delay') +const { Blob } = require('buffer') + +const { + fetch, + Headers, + Request, + Response, + setGlobalDispatcher, + Agent +} = require('../../index.js') +const HeadersOrig = require('../../lib/fetch/headers.js').Headers +const RequestOrig = require('../../lib/fetch/request.js').Request +const ResponseOrig = require('../../lib/fetch/response.js').Response +const TestServer = require('./utils/server.js') +const chaiTimeout = require('./utils/chai-timeout.js') +const { ReadableStream } = require('stream/web') + +function isNodeLowerThan (version) { + return !~process.version.localeCompare(version, undefined, { numeric: true }) +} + +const { + Uint8Array: VMUint8Array +} = vm.runInNewContext('this') + +chai.use(chaiPromised) +chai.use(chaiIterator) +chai.use(chaiString) +chai.use(chaiTimeout) +const { expect } = chai + +describe('node-fetch', () => { + const local = new TestServer() + let base + + before(async () => { + await local.start() + setGlobalDispatcher(new Agent({ + connect: { + rejectUnauthorized: false + } + })) + base = `http://${local.hostname}:${local.port}/` + }) + + after(async () => { + return local.stop() + }) + + it('should return a promise', () => { + const url = `${base}hello` + const p = fetch(url) + expect(p).to.be.an.instanceof(Promise) + expect(p).to.have.property('then') + }) + + it('should expose Headers, Response and Request constructors', () => { + expect(Headers).to.equal(HeadersOrig) + expect(Response).to.equal(ResponseOrig) + expect(Request).to.equal(RequestOrig) + }) + + it('should support proper toString output for Headers, Response and Request objects', () => { + expect(new Headers().toString()).to.equal('[object Headers]') + expect(new Response().toString()).to.equal('[object Response]') + expect(new Request(base).toString()).to.equal('[object Request]') + }) + + it('should reject with error if url is protocol relative', () => { + const url = '//example.com/' + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) + }) + + it('should reject with error if url is relative path', () => { + const url = '/some/path' + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) + }) + + it('should reject with error if protocol is unsupported', () => { + const url = 'ftp://example.com/' + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) + }) + + it('should reject with error on network failure', function () { + this.timeout(5000) + const url = 'http://localhost:50000/' + return expect(fetch(url)).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + }) + + it('should resolve into response', () => { + const url = `${base}hello` + return fetch(url).then(res => { + expect(res).to.be.an.instanceof(Response) + expect(res.headers).to.be.an.instanceof(Headers) + expect(res.body).to.be.an.instanceof(ReadableStream) + expect(res.bodyUsed).to.be.false + + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + expect(res.statusText).to.equal('OK') + }) + }) + + it('Response.redirect should resolve into response', () => { + const res = Response.redirect('http://localhost') + expect(res).to.be.an.instanceof(Response) + expect(res.headers).to.be.an.instanceof(Headers) + expect(res.headers.get('location')).to.equal('http://localhost/') + expect(res.status).to.equal(302) + }) + + it('Response.redirect /w invalid url should fail', () => { + expect(() => { + Response.redirect('localhost') + }).to.throw() + }) + + it('Response.redirect /w invalid status should fail', () => { + expect(() => { + Response.redirect('http://localhost', 200) + }).to.throw() + }) + + it('should accept plain text response', () => { + const url = `${base}plain` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.a('string') + expect(result).to.equal('text') + }) + }) + }) + + it('should accept html response (like plain text)', () => { + const url = `${base}html` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/html') + return res.text().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.a('string') + expect(result).to.equal('') + }) + }) + }) + + it('should accept json response', () => { + const url = `${base}json` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('application/json') + return res.json().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.an('object') + expect(result).to.deep.equal({ name: 'value' }) + }) + }) + }) + + it('should send request with custom headers', () => { + const url = `${base}inspect` + const options = { + headers: { 'x-custom-header': 'abc' } + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.headers['x-custom-header']).to.equal('abc') + }) + }) + + it('should accept headers instance', () => { + const url = `${base}inspect` + const options = { + headers: new Headers({ 'x-custom-header': 'abc' }) + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.headers['x-custom-header']).to.equal('abc') + }) + }) + + it('should follow redirect code 301', () => { + const url = `${base}redirect/301` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + }) + }) + + it('should follow redirect code 302', () => { + const url = `${base}redirect/302` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow redirect code 303', () => { + const url = `${base}redirect/303` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow redirect code 307', () => { + const url = `${base}redirect/307` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow redirect code 308', () => { + const url = `${base}redirect/308` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow redirect chain', () => { + const url = `${base}redirect/chain` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should follow POST request redirect code 301 with GET', () => { + const url = `${base}redirect/301` + const options = { + method: 'POST', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(result => { + expect(result.method).to.equal('GET') + expect(result.body).to.equal('') + }) + }) + }) + + it('should follow PATCH request redirect code 301 with PATCH', () => { + const url = `${base}redirect/301` + const options = { + method: 'PATCH', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(res => { + expect(res.method).to.equal('PATCH') + expect(res.body).to.equal('a=1') + }) + }) + }) + + it('should follow POST request redirect code 302 with GET', () => { + const url = `${base}redirect/302` + const options = { + method: 'POST', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(result => { + expect(result.method).to.equal('GET') + expect(result.body).to.equal('') + }) + }) + }) + + it('should follow PATCH request redirect code 302 with PATCH', () => { + const url = `${base}redirect/302` + const options = { + method: 'PATCH', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(res => { + expect(res.method).to.equal('PATCH') + expect(res.body).to.equal('a=1') + }) + }) + }) + + it('should follow redirect code 303 with GET', () => { + const url = `${base}redirect/303` + const options = { + method: 'PUT', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(result => { + expect(result.method).to.equal('GET') + expect(result.body).to.equal('') + }) + }) + }) + + it('should follow PATCH request redirect code 307 with PATCH', () => { + const url = `${base}redirect/307` + const options = { + method: 'PATCH', + body: 'a=1' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + return res.json().then(result => { + expect(result.method).to.equal('PATCH') + expect(result.body).to.equal('a=1') + }) + }) + }) + + it('should not follow non-GET redirect if body is a readable stream', () => { + const url = `${base}redirect/307` + const options = { + method: 'PATCH', + body: stream.Readable.from('tada') + } + return expect(fetch(url, options)).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + }) + + it('should obey maximum redirect, reject case', () => { + const url = `${base}redirect/chain/20` + return expect(fetch(url)).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + }) + + it('should obey redirect chain, resolve case', () => { + const url = `${base}redirect/chain/19` + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}inspect`) + expect(res.status).to.equal(200) + }) + }) + + it('should support redirect mode, error flag', () => { + const url = `${base}redirect/301` + const options = { + redirect: 'error' + } + return expect(fetch(url, options)).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + }) + + it('should support redirect mode, manual flag when there is no redirect', () => { + const url = `${base}hello` + const options = { + redirect: 'manual' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(url) + expect(res.status).to.equal(200) + expect(res.headers.get('location')).to.be.null + }) + }) + + it('should follow redirect code 301 and keep existing headers', () => { + const url = `${base}redirect/301` + const options = { + headers: new Headers({ 'x-custom-header': 'abc' }) + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(`${base}inspect`) + return res.json() + }).then(res => { + expect(res.headers['x-custom-header']).to.equal('abc') + }) + }) + + it('should treat broken redirect as ordinary response (follow)', () => { + const url = `${base}redirect/no-location` + return fetch(url).then(res => { + expect(res.url).to.equal(url) + expect(res.status).to.equal(301) + expect(res.headers.get('location')).to.be.null + }) + }) + + xit('should treat broken redirect as ordinary response (manual)', () => { + const url = `${base}redirect/no-location` + const options = { + redirect: 'manual' + } + return fetch(url, options).then(res => { + expect(res.url).to.equal(url) + expect(res.status).to.equal(301) + expect(res.headers.get('location')).to.be.null + }) + }) + + it('should throw a TypeError on an invalid redirect option', () => { + const url = `${base}redirect/301` + const options = { + redirect: 'foobar' + } + return fetch(url, options).then(() => { + expect.fail() + }, error => { + expect(error).to.be.an.instanceOf(TypeError) + }) + }) + + it('should set redirected property on response when redirect', () => { + const url = `${base}redirect/301` + return fetch(url).then(res => { + expect(res.redirected).to.be.true + }) + }) + + it('should not set redirected property on response without redirect', () => { + const url = `${base}hello` + return fetch(url).then(res => { + expect(res.redirected).to.be.false + }) + }) + + it('should handle client-error response', () => { + const url = `${base}error/400` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + expect(res.status).to.equal(400) + expect(res.statusText).to.equal('Bad Request') + expect(res.ok).to.be.false + return res.text().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.a('string') + expect(result).to.equal('client error') + }) + }) + }) + + it('should handle server-error response', () => { + const url = `${base}error/500` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + expect(res.status).to.equal(500) + expect(res.statusText).to.equal('Internal Server Error') + expect(res.ok).to.be.false + return res.text().then(result => { + expect(res.bodyUsed).to.be.true + expect(result).to.be.a('string') + expect(result).to.equal('server error') + }) + }) + }) + + it('should handle network-error response', () => { + const url = `${base}error/reset` + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) + }) + + it('should handle network-error partial response', () => { + const url = `${base}error/premature` + return fetch(url).then(res => { + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + return expect(res.text()).to.eventually.be.rejectedWith(Error) + }) + }) + + it('should handle network-error in chunked response async iterator', () => { + const url = `${base}error/premature/chunked` + return fetch(url).then(res => { + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + + const read = async body => { + const chunks = [] + for await (const chunk of body) { + chunks.push(chunk) + } + + return chunks + } + + return expect(read(res.body)) + .to.eventually.be.rejectedWith(Error) + }) + }) + + it('should handle network-error in chunked response in consumeBody', () => { + const url = `${base}error/premature/chunked` + return fetch(url).then(res => { + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + + return expect(res.text()).to.eventually.be.rejectedWith(Error) + }) + }) + + it('should handle DNS-error response', () => { + const url = 'http://domain.invalid' + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError) + }) + + it('should reject invalid json response', () => { + const url = `${base}error/json` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('application/json') + return expect(res.json()).to.eventually.be.rejectedWith(Error) + }) + }) + + it('should handle response with no status text', () => { + const url = `${base}no-status-text` + return fetch(url).then(res => { + expect(res.statusText).to.equal('') + }) + }) + + it('should handle no content response', () => { + const url = `${base}no-content` + return fetch(url).then(res => { + expect(res.status).to.equal(204) + expect(res.statusText).to.equal('No Content') + expect(res.ok).to.be.true + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + it('should reject when trying to parse no content response as json', () => { + const url = `${base}no-content` + return fetch(url).then(res => { + expect(res.status).to.equal(204) + expect(res.statusText).to.equal('No Content') + expect(res.ok).to.be.true + return expect(res.json()).to.eventually.be.rejectedWith(Error) + }) + }) + + xit('should handle no content response with gzip encoding', () => { + const url = `${base}no-content/gzip` + return fetch(url).then(res => { + expect(res.status).to.equal(204) + expect(res.statusText).to.equal('No Content') + expect(res.headers.get('content-encoding')).to.equal('gzip') + expect(res.ok).to.be.true + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + xit('should handle not modified response', () => { + const url = `${base}not-modified` + return fetch(url).then(res => { + expect(res.status).to.equal(304) + expect(res.statusText).to.equal('Not Modified') + expect(res.ok).to.be.false + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + xit('should handle not modified response with gzip encoding', () => { + const url = `${base}not-modified/gzip` + return fetch(url).then(res => { + expect(res.status).to.equal(304) + expect(res.statusText).to.equal('Not Modified') + expect(res.headers.get('content-encoding')).to.equal('gzip') + expect(res.ok).to.be.false + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + xit('should decompress gzip response', () => { + const url = `${base}gzip` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should decompress slightly invalid gzip response', () => { + const url = `${base}gzip-truncated` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should make capitalised Content-Encoding lowercase', () => { + const url = `${base}gzip-capital` + return fetch(url).then(res => { + expect(res.headers.get('content-encoding')).to.equal('gzip') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should decompress deflate response', () => { + const url = `${base}deflate` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should decompress deflate raw response from old apache server', () => { + const url = `${base}deflate-raw` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should decompress brotli response', function () { + if (typeof zlib.createBrotliDecompress !== 'function') { + this.skip() + } + + const url = `${base}brotli` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('hello world') + }) + }) + }) + + xit('should handle no content response with brotli encoding', function () { + if (typeof zlib.createBrotliDecompress !== 'function') { + this.skip() + } + + const url = `${base}no-content/brotli` + return fetch(url).then(res => { + expect(res.status).to.equal(204) + expect(res.statusText).to.equal('No Content') + expect(res.headers.get('content-encoding')).to.equal('br') + expect(res.ok).to.be.true + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.be.empty + }) + }) + }) + + xit('should skip decompression if unsupported', () => { + const url = `${base}sdch` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.equal('fake sdch string') + }) + }) + }) + + xit('should reject if response compression is invalid', () => { + const url = `${base}invalid-content-encoding` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return expect(res.text()).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + .and.have.property('code', 'Z_DATA_ERROR') + }) + }) + + it('should handle errors on the body stream even if it is not used', done => { + const url = `${base}invalid-content-encoding` + fetch(url) + .then(res => { + expect(res.status).to.equal(200) + }) + .catch(() => {}) + .then(() => { + // Wait a few ms to see if a uncaught error occurs + setTimeout(() => { + done() + }, 20) + }) + }) + + xit('should collect handled errors on the body stream to reject if the body is used later', () => { + const url = `${base}invalid-content-encoding` + return fetch(url).then(delay(20)).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return expect(res.text()).to.eventually.be.rejected + .and.be.an.instanceOf(TypeError) + .and.have.property('code', 'Z_DATA_ERROR') + }) + }) + + xit('should allow disabling auto decompression', () => { + const url = `${base}gzip` + const options = { + compress: false + } + return fetch(url, options).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(result => { + expect(result).to.be.a('string') + expect(result).to.not.equal('hello world') + }) + }) + }) + + xit('should not overwrite existing accept-encoding header when auto decompression is true', () => { + const url = `${base}inspect` + const options = { + compress: true, + headers: { + 'Accept-Encoding': 'gzip' + } + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.headers['accept-encoding']).to.equal('gzip') + }) + }) + + describe('AbortController', () => { + let controller + + beforeEach(() => { + controller = new AbortController() + }) + + it('should support request cancellation with signal', () => { + const fetches = [ + fetch( + `${base}timeout`, + { + method: 'POST', + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + body: JSON.stringify({ hello: 'world' }) + } + } + ) + ] + setTimeout(() => { + controller.abort() + }, 100) + + return Promise.all(fetches.map(fetched => expect(fetched) + .to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + )) + }) + + it('should support multiple request cancellation with signal', () => { + const fetches = [ + fetch(`${base}timeout`, { signal: controller.signal }), + fetch( + `${base}timeout`, + { + method: 'POST', + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + body: JSON.stringify({ hello: 'world' }) + } + } + ) + ] + setTimeout(() => { + controller.abort() + }, 100) + + return Promise.all(fetches.map(fetched => expect(fetched) + .to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + )) + }) + + it('should reject immediately if signal has already been aborted', () => { + const url = `${base}timeout` + const options = { + signal: controller.signal + } + controller.abort() + const fetched = fetch(url, options) + return expect(fetched).to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) + + it('should allow redirects to be aborted', () => { + const request = new Request(`${base}redirect/slow`, { + signal: controller.signal + }) + setTimeout(() => { + controller.abort() + }, 20) + return expect(fetch(request)).to.be.eventually.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) + + it('should allow redirected response body to be aborted', () => { + const request = new Request(`${base}redirect/slow-stream`, { + signal: controller.signal + }) + return expect(fetch(request).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + const result = res.text() + controller.abort() + return result + })).to.be.eventually.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError') + }) + + it('should reject response body with AbortError when aborted before stream has been read completely', () => { + return expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then(res => { + const promise = res.text() + controller.abort() + return expect(promise) + .to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + }) + }) + + it('should reject response body methods immediately with AbortError when aborted before stream is disturbed', () => { + return expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then(res => { + controller.abort() + return expect(res.text()) + .to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + }) + }) + }) + + it('should throw a TypeError if a signal is not of type AbortSignal or EventTarget', () => { + return Promise.all([ + expect(fetch(`${base}inspect`, { signal: {} })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError), + expect(fetch(`${base}inspect`, { signal: '' })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError), + expect(fetch(`${base}inspect`, { signal: Object.create(null) })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError) + ]) + }) + + it('should gracefully handle a null signal', () => { + return fetch(`${base}hello`, { signal: null }).then(res => { + return expect(res.ok).to.be.true + }) + }) + + it('should allow setting User-Agent', () => { + const url = `${base}inspect` + const options = { + headers: { + 'user-agent': 'faked' + } + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.headers['user-agent']).to.equal('faked') + }) + }) + + it('should set default Accept header', () => { + const url = `${base}inspect` + fetch(url).then(res => res.json()).then(res => { + expect(res.headers.accept).to.equal('*/*') + }) + }) + + it('should allow setting Accept header', () => { + const url = `${base}inspect` + const options = { + headers: { + accept: 'application/json' + } + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.headers.accept).to.equal('application/json') + }) + }) + + it('should allow POST request', () => { + const url = `${base}inspect` + const options = { + method: 'POST' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('0') + }) + }) + + it('should allow POST request with string body', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: 'a=1' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.equal('text/plain;charset=UTF-8') + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow POST request with buffer body', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: Buffer.from('a=1', 'utf-8') + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow POST request with ArrayBuffer body', () => { + const encoder = new TextEncoder() + const url = `${base}inspect` + const options = { + method: 'POST', + body: encoder.encode('Hello, world!\n').buffer + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBuffer body from a VM context', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: new VMUint8Array(Buffer.from('Hello, world!\n')).buffer + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBufferView (Uint8Array) body', () => { + const encoder = new TextEncoder() + const url = `${base}inspect` + const options = { + method: 'POST', + body: encoder.encode('Hello, world!\n') + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBufferView (DataView) body', () => { + const encoder = new TextEncoder() + const url = `${base}inspect` + const options = { + method: 'POST', + body: new DataView(encoder.encode('Hello, world!\n').buffer) + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBufferView (Uint8Array) body from a VM context', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: new VMUint8Array(Buffer.from('Hello, world!\n')) + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('Hello, world!\n') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('14') + }) + }) + + it('should allow POST request with ArrayBufferView (Uint8Array, offset, length) body', () => { + const encoder = new TextEncoder() + const url = `${base}inspect` + const options = { + method: 'POST', + body: encoder.encode('Hello, world!\n').subarray(7, 13) + } + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('world!') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('6') + }) + }) + + it('should allow POST request with blob body without type', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: new Blob(['a=1']) + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + // expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow POST request with blob body with type', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: new Blob(['a=1'], { + type: 'text/plain;charset=UTF-8' + }) + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-type']).to.equal('text/plain;charset=utf-8') + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow POST request with readable stream as body', () => { + const url = `${base}inspect` + const options = { + method: 'POST', + body: stream.Readable.from('a=1') + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.equal('chunked') + expect(res.headers['content-type']).to.be.undefined + expect(res.headers['content-length']).to.be.undefined + }) + }) + + it('should allow POST request with object body', () => { + const url = `${base}inspect` + // Note that fetch simply calls tostring on an object + const options = { + method: 'POST', + body: { a: 1 } + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.body).to.equal('[object Object]') + expect(res.headers['content-type']).to.equal('text/plain;charset=UTF-8') + expect(res.headers['content-length']).to.equal('15') + }) + }) + + it('constructing a Response with URLSearchParams as body should have a Content-Type', () => { + const parameters = new URLSearchParams() + const res = new Response(parameters) + res.headers.get('Content-Type') + expect(res.headers.get('Content-Type')).to.equal('application/x-www-form-urlencoded;charset=UTF-8') + }) + + it('constructing a Request with URLSearchParams as body should have a Content-Type', () => { + const parameters = new URLSearchParams() + const request = new Request(base, { method: 'POST', body: parameters }) + expect(request.headers.get('Content-Type')).to.equal('application/x-www-form-urlencoded;charset=UTF-8') + }) + + it('Reading a body with URLSearchParams should echo back the result', () => { + const parameters = new URLSearchParams() + parameters.append('a', '1') + return new Response(parameters).text().then(text => { + expect(text).to.equal('a=1') + }) + }) + + // Body should been cloned... + it('constructing a Request/Response with URLSearchParams and mutating it should not affected body', () => { + const parameters = new URLSearchParams() + const request = new Request(`${base}inspect`, { method: 'POST', body: parameters }) + parameters.append('a', '1') + return request.text().then(text => { + expect(text).to.equal('') + }) + }) + + it('should allow POST request with URLSearchParams as body', () => { + const parameters = new URLSearchParams() + parameters.append('a', '1') + + const url = `${base}inspect` + const options = { + method: 'POST', + body: parameters + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.headers['content-type']).to.equal('application/x-www-form-urlencoded;charset=UTF-8') + expect(res.headers['content-length']).to.equal('3') + expect(res.body).to.equal('a=1') + }) + }) + + it('should still recognize URLSearchParams when extended', () => { + class CustomSearchParameters extends URLSearchParams {} + const parameters = new CustomSearchParameters() + parameters.append('a', '1') + + const url = `${base}inspect` + const options = { + method: 'POST', + body: parameters + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('POST') + expect(res.headers['content-type']).to.equal('application/x-www-form-urlencoded;charset=UTF-8') + expect(res.headers['content-length']).to.equal('3') + expect(res.body).to.equal('a=1') + }) + }) + + it('should allow PUT request', () => { + const url = `${base}inspect` + const options = { + method: 'PUT', + body: 'a=1' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('PUT') + expect(res.body).to.equal('a=1') + }) + }) + + it('should allow DELETE request', () => { + const url = `${base}inspect` + const options = { + method: 'DELETE' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('DELETE') + }) + }) + + it('should allow DELETE request with string body', () => { + const url = `${base}inspect` + const options = { + method: 'DELETE', + body: 'a=1' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('DELETE') + expect(res.body).to.equal('a=1') + expect(res.headers['transfer-encoding']).to.be.undefined + expect(res.headers['content-length']).to.equal('3') + }) + }) + + it('should allow PATCH request', () => { + const url = `${base}inspect` + const options = { + method: 'PATCH', + body: 'a=1' + } + return fetch(url, options).then(res => { + return res.json() + }).then(res => { + expect(res.method).to.equal('PATCH') + expect(res.body).to.equal('a=1') + }) + }) + + it('should allow HEAD request', () => { + const url = `${base}hello` + const options = { + method: 'HEAD' + } + return fetch(url, options).then(res => { + expect(res.status).to.equal(200) + expect(res.statusText).to.equal('OK') + expect(res.headers.get('content-type')).to.equal('text/plain') + // expect(res.body).to.be.an.instanceof(stream.Transform) + return res.text() + }).then(text => { + expect(text).to.equal('') + }) + }) + + it('should allow HEAD request with content-encoding header', () => { + const url = `${base}error/404` + const options = { + method: 'HEAD' + } + return fetch(url, options).then(res => { + expect(res.status).to.equal(404) + expect(res.headers.get('content-encoding')).to.equal('gzip') + return res.text() + }).then(text => { + expect(text).to.equal('') + }) + }) + + it('should allow OPTIONS request', () => { + const url = `${base}options` + const options = { + method: 'OPTIONS' + } + return fetch(url, options).then(res => { + expect(res.status).to.equal(200) + expect(res.statusText).to.equal('OK') + expect(res.headers.get('allow')).to.equal('GET, HEAD, OPTIONS') + // expect(res.body).to.be.an.instanceof(stream.Transform) + }) + }) + + it('should reject decoding body twice', () => { + const url = `${base}plain` + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain') + return res.text().then(() => { + expect(res.bodyUsed).to.be.true + return expect(res.text()).to.eventually.be.rejectedWith(Error) + }) + }) + }) + + it('should allow cloning a json response and log it as text response', () => { + const url = `${base}json` + return fetch(url).then(res => { + const r1 = res.clone() + return Promise.all([res.json(), r1.text()]).then(results => { + expect(results[0]).to.deep.equal({ name: 'value' }) + expect(results[1]).to.equal('{"name":"value"}') + }) + }) + }) + + it('should allow cloning a json response, and then log it as text response', () => { + const url = `${base}json` + return fetch(url).then(res => { + const r1 = res.clone() + return res.json().then(result => { + expect(result).to.deep.equal({ name: 'value' }) + return r1.text().then(result => { + expect(result).to.equal('{"name":"value"}') + }) + }) + }) + }) + + it('should allow cloning a json response, first log as text response, then return json object', () => { + const url = `${base}json` + return fetch(url).then(res => { + const r1 = res.clone() + return r1.text().then(result => { + expect(result).to.equal('{"name":"value"}') + return res.json().then(result => { + expect(result).to.deep.equal({ name: 'value' }) + }) + }) + }) + }) + + it('should not allow cloning a response after its been used', () => { + const url = `${base}hello` + return fetch(url).then(res => + res.text().then(() => { + expect(() => { + res.clone() + }).to.throw(Error) + }) + ) + }) + + xit('should timeout on cloning response without consuming one of the streams when the second packet size is equal default highWaterMark', function () { + this.timeout(300) + const url = local.mockState(res => { + // Observed behavior of TCP packets splitting: + // - response body size <= 65438 → single packet sent + // - response body size > 65438 → multiple packets sent + // Max TCP packet size is 64kB (http://stackoverflow.com/a/2614188/5763764), + // but first packet probably transfers more than the response body. + const firstPacketMaxSize = 65438 + const secondPacketSize = 16 * 1024 // = defaultHighWaterMark + res.end(crypto.randomBytes(firstPacketMaxSize + secondPacketSize)) + }) + return expect( + fetch(url).then(res => res.clone().buffer()) + ).to.timeout + }) + + xit('should timeout on cloning response without consuming one of the streams when the second packet size is equal custom highWaterMark', function () { + this.timeout(300) + const url = local.mockState(res => { + const firstPacketMaxSize = 65438 + const secondPacketSize = 10 + res.end(crypto.randomBytes(firstPacketMaxSize + secondPacketSize)) + }) + return expect( + fetch(url, { highWaterMark: 10 }).then(res => res.clone().buffer()) + ).to.timeout + }) + + xit('should not timeout on cloning response without consuming one of the streams when the second packet size is less than default highWaterMark', function () { + // TODO: fix test. + if (!isNodeLowerThan('v16.0.0')) { + this.skip() + } + + this.timeout(300) + const url = local.mockState(res => { + const firstPacketMaxSize = 65438 + const secondPacketSize = 16 * 1024 // = defaultHighWaterMark + res.end(crypto.randomBytes(firstPacketMaxSize + secondPacketSize - 1)) + }) + return expect( + fetch(url).then(res => res.clone().buffer()) + ).not.to.timeout + }) + + xit('should not timeout on cloning response without consuming one of the streams when the second packet size is less than custom highWaterMark', function () { + // TODO: fix test. + if (!isNodeLowerThan('v16.0.0')) { + this.skip() + } + + this.timeout(300) + const url = local.mockState(res => { + const firstPacketMaxSize = 65438 + const secondPacketSize = 10 + res.end(crypto.randomBytes(firstPacketMaxSize + secondPacketSize - 1)) + }) + return expect( + fetch(url, { highWaterMark: 10 }).then(res => res.clone().buffer()) + ).not.to.timeout + }) + + xit('should not timeout on cloning response without consuming one of the streams when the response size is double the custom large highWaterMark - 1', function () { + // TODO: fix test. + if (!isNodeLowerThan('v16.0.0')) { + this.skip() + } + + this.timeout(300) + const url = local.mockState(res => { + res.end(crypto.randomBytes((2 * 512 * 1024) - 1)) + }) + return expect( + fetch(url, { highWaterMark: 512 * 1024 }).then(res => res.clone().buffer()) + ).not.to.timeout + }) + + it('should allow get all responses of a header', () => { + const url = `${base}cookie` + return fetch(url).then(res => { + const expected = 'a=1, b=1' + expect(res.headers.get('set-cookie')).to.equal(expected) + expect(res.headers.get('Set-Cookie')).to.equal(expected) + }) + }) + + it('should support fetch with Request instance', () => { + const url = `${base}hello` + const request = new Request(url) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should support fetch with Node.js URL object', () => { + const url = `${base}hello` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should support fetch with WHATWG URL object', () => { + const url = `${base}hello` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should keep `?` sign in URL when no params are given', () => { + const url = `${base}question?` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('if params are given, do not modify anything', () => { + const url = `${base}question?a=1` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should preserve the hash (#) symbol', () => { + const url = `${base}question?#` + const urlObject = new URL(url) + const request = new Request(urlObject) + return fetch(request).then(res => { + expect(res.url).to.equal(url) + expect(res.ok).to.be.true + expect(res.status).to.equal(200) + }) + }) + + it('should support reading blob as text', () => { + return new Response('hello') + .blob() + .then(blob => blob.text()) + .then(body => { + expect(body).to.equal('hello') + }) + }) + + it('should support reading blob as arrayBuffer', () => { + return new Response('hello') + .blob() + .then(blob => blob.arrayBuffer()) + .then(ab => { + const string = String.fromCharCode.apply(null, new Uint8Array(ab)) + expect(string).to.equal('hello') + }) + }) + + it('should support blob round-trip', () => { + const url = `${base}hello` + + let length + let type + + return fetch(url).then(res => res.blob()).then(async blob => { + const url = `${base}inspect` + length = blob.size + type = blob.type + return fetch(url, { + method: 'POST', + body: blob + }) + }).then(res => res.json()).then(({ body, headers }) => { + expect(body).to.equal('world') + expect(headers['content-type']).to.equal(type) + expect(headers['content-length']).to.equal(String(length)) + }) + }) + + it('should support overwrite Request instance', () => { + const url = `${base}inspect` + const request = new Request(url, { + method: 'POST', + headers: { + a: '1' + } + }) + return fetch(request, { + method: 'GET', + headers: { + a: '2' + } + }).then(res => { + return res.json() + }).then(body => { + expect(body.method).to.equal('GET') + expect(body.headers.a).to.equal('2') + }) + }) + + it('should support http request', function () { + this.timeout(5000) + const url = 'https://github.com/' + const options = { + method: 'HEAD' + } + return fetch(url, options).then(res => { + expect(res.status).to.equal(200) + expect(res.ok).to.be.true + }) + }) + + it('should encode URLs as UTF-8', async () => { + const url = `${base}möbius` + const res = await fetch(url) + expect(res.url).to.equal(`${base}m%C3%B6bius`) + }) +}) diff --git a/test/node-fetch/request.js b/test/node-fetch/request.js new file mode 100644 index 00000000000..4cf075536d0 --- /dev/null +++ b/test/node-fetch/request.js @@ -0,0 +1,259 @@ +const stream = require('stream') +const http = require('http') + +const AbortController = require('abort-controller') +const chai = require('chai') +const { Blob } = require('buffer') + +const Request = require('../../lib/fetch/request.js').Request +const TestServer = require('./utils/server.js') + +const { expect } = chai + +describe('Request', () => { + const local = new TestServer() + let base + + before(async () => { + await local.start() + base = `http://${local.hostname}:${local.port}/` + }) + + after(async () => { + return local.stop() + }) + + it('should have attributes conforming to Web IDL', () => { + const request = new Request('http://github.com/') + const enumerableProperties = [] + for (const property in request) { + enumerableProperties.push(property) + } + + for (const toCheck of [ + 'body', + 'bodyUsed', + 'arrayBuffer', + 'blob', + 'json', + 'text', + 'method', + 'url', + 'headers', + 'redirect', + 'clone', + 'signal' + ]) { + expect(enumerableProperties).to.contain(toCheck) + } + + // for (const toCheck of [ + // 'body', 'bodyUsed', 'method', 'url', 'headers', 'redirect', 'signal' + // ]) { + // expect(() => { + // request[toCheck] = 'abc' + // }).to.throw() + // } + }) + + // it('should support wrapping Request instance', () => { + // const url = `${base}hello` + + // const form = new FormData() + // form.append('a', '1') + // const { signal } = new AbortController() + + // const r1 = new Request(url, { + // method: 'POST', + // follow: 1, + // body: form, + // signal + // }) + // const r2 = new Request(r1, { + // follow: 2 + // }) + + // expect(r2.url).to.equal(url) + // expect(r2.method).to.equal('POST') + // expect(r2.signal).to.equal(signal) + // // Note that we didn't clone the body + // expect(r2.body).to.equal(form) + // expect(r1.follow).to.equal(1) + // expect(r2.follow).to.equal(2) + // expect(r1.counter).to.equal(0) + // expect(r2.counter).to.equal(0) + // }) + + xit('should override signal on derived Request instances', () => { + const parentAbortController = new AbortController() + const derivedAbortController = new AbortController() + const parentRequest = new Request(`${base}hello`, { + signal: parentAbortController.signal + }) + const derivedRequest = new Request(parentRequest, { + signal: derivedAbortController.signal + }) + expect(parentRequest.signal).to.equal(parentAbortController.signal) + expect(derivedRequest.signal).to.equal(derivedAbortController.signal) + }) + + xit('should allow removing signal on derived Request instances', () => { + const parentAbortController = new AbortController() + const parentRequest = new Request(`${base}hello`, { + signal: parentAbortController.signal + }) + const derivedRequest = new Request(parentRequest, { + signal: null + }) + expect(parentRequest.signal).to.equal(parentAbortController.signal) + expect(derivedRequest.signal).to.equal(null) + }) + + it('should throw error with GET/HEAD requests with body', () => { + expect(() => new Request(base, { body: '' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: 'a' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: '', method: 'HEAD' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: 'a', method: 'HEAD' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: 'a', method: 'get' })) + .to.throw(TypeError) + expect(() => new Request(base, { body: 'a', method: 'head' })) + .to.throw(TypeError) + }) + + it('should default to null as body', () => { + const request = new Request(base) + expect(request.body).to.equal(null) + return request.text().then(result => expect(result).to.equal('')) + }) + + it('should support parsing headers', () => { + const url = base + const request = new Request(url, { + headers: { + a: '1' + } + }) + expect(request.url).to.equal(url) + expect(request.headers.get('a')).to.equal('1') + }) + + it('should support arrayBuffer() method', () => { + const url = base + const request = new Request(url, { + method: 'POST', + body: 'a=1' + }) + expect(request.url).to.equal(url) + return request.arrayBuffer().then(result => { + expect(result).to.be.an.instanceOf(ArrayBuffer) + const string = String.fromCharCode.apply(null, new Uint8Array(result)) + expect(string).to.equal('a=1') + }) + }) + + it('should support text() method', () => { + const url = base + const request = new Request(url, { + method: 'POST', + body: 'a=1' + }) + expect(request.url).to.equal(url) + return request.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support json() method', () => { + const url = base + const request = new Request(url, { + method: 'POST', + body: '{"a":1}' + }) + expect(request.url).to.equal(url) + return request.json().then(result => { + expect(result.a).to.equal(1) + }) + }) + + it('should support blob() method', () => { + const url = base + const request = new Request(url, { + method: 'POST', + body: Buffer.from('a=1') + }) + expect(request.url).to.equal(url) + return request.blob().then(result => { + expect(result).to.be.an.instanceOf(Blob) + expect(result.size).to.equal(3) + expect(result.type).to.equal('') + }) + }) + + xit('should support clone() method', () => { + const url = base + const body = stream.Readable.from('a=1') + const agent = new http.Agent() + const { signal } = new AbortController() + const request = new Request(url, { + body, + method: 'POST', + redirect: 'manual', + headers: { + b: '2' + }, + follow: 3, + compress: false, + agent, + signal + }) + const cl = request.clone() + expect(cl.url).to.equal(url) + expect(cl.method).to.equal('POST') + expect(cl.redirect).to.equal('manual') + expect(cl.headers.get('b')).to.equal('2') + expect(cl.method).to.equal('POST') + // Clone body shouldn't be the same body + expect(cl.body).to.not.equal(body) + return Promise.all([cl.text(), request.text()]).then(results => { + expect(results[0]).to.equal('a=1') + expect(results[1]).to.equal('a=1') + }) + }) + + it('should support ArrayBuffer as body', () => { + const encoder = new TextEncoder() + const request = new Request(base, { + method: 'POST', + body: encoder.encode('a=1').buffer + }) + return request.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support Uint8Array as body', () => { + const encoder = new TextEncoder() + const request = new Request(base, { + method: 'POST', + body: encoder.encode('a=1') + }) + return request.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support DataView as body', () => { + const encoder = new TextEncoder() + const request = new Request(base, { + method: 'POST', + body: new DataView(encoder.encode('a=1').buffer) + }) + return request.text().then(result => { + expect(result).to.equal('a=1') + }) + }) +}) diff --git a/test/node-fetch/response.js b/test/node-fetch/response.js new file mode 100644 index 00000000000..86153873c3a --- /dev/null +++ b/test/node-fetch/response.js @@ -0,0 +1,217 @@ +/* eslint no-unused-expressions: "off" */ + +const chai = require('chai') +const stream = require('stream') +const { Response } = require('../../lib/fetch/response.js') +const TestServer = require('./utils/server.js') +const { Blob } = require('buffer') +const { kState } = require('../../lib/fetch/symbols.js') + +const { expect } = chai + +describe('Response', () => { + const local = new TestServer() + let base + + before(async () => { + await local.start() + base = `http://${local.hostname}:${local.port}/` + }) + + after(async () => { + return local.stop() + }) + + it('should have attributes conforming to Web IDL', () => { + const res = new Response() + const enumerableProperties = [] + for (const property in res) { + enumerableProperties.push(property) + } + + for (const toCheck of [ + 'body', + 'bodyUsed', + 'arrayBuffer', + 'blob', + 'json', + 'text', + 'type', + 'url', + 'status', + 'ok', + 'redirected', + 'statusText', + 'headers', + 'clone' + ]) { + expect(enumerableProperties).to.contain(toCheck) + } + + // TODO + // for (const toCheck of [ + // 'body', + // 'bodyUsed', + // 'type', + // 'url', + // 'status', + // 'ok', + // 'redirected', + // 'statusText', + // 'headers' + // ]) { + // expect(() => { + // res[toCheck] = 'abc' + // }).to.throw() + // } + }) + + it('should support empty options', () => { + const res = new Response(stream.Readable.from('a=1')) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support parsing headers', () => { + const res = new Response(null, { + headers: { + a: '1' + } + }) + expect(res.headers.get('a')).to.equal('1') + }) + + it('should support text() method', () => { + const res = new Response('a=1') + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support json() method', () => { + const res = new Response('{"a":1}') + return res.json().then(result => { + expect(result.a).to.equal(1) + }) + }) + + if (Blob) { + it('should support blob() method', () => { + const res = new Response('a=1', { + method: 'POST', + headers: { + 'Content-Type': 'text/plain' + } + }) + return res.blob().then(result => { + expect(result).to.be.an.instanceOf(Blob) + expect(result.size).to.equal(3) + expect(result.type).to.equal('text/plain') + }) + }) + } + + xit('should support clone() method', () => { + const body = stream.Readable.from('a=1') + const res = new Response(body, { + headers: { + a: '1' + }, + status: 346, + statusText: 'production' + }) + res[kState].urlList = [base] + const cl = res.clone() + expect(cl.headers.get('a')).to.equal('1') + expect(cl.type).to.equal('default') + expect(cl.url).to.equal(base) + expect(cl.status).to.equal(346) + expect(cl.statusText).to.equal('production') + expect(cl.ok).to.be.false + // Clone body shouldn't be the same body + expect(cl.body).to.not.equal(body) + return Promise.all([cl.text(), res.text()]).then(results => { + expect(results[0]).to.equal('a=1') + expect(results[1]).to.equal('a=1') + }) + }) + + it('should support stream as body', () => { + const body = stream.Readable.from('a=1') + const res = new Response(body) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support string as body', () => { + const res = new Response('a=1') + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support buffer as body', () => { + const res = new Response(Buffer.from('a=1')) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support ArrayBuffer as body', () => { + const encoder = new TextEncoder() + const res = new Response(encoder.encode('a=1')) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support blob as body', async () => { + const res = new Response(new Blob(['a=1'])) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support Uint8Array as body', () => { + const encoder = new TextEncoder() + const res = new Response(encoder.encode('a=1')) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should support DataView as body', () => { + const encoder = new TextEncoder() + const res = new Response(new DataView(encoder.encode('a=1').buffer)) + return res.text().then(result => { + expect(result).to.equal('a=1') + }) + }) + + it('should default to null as body', () => { + const res = new Response() + expect(res.body).to.equal(null) + + return res.text().then(result => expect(result).to.equal('')) + }) + + it('should default to 200 as status code', () => { + const res = new Response(null) + expect(res.status).to.equal(200) + }) + + it('should default to empty string as url', () => { + const res = new Response() + expect(res.url).to.equal('') + }) + + it('should support error() static method', () => { + const res = Response.error() + expect(res).to.be.an.instanceof(Response) + expect(res.type).to.equal('error') + expect(res.status).to.equal(0) + expect(res.statusText).to.equal('') + }) +}) diff --git a/test/node-fetch/utils/chai-timeout.js b/test/node-fetch/utils/chai-timeout.js new file mode 100644 index 00000000000..6838a4cc322 --- /dev/null +++ b/test/node-fetch/utils/chai-timeout.js @@ -0,0 +1,15 @@ +const pTimeout = require('p-timeout') + +module.exports = ({ Assertion }, utils) => { + utils.addProperty(Assertion.prototype, 'timeout', async function () { + let timeouted = false + await pTimeout(this._obj, 150, () => { + timeouted = true + }) + return this.assert( + timeouted, + 'expected promise to timeout but it was resolved', + 'expected promise not to timeout but it timed out' + ) + }) +} diff --git a/test/node-fetch/utils/dummy.txt b/test/node-fetch/utils/dummy.txt new file mode 100644 index 00000000000..5ca51916b39 --- /dev/null +++ b/test/node-fetch/utils/dummy.txt @@ -0,0 +1 @@ +i am a dummy \ No newline at end of file diff --git a/test/node-fetch/utils/read-stream.js b/test/node-fetch/utils/read-stream.js new file mode 100644 index 00000000000..7d791532934 --- /dev/null +++ b/test/node-fetch/utils/read-stream.js @@ -0,0 +1,9 @@ +module.exports = async function readStream (stream) { + const chunks = [] + + for await (const chunk of stream) { + chunks.push(chunk instanceof Buffer ? chunk : Buffer.from(chunk)) + } + + return Buffer.concat(chunks) +} diff --git a/test/node-fetch/utils/server.js b/test/node-fetch/utils/server.js new file mode 100644 index 00000000000..92426be3e15 --- /dev/null +++ b/test/node-fetch/utils/server.js @@ -0,0 +1,437 @@ +const http = require('http') +const zlib = require('zlib') +const { once } = require('events') +const Busboy = require('busboy') + +module.exports = class TestServer { + constructor () { + this.server = http.createServer(this.router) + // Node 8 default keepalive timeout is 5000ms + // make it shorter here as we want to close server quickly at the end of tests + this.server.keepAliveTimeout = 1000 + this.server.on('error', err => { + console.log(err.stack) + }) + this.server.on('connection', socket => { + socket.setTimeout(1500) + }) + } + + async start () { + this.server.listen(0, 'localhost') + return once(this.server, 'listening') + } + + async stop () { + this.server.close() + return once(this.server, 'close') + } + + get port () { + return this.server.address().port + } + + get hostname () { + return 'localhost' + } + + mockState (responseHandler) { + this.server.nextResponseHandler = responseHandler + return `http://${this.hostname}:${this.port}/mocked` + } + + router (request, res) { + const p = request.url + + if (p === '/mocked') { + if (this.nextResponseHandler) { + this.nextResponseHandler(res) + this.nextResponseHandler = undefined + } else { + throw new Error('No mocked response. Use ’TestServer.mockState()’.') + } + } + + if (p === '/hello') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('world') + } + + if (p.includes('question')) { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('ok') + } + + if (p === '/plain') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('text') + } + + if (p === '/no-status-text') { + res.writeHead(200, '', {}).end() + } + + if (p === '/options') { + res.statusCode = 200 + res.setHeader('Allow', 'GET, HEAD, OPTIONS') + res.end('hello world') + } + + if (p === '/html') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/html') + res.end('') + } + + if (p === '/json') { + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.end(JSON.stringify({ + name: 'value' + })) + } + + if (p === '/gzip') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'gzip') + zlib.gzip('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + + if (p === '/gzip-truncated') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'gzip') + zlib.gzip('hello world', (err, buffer) => { + if (err) { + throw err + } + + // Truncate the CRC checksum and size check at the end of the stream + res.end(buffer.slice(0, -8)) + }) + } + + if (p === '/gzip-capital') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'GZip') + zlib.gzip('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + + if (p === '/deflate') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'deflate') + zlib.deflate('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + + if (p === '/brotli') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + if (typeof zlib.createBrotliDecompress === 'function') { + res.setHeader('Content-Encoding', 'br') + zlib.brotliCompress('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + } + + if (p === '/deflate-raw') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'deflate') + zlib.deflateRaw('hello world', (err, buffer) => { + if (err) { + throw err + } + + res.end(buffer) + }) + } + + if (p === '/sdch') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'sdch') + res.end('fake sdch string') + } + + if (p === '/invalid-content-encoding') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.setHeader('Content-Encoding', 'gzip') + res.end('fake gzip string') + } + + if (p === '/timeout') { + setTimeout(() => { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('text') + }, 1000) + } + + if (p === '/slow') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.write('test') + setTimeout(() => { + res.end('test') + }, 1000) + } + + if (p === '/cookie') { + res.statusCode = 200 + res.setHeader('Set-Cookie', ['a=1', 'b=1']) + res.end('cookie') + } + + if (p === '/size/chunk') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + setTimeout(() => { + res.write('test') + }, 10) + setTimeout(() => { + res.end('test') + }, 20) + } + + if (p === '/size/long') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('testtest') + } + + if (p === '/redirect/301') { + res.statusCode = 301 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/302') { + res.statusCode = 302 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/303') { + res.statusCode = 303 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/307') { + res.statusCode = 307 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/308') { + res.statusCode = 308 + res.setHeader('Location', '/inspect') + res.end() + } + + if (p === '/redirect/chain') { + res.statusCode = 301 + res.setHeader('Location', '/redirect/301') + res.end() + } + + if (p.startsWith('/redirect/chain/')) { + const count = parseInt(p.split('/').pop()) - 1 + res.statusCode = 301 + res.setHeader('Location', count ? `/redirect/chain/${count}` : '/redirect/301') + res.end() + } + + if (p === '/redirect/no-location') { + res.statusCode = 301 + res.end() + } + + if (p === '/redirect/slow') { + res.statusCode = 301 + res.setHeader('Location', '/redirect/301') + setTimeout(() => { + res.end() + }, 1000) + } + + if (p === '/redirect/slow-chain') { + res.statusCode = 301 + res.setHeader('Location', '/redirect/slow') + setTimeout(() => { + res.end() + }, 10) + } + + if (p === '/redirect/slow-stream') { + res.statusCode = 301 + res.setHeader('Location', '/slow') + res.end() + } + + if (p === '/redirect/bad-location') { + res.socket.write('HTTP/1.1 301\r\nLocation: ☃\r\nContent-Length: 0\r\n') + res.socket.end('\r\n') + } + + if (p === '/error/400') { + res.statusCode = 400 + res.setHeader('Content-Type', 'text/plain') + res.end('client error') + } + + if (p === '/error/404') { + res.statusCode = 404 + res.setHeader('Content-Encoding', 'gzip') + res.end() + } + + if (p === '/error/500') { + res.statusCode = 500 + res.setHeader('Content-Type', 'text/plain') + res.end('server error') + } + + if (p === '/error/reset') { + res.destroy() + } + + if (p === '/error/premature') { + res.writeHead(200, { 'content-length': 50 }) + res.write('foo') + setTimeout(() => { + res.destroy() + }, 100) + } + + if (p === '/error/premature/chunked') { + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked' + }) + + res.write(`${JSON.stringify({ data: 'hi' })}\n`) + + setTimeout(() => { + res.write(`${JSON.stringify({ data: 'bye' })}\n`) + }, 200) + + setTimeout(() => { + res.destroy() + }, 400) + } + + if (p === '/error/json') { + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + res.end('invalid json') + } + + if (p === '/no-content') { + res.statusCode = 204 + res.end() + } + + if (p === '/no-content/gzip') { + res.statusCode = 204 + res.setHeader('Content-Encoding', 'gzip') + res.end() + } + + if (p === '/no-content/brotli') { + res.statusCode = 204 + res.setHeader('Content-Encoding', 'br') + res.end() + } + + if (p === '/not-modified') { + res.statusCode = 304 + res.end() + } + + if (p === '/not-modified/gzip') { + res.statusCode = 304 + res.setHeader('Content-Encoding', 'gzip') + res.end() + } + + if (p === '/inspect') { + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + let body = '' + request.on('data', c => { + body += c + }) + request.on('end', () => { + res.end(JSON.stringify({ + method: request.method, + url: request.url, + headers: request.headers, + body + })) + }) + } + + if (p === '/multipart') { + res.statusCode = 200 + res.setHeader('Content-Type', 'application/json') + const busboy = new Busboy({ headers: request.headers }) + let body = '' + busboy.on('file', async (fieldName, file, fileName) => { + body += `${fieldName}=${fileName}` + // consume file data + // eslint-disable-next-line no-empty, no-unused-vars + for await (const c of file) {} + }) + + busboy.on('field', (fieldName, value) => { + body += `${fieldName}=${value}` + }) + busboy.on('finish', () => { + res.end(JSON.stringify({ + method: request.method, + url: request.url, + headers: request.headers, + body + })) + }) + request.pipe(busboy) + } + + if (p === '/m%C3%B6bius') { + res.statusCode = 200 + res.setHeader('Content-Type', 'text/plain') + res.end('ok') + } + } +}