const { Request, Response } = require('minipass-fetch')
const Minipass = require('minipass')
const MinipassFlush = require('minipass-flush')
const cacache = require('cacache')
const url = require('url')
const CachingMinipassPipeline = require('../pipeline.js')
const CachePolicy = require('./policy.js')
const cacheKey = require('./key.js')
const remote = require('../remote.js')
const hasOwnProperty = (obj, prop) => Object.prototype.hasOwnProperty.call(obj, prop)
// allow list for request headers that will be written to the cache index
// note: we will also store any request headers
// that are named in a response's vary header
const KEEP_REQUEST_HEADERS = [
'accept-charset',
'accept-encoding',
'accept-language',
'accept',
'cache-control',
]
// allow list for response headers that will be written to the cache index
// note: we must not store the real response's age header, or when we load
// a cache policy based on the metadata it will think the cached response
// is always stale
const KEEP_RESPONSE_HEADERS = [
'cache-control',
'content-encoding',
'content-language',
'content-type',
'date',
'etag',
'expires',
'last-modified',
'link',
'location',
'pragma',
'vary',
]
// return an object containing all metadata to be written to the index
const getMetadata = (request, response, options) => {
const metadata = {
time: Date.now(),
url: request.url,
reqHeaders: {},
resHeaders: {},
// options on which we must match the request and vary the response
options: {
compress: options.compress != null ? options.compress : request.compress,
},
}
// only save the status if it's not a 200 or 304
if (response.status !== 200 && response.status !== 304) {
metadata.status = response.status
}
for (const name of KEEP_REQUEST_HEADERS) {
if (request.headers.has(name)) {
metadata.reqHeaders[name] = request.headers.get(name)
}
}
// if the request's host header differs from the host in the url
// we need to keep it, otherwise it's just noise and we ignore it
const host = request.headers.get('host')
const parsedUrl = new url.URL(request.url)
if (host && parsedUrl.host !== host) {
metadata.reqHeaders.host = host
}
// if the response has a vary header, make sure
// we store the relevant request headers too
if (response.headers.has('vary')) {
const vary = response.headers.get('vary')
// a vary of "*" means every header causes a different response.
// in that scenario, we do not include any additional headers
// as the freshness check will always fail anyway and we don't
// want to bloat the cache indexes
if (vary !== '*') {
// copy any other request headers that will vary the response
const varyHeaders = vary.trim().toLowerCase().split(/\s*,\s*/)
for (const name of varyHeaders) {
if (request.headers.has(name)) {
metadata.reqHeaders[name] = request.headers.get(name)
}
}
}
}
for (const name of KEEP_RESPONSE_HEADERS) {
if (response.headers.has(name)) {
metadata.resHeaders[name] = response.headers.get(name)
}
}
return metadata
}
// symbols used to hide objects that may be lazily evaluated in a getter
const _request = Symbol('request')
const _response = Symbol('response')
const _policy = Symbol('policy')
class CacheEntry {
constructor ({ entry, request, response, options }) {
if (entry) {
this.key = entry.key
this.entry = entry
// previous versions of this module didn't write an explicit timestamp in
// the metadata, so fall back to the entry's timestamp. we can't use the
// entry timestamp to determine staleness because cacache will update it
// when it verifies its data
this.entry.metadata.time = this.entry.metadata.time || this.entry.time
} else {
this.key = cacheKey(request)
}
this.options = options
// these properties are behind getters that lazily evaluate
this[_request] = request
this[_response] = response
this[_policy] = null
}
// returns a CacheEntry instance that satisfies the given request
// or undefined if no existing entry satisfies
static async find (request, options) {
try {
// compacts the index and returns an array of unique entries
var matches = await cacache.index.compact(options.cachePath, cacheKey(request), (A, B) => {
const entryA = new CacheEntry({ entry: A, options })
const entryB = new CacheEntry({ entry: B, options })
return entryA.policy.satisfies(entryB.request)
}, {
validateEntry: (entry) => {
// clean out entries with a buggy content-encoding value
if (entry.metadata &&
entry.metadata.resHeaders &&
entry.metadata.resHeaders['content-encoding'] === null) {
return false
}
// if an integrity is null, it needs to have a status specified
if (entry.integrity === null) {
return !!(entry.metadata && entry.metadata.status)
}
return true
},
})
} catch (err) {
// if the compact request fails, ignore the error and return
return
}
// a cache mode of 'reload' means to behave as though we have no cache
// on the way to the network. return undefined to allow cacheFetch to
// create a brand new request no matter what.
if (options.cache === 'reload') {
return
}
// find the specific entry that satisfies the request
let match
for (const entry of matches) {
const _entry = new CacheEntry({
entry,
options,
})
if (_entry.policy.satisfies(request)) {
match = _entry
break
}
}
return match
}
// if the user made a PUT/POST/PATCH then we invalidate our
// cache for the same url by deleting the index entirely
static async invalidate (request, options) {
const key = cacheKey(request)
try {
await cacache.rm.entry(options.cachePath, key, { removeFully: true })
} catch (err) {
// ignore errors
}
}
get request () {
if (!this[_request]) {
this[_request] = new Request(this.entry.metadata.url, {
method: 'GET',
headers: this.entry.metadata.reqHeaders,
...this.entry.metadata.options,
})
}
return this[_request]
}
get response () {
if (!this[_response]) {
this[_response] = new Response(null, {
url: this.entry.metadata.url,
counter: this.options.counter,
status: this.entry.metadata.status || 200,
headers: {
...this.entry.metadata.resHeaders,
'content-length': this.entry.size,
},
})
}
return this[_response]
}
get policy () {
if (!this[_policy]) {
this[_policy] = new CachePolicy({
entry: this.entry,
request: this.request,
response: this.response,
options: this.options,
})
}
return this[_policy]
}
// wraps the response in a pipeline that stores the data
// in the cache while the user consumes it
async store (status) {
// if we got a status other than 200, 301, or 308,
// or the CachePolicy forbid storage, append the
// cache status header and return it untouched
if (
this.request.method !== 'GET' ||
![200, 301, 308].includes(this.response.status) ||
!this.policy.storable()
) {
this.response.headers.set('x-local-cache-status', 'skip')
return this.response
}
const size = this.response.headers.get('content-length')
const cacheOpts = {
algorithms: this.options.algorithms,
metadata: getMetadata(this.request, this.response, this.options),
size,
integrity: this.options.integrity,
integrityEmitter: this.response.body.hasIntegrityEmitter && this.response.body,
}
let body = null
// we only set a body if the status is a 200, redirects are
// stored as metadata only
if (this.response.status === 200) {
let cacheWriteResolve, cacheWriteReject
const cacheWritePromise = new Promise((resolve, reject) => {
cacheWriteResolve = resolve
cacheWriteReject = reject
})
body = new CachingMinipassPipeline({ events: ['integrity', 'size'] }, new MinipassFlush({
flush () {
return cacheWritePromise
},
}))
// this is always true since if we aren't reusing the one from the remote fetch, we
// are using the one from cacache
body.hasIntegrityEmitter = true
const onResume = () => {
const tee = new Minipass()
const cacheStream = cacache.put.stream(this.options.cachePath, this.key, cacheOpts)
// re-emit the integrity and size events on our new response body so they can be reused
cacheStream.on('integrity', i => body.emit('integrity', i))
cacheStream.on('size', s => body.emit('size', s))
// stick a flag on here so downstream users will know if they can expect integrity events
tee.pipe(cacheStream)
// TODO if the cache write fails, log a warning but return the response anyway
// eslint-disable-next-line promise/catch-or-return
cacheStream.promise().then(cacheWriteResolve, cacheWriteReject)
body.unshift(tee)
body.unshift(this.response.body)
}
body.once('resume', onResume)
body.once('end', () => body.removeListener('resume', onResume))
} else {
await cacache.index.insert(this.options.cachePath, this.key, null, cacheOpts)
}
// note: we do not set the x-local-cache-hash header because we do not know
// the hash value until after the write to the cache completes, which doesn't
// happen until after the response has been sent and it's too late to write
// the header anyway
this.response.headers.set('x-local-cache', encodeURIComponent(this.options.cachePath))
this.response.headers.set('x-local-cache-key', encodeURIComponent(this.key))
this.response.headers.set('x-local-cache-mode', 'stream')
this.response.headers.set('x-local-cache-status', status)
this.response.headers.set('x-local-cache-time', new Date().toISOString())
const newResponse = new Response(body, {
url: this.response.url,
status: this.response.status,
headers: this.response.headers,
counter: this.options.counter,
})
return newResponse
}
// use the cached data to create a response and return it
async respond (method, options, status) {
let response
if (method === 'HEAD' || [301, 308].includes(this.response.status)) {
// if the request is a HEAD, or the response is a redirect,
// then the metadata in the entry already includes everything
// we need to build a response
response = this.response
} else {
// we're responding with a full cached response, so create a body
// that reads from cacache and attach it to a new Response
const body = new Minipass()
const headers = { ...this.policy.responseHeaders() }
const onResume = () => {
const cacheStream = cacache.get.stream.byDigest(
this.options.cachePath, this.entry.integrity, { memoize: this.options.memoize }
)
cacheStream.on('error', async (err) => {
cacheStream.pause()
if (err.code === 'EINTEGRITY') {
await cacache.rm.content(
this.options.cachePath, this.entry.integrity, { memoize: this.options.memoize }
)
}
if (err.code === 'ENOENT' || err.code === 'EINTEGRITY') {
await CacheEntry.invalidate(this.request, this.options)
}
body.emit('error', err)
cacheStream.resume()
})
// emit the integrity and size events based on our metadata so we're consistent
body.emit('integrity', this.entry.integrity)
body.emit('size', Number(headers['content-length']))
cacheStream.pipe(body)
}
body.once('resume', onResume)
body.once('end', () => body.removeListener('resume', onResume))
response = new Response(body, {
url: this.entry.metadata.url,
counter: options.counter,
status: 200,
headers,
})
}
response.headers.set('x-local-cache', encodeURIComponent(this.options.cachePath))
response.headers.set('x-local-cache-hash', encodeURIComponent(this.entry.integrity))
response.headers.set('x-local-cache-key', encodeURIComponent(this.key))
response.headers.set('x-local-cache-mode', 'stream')
response.headers.set('x-local-cache-status', status)
response.headers.set('x-local-cache-time', new Date(this.entry.metadata.time).toUTCString())
return response
}
// use the provided request along with this cache entry to
// revalidate the stored response. returns a response, either
// from the cache or from the update
async revalidate (request, options) {
const revalidateRequest = new Request(request, {
headers: this.policy.revalidationHeaders(request),
})
try {
// NOTE: be sure to remove the headers property from the
// user supplied options, since we have already defined
// them on the new request object. if they're still in the
// options then those will overwrite the ones from the policy
var response = await remote(revalidateRequest, {
...options,
headers: undefined,
})
} catch (err) {
// if the network fetch fails, return the stale
// cached response unless it has a cache-control
// of 'must-revalidate'
if (!this.policy.mustRevalidate) {
return this.respond(request.method, options, 'stale')
}
throw err
}
if (this.policy.revalidated(revalidateRequest, response)) {
// we got a 304, write a new index to the cache and respond from cache
const metadata = getMetadata(request, response, options)
// 304 responses do not include headers that are specific to the response data
// since they do not include a body, so we copy values for headers that were
// in the old cache entry to the new one, if the new metadata does not already
// include that header
for (const name of KEEP_RESPONSE_HEADERS) {
if (
!hasOwnProperty(metadata.resHeaders, name) &&
hasOwnProperty(this.entry.metadata.resHeaders, name)
) {
metadata.resHeaders[name] = this.entry.metadata.resHeaders[name]
}
}
try {
await cacache.index.insert(options.cachePath, this.key, this.entry.integrity, {
size: this.entry.size,
metadata,
})
} catch (err) {
// if updating the cache index fails, we ignore it and
// respond anyway
}
return this.respond(request.method, options, 'revalidated')
}
// if we got a modified response, create a new entry based on it
const newEntry = new CacheEntry({
request,
response,
options,
})
// respond with the new entry while writing it to the cache
return newEntry.store('updated')
}
}
module.exports = CacheEntry
|