entry.js 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444
  1. const { Request, Response } = require('minipass-fetch')
  2. const Minipass = require('minipass')
  3. const MinipassFlush = require('minipass-flush')
  4. const cacache = require('cacache')
  5. const url = require('url')
  6. const CachingMinipassPipeline = require('../pipeline.js')
  7. const CachePolicy = require('./policy.js')
  8. const cacheKey = require('./key.js')
  9. const remote = require('../remote.js')
  10. const hasOwnProperty = (obj, prop) => Object.prototype.hasOwnProperty.call(obj, prop)
  11. // allow list for request headers that will be written to the cache index
  12. // note: we will also store any request headers
  13. // that are named in a response's vary header
  14. const KEEP_REQUEST_HEADERS = [
  15. 'accept-charset',
  16. 'accept-encoding',
  17. 'accept-language',
  18. 'accept',
  19. 'cache-control',
  20. ]
  21. // allow list for response headers that will be written to the cache index
  22. // note: we must not store the real response's age header, or when we load
  23. // a cache policy based on the metadata it will think the cached response
  24. // is always stale
  25. const KEEP_RESPONSE_HEADERS = [
  26. 'cache-control',
  27. 'content-encoding',
  28. 'content-language',
  29. 'content-type',
  30. 'date',
  31. 'etag',
  32. 'expires',
  33. 'last-modified',
  34. 'link',
  35. 'location',
  36. 'pragma',
  37. 'vary',
  38. ]
  39. // return an object containing all metadata to be written to the index
  40. const getMetadata = (request, response, options) => {
  41. const metadata = {
  42. time: Date.now(),
  43. url: request.url,
  44. reqHeaders: {},
  45. resHeaders: {},
  46. // options on which we must match the request and vary the response
  47. options: {
  48. compress: options.compress != null ? options.compress : request.compress,
  49. },
  50. }
  51. // only save the status if it's not a 200 or 304
  52. if (response.status !== 200 && response.status !== 304) {
  53. metadata.status = response.status
  54. }
  55. for (const name of KEEP_REQUEST_HEADERS) {
  56. if (request.headers.has(name)) {
  57. metadata.reqHeaders[name] = request.headers.get(name)
  58. }
  59. }
  60. // if the request's host header differs from the host in the url
  61. // we need to keep it, otherwise it's just noise and we ignore it
  62. const host = request.headers.get('host')
  63. const parsedUrl = new url.URL(request.url)
  64. if (host && parsedUrl.host !== host) {
  65. metadata.reqHeaders.host = host
  66. }
  67. // if the response has a vary header, make sure
  68. // we store the relevant request headers too
  69. if (response.headers.has('vary')) {
  70. const vary = response.headers.get('vary')
  71. // a vary of "*" means every header causes a different response.
  72. // in that scenario, we do not include any additional headers
  73. // as the freshness check will always fail anyway and we don't
  74. // want to bloat the cache indexes
  75. if (vary !== '*') {
  76. // copy any other request headers that will vary the response
  77. const varyHeaders = vary.trim().toLowerCase().split(/\s*,\s*/)
  78. for (const name of varyHeaders) {
  79. if (request.headers.has(name)) {
  80. metadata.reqHeaders[name] = request.headers.get(name)
  81. }
  82. }
  83. }
  84. }
  85. for (const name of KEEP_RESPONSE_HEADERS) {
  86. if (response.headers.has(name)) {
  87. metadata.resHeaders[name] = response.headers.get(name)
  88. }
  89. }
  90. return metadata
  91. }
  92. // symbols used to hide objects that may be lazily evaluated in a getter
  93. const _request = Symbol('request')
  94. const _response = Symbol('response')
  95. const _policy = Symbol('policy')
  96. class CacheEntry {
  97. constructor ({ entry, request, response, options }) {
  98. if (entry) {
  99. this.key = entry.key
  100. this.entry = entry
  101. // previous versions of this module didn't write an explicit timestamp in
  102. // the metadata, so fall back to the entry's timestamp. we can't use the
  103. // entry timestamp to determine staleness because cacache will update it
  104. // when it verifies its data
  105. this.entry.metadata.time = this.entry.metadata.time || this.entry.time
  106. } else {
  107. this.key = cacheKey(request)
  108. }
  109. this.options = options
  110. // these properties are behind getters that lazily evaluate
  111. this[_request] = request
  112. this[_response] = response
  113. this[_policy] = null
  114. }
  115. // returns a CacheEntry instance that satisfies the given request
  116. // or undefined if no existing entry satisfies
  117. static async find (request, options) {
  118. try {
  119. // compacts the index and returns an array of unique entries
  120. var matches = await cacache.index.compact(options.cachePath, cacheKey(request), (A, B) => {
  121. const entryA = new CacheEntry({ entry: A, options })
  122. const entryB = new CacheEntry({ entry: B, options })
  123. return entryA.policy.satisfies(entryB.request)
  124. }, {
  125. validateEntry: (entry) => {
  126. // clean out entries with a buggy content-encoding value
  127. if (entry.metadata &&
  128. entry.metadata.resHeaders &&
  129. entry.metadata.resHeaders['content-encoding'] === null) {
  130. return false
  131. }
  132. // if an integrity is null, it needs to have a status specified
  133. if (entry.integrity === null) {
  134. return !!(entry.metadata && entry.metadata.status)
  135. }
  136. return true
  137. },
  138. })
  139. } catch (err) {
  140. // if the compact request fails, ignore the error and return
  141. return
  142. }
  143. // a cache mode of 'reload' means to behave as though we have no cache
  144. // on the way to the network. return undefined to allow cacheFetch to
  145. // create a brand new request no matter what.
  146. if (options.cache === 'reload') {
  147. return
  148. }
  149. // find the specific entry that satisfies the request
  150. let match
  151. for (const entry of matches) {
  152. const _entry = new CacheEntry({
  153. entry,
  154. options,
  155. })
  156. if (_entry.policy.satisfies(request)) {
  157. match = _entry
  158. break
  159. }
  160. }
  161. return match
  162. }
  163. // if the user made a PUT/POST/PATCH then we invalidate our
  164. // cache for the same url by deleting the index entirely
  165. static async invalidate (request, options) {
  166. const key = cacheKey(request)
  167. try {
  168. await cacache.rm.entry(options.cachePath, key, { removeFully: true })
  169. } catch (err) {
  170. // ignore errors
  171. }
  172. }
  173. get request () {
  174. if (!this[_request]) {
  175. this[_request] = new Request(this.entry.metadata.url, {
  176. method: 'GET',
  177. headers: this.entry.metadata.reqHeaders,
  178. ...this.entry.metadata.options,
  179. })
  180. }
  181. return this[_request]
  182. }
  183. get response () {
  184. if (!this[_response]) {
  185. this[_response] = new Response(null, {
  186. url: this.entry.metadata.url,
  187. counter: this.options.counter,
  188. status: this.entry.metadata.status || 200,
  189. headers: {
  190. ...this.entry.metadata.resHeaders,
  191. 'content-length': this.entry.size,
  192. },
  193. })
  194. }
  195. return this[_response]
  196. }
  197. get policy () {
  198. if (!this[_policy]) {
  199. this[_policy] = new CachePolicy({
  200. entry: this.entry,
  201. request: this.request,
  202. response: this.response,
  203. options: this.options,
  204. })
  205. }
  206. return this[_policy]
  207. }
  208. // wraps the response in a pipeline that stores the data
  209. // in the cache while the user consumes it
  210. async store (status) {
  211. // if we got a status other than 200, 301, or 308,
  212. // or the CachePolicy forbid storage, append the
  213. // cache status header and return it untouched
  214. if (
  215. this.request.method !== 'GET' ||
  216. ![200, 301, 308].includes(this.response.status) ||
  217. !this.policy.storable()
  218. ) {
  219. this.response.headers.set('x-local-cache-status', 'skip')
  220. return this.response
  221. }
  222. const size = this.response.headers.get('content-length')
  223. const cacheOpts = {
  224. algorithms: this.options.algorithms,
  225. metadata: getMetadata(this.request, this.response, this.options),
  226. size,
  227. integrity: this.options.integrity,
  228. integrityEmitter: this.response.body.hasIntegrityEmitter && this.response.body,
  229. }
  230. let body = null
  231. // we only set a body if the status is a 200, redirects are
  232. // stored as metadata only
  233. if (this.response.status === 200) {
  234. let cacheWriteResolve, cacheWriteReject
  235. const cacheWritePromise = new Promise((resolve, reject) => {
  236. cacheWriteResolve = resolve
  237. cacheWriteReject = reject
  238. })
  239. body = new CachingMinipassPipeline({ events: ['integrity', 'size'] }, new MinipassFlush({
  240. flush () {
  241. return cacheWritePromise
  242. },
  243. }))
  244. // this is always true since if we aren't reusing the one from the remote fetch, we
  245. // are using the one from cacache
  246. body.hasIntegrityEmitter = true
  247. const onResume = () => {
  248. const tee = new Minipass()
  249. const cacheStream = cacache.put.stream(this.options.cachePath, this.key, cacheOpts)
  250. // re-emit the integrity and size events on our new response body so they can be reused
  251. cacheStream.on('integrity', i => body.emit('integrity', i))
  252. cacheStream.on('size', s => body.emit('size', s))
  253. // stick a flag on here so downstream users will know if they can expect integrity events
  254. tee.pipe(cacheStream)
  255. // TODO if the cache write fails, log a warning but return the response anyway
  256. // eslint-disable-next-line promise/catch-or-return
  257. cacheStream.promise().then(cacheWriteResolve, cacheWriteReject)
  258. body.unshift(tee)
  259. body.unshift(this.response.body)
  260. }
  261. body.once('resume', onResume)
  262. body.once('end', () => body.removeListener('resume', onResume))
  263. } else {
  264. await cacache.index.insert(this.options.cachePath, this.key, null, cacheOpts)
  265. }
  266. // note: we do not set the x-local-cache-hash header because we do not know
  267. // the hash value until after the write to the cache completes, which doesn't
  268. // happen until after the response has been sent and it's too late to write
  269. // the header anyway
  270. this.response.headers.set('x-local-cache', encodeURIComponent(this.options.cachePath))
  271. this.response.headers.set('x-local-cache-key', encodeURIComponent(this.key))
  272. this.response.headers.set('x-local-cache-mode', 'stream')
  273. this.response.headers.set('x-local-cache-status', status)
  274. this.response.headers.set('x-local-cache-time', new Date().toISOString())
  275. const newResponse = new Response(body, {
  276. url: this.response.url,
  277. status: this.response.status,
  278. headers: this.response.headers,
  279. counter: this.options.counter,
  280. })
  281. return newResponse
  282. }
  283. // use the cached data to create a response and return it
  284. async respond (method, options, status) {
  285. let response
  286. if (method === 'HEAD' || [301, 308].includes(this.response.status)) {
  287. // if the request is a HEAD, or the response is a redirect,
  288. // then the metadata in the entry already includes everything
  289. // we need to build a response
  290. response = this.response
  291. } else {
  292. // we're responding with a full cached response, so create a body
  293. // that reads from cacache and attach it to a new Response
  294. const body = new Minipass()
  295. const headers = { ...this.policy.responseHeaders() }
  296. const onResume = () => {
  297. const cacheStream = cacache.get.stream.byDigest(
  298. this.options.cachePath, this.entry.integrity, { memoize: this.options.memoize }
  299. )
  300. cacheStream.on('error', async (err) => {
  301. cacheStream.pause()
  302. if (err.code === 'EINTEGRITY') {
  303. await cacache.rm.content(
  304. this.options.cachePath, this.entry.integrity, { memoize: this.options.memoize }
  305. )
  306. }
  307. if (err.code === 'ENOENT' || err.code === 'EINTEGRITY') {
  308. await CacheEntry.invalidate(this.request, this.options)
  309. }
  310. body.emit('error', err)
  311. cacheStream.resume()
  312. })
  313. // emit the integrity and size events based on our metadata so we're consistent
  314. body.emit('integrity', this.entry.integrity)
  315. body.emit('size', Number(headers['content-length']))
  316. cacheStream.pipe(body)
  317. }
  318. body.once('resume', onResume)
  319. body.once('end', () => body.removeListener('resume', onResume))
  320. response = new Response(body, {
  321. url: this.entry.metadata.url,
  322. counter: options.counter,
  323. status: 200,
  324. headers,
  325. })
  326. }
  327. response.headers.set('x-local-cache', encodeURIComponent(this.options.cachePath))
  328. response.headers.set('x-local-cache-hash', encodeURIComponent(this.entry.integrity))
  329. response.headers.set('x-local-cache-key', encodeURIComponent(this.key))
  330. response.headers.set('x-local-cache-mode', 'stream')
  331. response.headers.set('x-local-cache-status', status)
  332. response.headers.set('x-local-cache-time', new Date(this.entry.metadata.time).toUTCString())
  333. return response
  334. }
  335. // use the provided request along with this cache entry to
  336. // revalidate the stored response. returns a response, either
  337. // from the cache or from the update
  338. async revalidate (request, options) {
  339. const revalidateRequest = new Request(request, {
  340. headers: this.policy.revalidationHeaders(request),
  341. })
  342. try {
  343. // NOTE: be sure to remove the headers property from the
  344. // user supplied options, since we have already defined
  345. // them on the new request object. if they're still in the
  346. // options then those will overwrite the ones from the policy
  347. var response = await remote(revalidateRequest, {
  348. ...options,
  349. headers: undefined,
  350. })
  351. } catch (err) {
  352. // if the network fetch fails, return the stale
  353. // cached response unless it has a cache-control
  354. // of 'must-revalidate'
  355. if (!this.policy.mustRevalidate) {
  356. return this.respond(request.method, options, 'stale')
  357. }
  358. throw err
  359. }
  360. if (this.policy.revalidated(revalidateRequest, response)) {
  361. // we got a 304, write a new index to the cache and respond from cache
  362. const metadata = getMetadata(request, response, options)
  363. // 304 responses do not include headers that are specific to the response data
  364. // since they do not include a body, so we copy values for headers that were
  365. // in the old cache entry to the new one, if the new metadata does not already
  366. // include that header
  367. for (const name of KEEP_RESPONSE_HEADERS) {
  368. if (
  369. !hasOwnProperty(metadata.resHeaders, name) &&
  370. hasOwnProperty(this.entry.metadata.resHeaders, name)
  371. ) {
  372. metadata.resHeaders[name] = this.entry.metadata.resHeaders[name]
  373. }
  374. }
  375. try {
  376. await cacache.index.insert(options.cachePath, this.key, this.entry.integrity, {
  377. size: this.entry.size,
  378. metadata,
  379. })
  380. } catch (err) {
  381. // if updating the cache index fails, we ignore it and
  382. // respond anyway
  383. }
  384. return this.respond(request.method, options, 'revalidated')
  385. }
  386. // if we got a modified response, create a new entry based on it
  387. const newEntry = new CacheEntry({
  388. request,
  389. response,
  390. options,
  391. })
  392. // respond with the new entry while writing it to the cache
  393. return newEntry.store('updated')
  394. }
  395. }
  396. module.exports = CacheEntry