read-json.js 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589
  1. var fs = require('fs')
  2. var path = require('path')
  3. var { glob } = require('glob')
  4. var normalizeData = require('normalize-package-data')
  5. var safeJSON = require('json-parse-even-better-errors')
  6. var util = require('util')
  7. var normalizePackageBin = require('npm-normalize-package-bin')
  8. module.exports = readJson
  9. // put more stuff on here to customize.
  10. readJson.extraSet = [
  11. bundleDependencies,
  12. gypfile,
  13. serverjs,
  14. scriptpath,
  15. authors,
  16. readme,
  17. mans,
  18. bins,
  19. githead,
  20. fillTypes,
  21. ]
  22. var typoWarned = {}
  23. var cache = {}
  24. function readJson (file, log_, strict_, cb_) {
  25. var log, strict, cb
  26. for (var i = 1; i < arguments.length - 1; i++) {
  27. if (typeof arguments[i] === 'boolean') {
  28. strict = arguments[i]
  29. } else if (typeof arguments[i] === 'function') {
  30. log = arguments[i]
  31. }
  32. }
  33. if (!log) {
  34. log = function () {}
  35. }
  36. cb = arguments[arguments.length - 1]
  37. readJson_(file, log, strict, cb)
  38. }
  39. function readJson_ (file, log, strict, cb) {
  40. fs.readFile(file, 'utf8', function (er, d) {
  41. parseJson(file, er, d, log, strict, cb)
  42. })
  43. }
  44. function stripBOM (content) {
  45. // Remove byte order marker. This catches EF BB BF (the UTF-8 BOM)
  46. // because the buffer-to-string conversion in `fs.readFileSync()`
  47. // translates it to FEFF, the UTF-16 BOM.
  48. if (content.charCodeAt(0) === 0xFEFF) {
  49. content = content.slice(1)
  50. }
  51. return content
  52. }
  53. function jsonClone (obj) {
  54. if (obj == null) {
  55. return obj
  56. } else if (Array.isArray(obj)) {
  57. var newarr = new Array(obj.length)
  58. for (var ii in obj) {
  59. newarr[ii] = jsonClone(obj[ii])
  60. }
  61. return newarr
  62. } else if (typeof obj === 'object') {
  63. var newobj = {}
  64. for (var kk in obj) {
  65. newobj[kk] = jsonClone(obj[kk])
  66. }
  67. return newobj
  68. } else {
  69. return obj
  70. }
  71. }
  72. function parseJson (file, er, d, log, strict, cb) {
  73. if (er && er.code === 'ENOENT') {
  74. return fs.stat(path.dirname(file), function (err, stat) {
  75. if (!err && stat && !stat.isDirectory()) {
  76. // ENOTDIR isn't used on Windows, but npm expects it.
  77. er = Object.create(er)
  78. er.code = 'ENOTDIR'
  79. return cb(er)
  80. } else {
  81. return indexjs(file, er, log, strict, cb)
  82. }
  83. })
  84. }
  85. if (er) {
  86. return cb(er)
  87. }
  88. if (cache[d]) {
  89. return cb(null, jsonClone(cache[d]))
  90. }
  91. var data
  92. try {
  93. data = safeJSON(stripBOM(d))
  94. for (var key in data) {
  95. if (/^_/.test(key)) {
  96. delete data[key]
  97. }
  98. }
  99. } catch (jsonErr) {
  100. data = parseIndex(d)
  101. if (!data) {
  102. return cb(parseError(jsonErr, file))
  103. }
  104. }
  105. extrasCached(file, d, data, log, strict, cb)
  106. }
  107. function extrasCached (file, d, data, log, strict, cb) {
  108. extras(file, data, log, strict, function (err, extrasData) {
  109. if (!err) {
  110. cache[d] = jsonClone(extrasData)
  111. }
  112. cb(err, extrasData)
  113. })
  114. }
  115. function indexjs (file, er, log, strict, cb) {
  116. if (path.basename(file) === 'index.js') {
  117. return cb(er)
  118. }
  119. var index = path.resolve(path.dirname(file), 'index.js')
  120. fs.readFile(index, 'utf8', function (er2, d) {
  121. if (er2) {
  122. return cb(er)
  123. }
  124. if (cache[d]) {
  125. return cb(null, cache[d])
  126. }
  127. var data = parseIndex(d)
  128. if (!data) {
  129. return cb(er)
  130. }
  131. extrasCached(file, d, data, log, strict, cb)
  132. })
  133. }
  134. readJson.extras = extras
  135. function extras (file, data, log_, strict_, cb_) {
  136. var log, strict, cb
  137. for (var i = 2; i < arguments.length - 1; i++) {
  138. if (typeof arguments[i] === 'boolean') {
  139. strict = arguments[i]
  140. } else if (typeof arguments[i] === 'function') {
  141. log = arguments[i]
  142. }
  143. }
  144. if (!log) {
  145. log = function () {}
  146. }
  147. cb = arguments[i]
  148. var set = readJson.extraSet
  149. var n = set.length
  150. var errState = null
  151. set.forEach(function (fn) {
  152. fn(file, data, then)
  153. })
  154. function then (er) {
  155. if (errState) {
  156. return
  157. }
  158. if (er) {
  159. return cb(errState = er)
  160. }
  161. if (--n > 0) {
  162. return
  163. }
  164. final(file, data, log, strict, cb)
  165. }
  166. }
  167. function scriptpath (file, data, cb) {
  168. if (!data.scripts) {
  169. return cb(null, data)
  170. }
  171. var k = Object.keys(data.scripts)
  172. k.forEach(scriptpath_, data.scripts)
  173. cb(null, data)
  174. }
  175. function scriptpath_ (key) {
  176. var s = this[key]
  177. // This is never allowed, and only causes problems
  178. if (typeof s !== 'string') {
  179. return delete this[key]
  180. }
  181. var spre = /^(\.[/\\])?node_modules[/\\].bin[\\/]/
  182. if (s.match(spre)) {
  183. this[key] = this[key].replace(spre, '')
  184. }
  185. }
  186. function gypfile (file, data, cb) {
  187. var dir = path.dirname(file)
  188. var s = data.scripts || {}
  189. if (s.install || s.preinstall) {
  190. return cb(null, data)
  191. }
  192. if (data.gypfile === false) {
  193. return cb(null, data)
  194. }
  195. glob('*.gyp', { cwd: dir })
  196. .then(files => gypfile_(file, data, files, cb))
  197. .catch(er => cb(er))
  198. }
  199. function gypfile_ (file, data, files, cb) {
  200. if (!files.length) {
  201. return cb(null, data)
  202. }
  203. var s = data.scripts || {}
  204. s.install = 'node-gyp rebuild'
  205. data.scripts = s
  206. data.gypfile = true
  207. return cb(null, data)
  208. }
  209. function serverjs (file, data, cb) {
  210. var dir = path.dirname(file)
  211. var s = data.scripts || {}
  212. if (s.start) {
  213. return cb(null, data)
  214. }
  215. fs.access(path.join(dir, 'server.js'), (err) => {
  216. if (!err) {
  217. s.start = 'node server.js'
  218. data.scripts = s
  219. }
  220. return cb(null, data)
  221. })
  222. }
  223. function authors (file, data, cb) {
  224. if (data.contributors) {
  225. return cb(null, data)
  226. }
  227. var af = path.resolve(path.dirname(file), 'AUTHORS')
  228. fs.readFile(af, 'utf8', function (er, ad) {
  229. // ignore error. just checking it.
  230. if (er) {
  231. return cb(null, data)
  232. }
  233. authors_(file, data, ad, cb)
  234. })
  235. }
  236. function authors_ (file, data, ad, cb) {
  237. ad = ad.split(/\r?\n/g).map(function (line) {
  238. return line.replace(/^\s*#.*$/, '').trim()
  239. }).filter(function (line) {
  240. return line
  241. })
  242. data.contributors = ad
  243. return cb(null, data)
  244. }
  245. function readme (file, data, cb) {
  246. if (data.readme) {
  247. return cb(null, data)
  248. }
  249. var dir = path.dirname(file)
  250. var globOpts = { cwd: dir, nocase: true, mark: true }
  251. glob('{README,README.*}', globOpts)
  252. .then(files => {
  253. // don't accept directories.
  254. files = files.filter(function (filtered) {
  255. return !filtered.match(/\/$/)
  256. })
  257. if (!files.length) {
  258. return cb()
  259. }
  260. var fn = preferMarkdownReadme(files)
  261. var rm = path.resolve(dir, fn)
  262. return readme_(file, data, rm, cb)
  263. })
  264. .catch(er => cb(er))
  265. }
  266. function preferMarkdownReadme (files) {
  267. var fallback = 0
  268. var re = /\.m?a?r?k?d?o?w?n?$/i
  269. for (var i = 0; i < files.length; i++) {
  270. if (files[i].match(re)) {
  271. return files[i]
  272. } else if (files[i].match(/README$/)) {
  273. fallback = i
  274. }
  275. }
  276. // prefer README.md, followed by README; otherwise, return
  277. // the first filename (which could be README)
  278. return files[fallback]
  279. }
  280. function readme_ (file, data, rm, cb) {
  281. var rmfn = path.basename(rm)
  282. fs.readFile(rm, 'utf8', function (er, rmData) {
  283. // maybe not readable, or something.
  284. if (er) {
  285. return cb()
  286. }
  287. data.readme = rmData
  288. data.readmeFilename = rmfn
  289. return cb(er, data)
  290. })
  291. }
  292. function mans (file, data, cb) {
  293. let cwd = data.directories && data.directories.man
  294. if (data.man || !cwd) {
  295. return cb(null, data)
  296. }
  297. const dirname = path.dirname(file)
  298. cwd = path.resolve(path.dirname(file), cwd)
  299. glob('**/*.[0-9]', { cwd })
  300. .then(mansGlob => {
  301. data.man = mansGlob.map(man =>
  302. path.relative(dirname, path.join(cwd, man)).split(path.sep).join('/')
  303. )
  304. return cb(null, data)
  305. })
  306. .catch(er => cb(er))
  307. }
  308. function bins (file, data, cb) {
  309. data = normalizePackageBin(data)
  310. var m = data.directories && data.directories.bin
  311. if (data.bin || !m) {
  312. return cb(null, data)
  313. }
  314. m = path.resolve(path.dirname(file), path.join('.', path.join('/', m)))
  315. glob('**', { cwd: m })
  316. .then(binsGlob => bins_(file, data, binsGlob, cb))
  317. .catch(er => cb(er))
  318. }
  319. function bins_ (file, data, binsGlob, cb) {
  320. var m = (data.directories && data.directories.bin) || '.'
  321. data.bin = binsGlob.reduce(function (acc, mf) {
  322. if (mf && mf.charAt(0) !== '.') {
  323. var f = path.basename(mf)
  324. acc[f] = path.join(m, mf)
  325. }
  326. return acc
  327. }, {})
  328. return cb(null, normalizePackageBin(data))
  329. }
  330. function bundleDependencies (file, data, cb) {
  331. var bd = 'bundleDependencies'
  332. var bdd = 'bundledDependencies'
  333. // normalize key name
  334. if (data[bdd] !== undefined) {
  335. if (data[bd] === undefined) {
  336. data[bd] = data[bdd]
  337. }
  338. delete data[bdd]
  339. }
  340. if (data[bd] === false) {
  341. delete data[bd]
  342. } else if (data[bd] === true) {
  343. data[bd] = Object.keys(data.dependencies || {})
  344. } else if (data[bd] !== undefined && !Array.isArray(data[bd])) {
  345. delete data[bd]
  346. }
  347. return cb(null, data)
  348. }
  349. function githead (file, data, cb) {
  350. if (data.gitHead) {
  351. return cb(null, data)
  352. }
  353. var dir = path.dirname(file)
  354. var head = path.resolve(dir, '.git/HEAD')
  355. fs.readFile(head, 'utf8', function (er, headData) {
  356. if (er) {
  357. var parent = path.dirname(dir)
  358. if (parent === dir) {
  359. return cb(null, data)
  360. }
  361. return githead(dir, data, cb)
  362. }
  363. githead_(data, dir, headData, cb)
  364. })
  365. }
  366. function githead_ (data, dir, head, cb) {
  367. if (!head.match(/^ref: /)) {
  368. data.gitHead = head.trim()
  369. return cb(null, data)
  370. }
  371. var headRef = head.replace(/^ref: /, '').trim()
  372. var headFile = path.resolve(dir, '.git', headRef)
  373. fs.readFile(headFile, 'utf8', function (er, headData) {
  374. if (er || !headData) {
  375. var packFile = path.resolve(dir, '.git/packed-refs')
  376. return fs.readFile(packFile, 'utf8', function (readFileErr, refs) {
  377. if (readFileErr || !refs) {
  378. return cb(null, data)
  379. }
  380. refs = refs.split('\n')
  381. for (var i = 0; i < refs.length; i++) {
  382. var match = refs[i].match(/^([0-9a-f]{40}) (.+)$/)
  383. if (match && match[2].trim() === headRef) {
  384. data.gitHead = match[1]
  385. break
  386. }
  387. }
  388. return cb(null, data)
  389. })
  390. }
  391. headData = headData.replace(/^ref: /, '').trim()
  392. data.gitHead = headData
  393. return cb(null, data)
  394. })
  395. }
  396. /**
  397. * Warn if the bin references don't point to anything. This might be better in
  398. * normalize-package-data if it had access to the file path.
  399. */
  400. function checkBinReferences_ (file, data, warn, cb) {
  401. if (!(data.bin instanceof Object)) {
  402. return cb()
  403. }
  404. var keys = Object.keys(data.bin)
  405. var keysLeft = keys.length
  406. if (!keysLeft) {
  407. return cb()
  408. }
  409. function handleExists (relName, result) {
  410. keysLeft--
  411. if (!result) {
  412. warn('No bin file found at ' + relName)
  413. }
  414. if (!keysLeft) {
  415. cb()
  416. }
  417. }
  418. keys.forEach(function (key) {
  419. var dirName = path.dirname(file)
  420. var relName = data.bin[key]
  421. /* istanbul ignore if - impossible, bins have been normalized */
  422. if (typeof relName !== 'string') {
  423. var msg = 'Bin filename for ' + key +
  424. ' is not a string: ' + util.inspect(relName)
  425. warn(msg)
  426. delete data.bin[key]
  427. handleExists(relName, true)
  428. return
  429. }
  430. var binPath = path.resolve(dirName, relName)
  431. fs.stat(binPath, (err) => handleExists(relName, !err))
  432. })
  433. }
  434. function final (file, data, log, strict, cb) {
  435. var pId = makePackageId(data)
  436. function warn (msg) {
  437. if (typoWarned[pId]) {
  438. return
  439. }
  440. if (log) {
  441. log('package.json', pId, msg)
  442. }
  443. }
  444. try {
  445. normalizeData(data, warn, strict)
  446. } catch (error) {
  447. return cb(error)
  448. }
  449. checkBinReferences_(file, data, warn, function () {
  450. typoWarned[pId] = true
  451. cb(null, data)
  452. })
  453. }
  454. function fillTypes (file, data, cb) {
  455. var index = data.main || 'index.js'
  456. if (typeof index !== 'string') {
  457. return cb(new TypeError('The "main" attribute must be of type string.'))
  458. }
  459. // TODO exports is much more complicated than this in verbose format
  460. // We need to support for instance
  461. // "exports": {
  462. // ".": [
  463. // {
  464. // "default": "./lib/npm.js"
  465. // },
  466. // "./lib/npm.js"
  467. // ],
  468. // "./package.json": "./package.json"
  469. // },
  470. // as well as conditional exports
  471. // if (data.exports && typeof data.exports === 'string') {
  472. // index = data.exports
  473. // }
  474. // if (data.exports && data.exports['.']) {
  475. // index = data.exports['.']
  476. // if (typeof index !== 'string') {
  477. // }
  478. // }
  479. var extless =
  480. path.join(path.dirname(index), path.basename(index, path.extname(index)))
  481. var dts = `./${extless}.d.ts`
  482. var dtsPath = path.join(path.dirname(file), dts)
  483. var hasDTSFields = 'types' in data || 'typings' in data
  484. if (!hasDTSFields && fs.existsSync(dtsPath)) {
  485. data.types = dts.split(path.sep).join('/')
  486. }
  487. cb(null, data)
  488. }
  489. function makePackageId (data) {
  490. var name = cleanString(data.name)
  491. var ver = cleanString(data.version)
  492. return name + '@' + ver
  493. }
  494. function cleanString (str) {
  495. return (!str || typeof (str) !== 'string') ? '' : str.trim()
  496. }
  497. // /**package { "name": "foo", "version": "1.2.3", ... } **/
  498. function parseIndex (data) {
  499. data = data.split(/^\/\*\*package(?:\s|$)/m)
  500. if (data.length < 2) {
  501. return null
  502. }
  503. data = data[1]
  504. data = data.split(/\*\*\/$/m)
  505. if (data.length < 2) {
  506. return null
  507. }
  508. data = data[0]
  509. data = data.replace(/^\s*\*/mg, '')
  510. try {
  511. return safeJSON(data)
  512. } catch (er) {
  513. return null
  514. }
  515. }
  516. function parseError (ex, file) {
  517. var e = new Error('Failed to parse json\n' + ex.message)
  518. e.code = 'EJSONPARSE'
  519. e.path = file
  520. return e
  521. }