git.js 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327
  1. const Fetcher = require('./fetcher.js')
  2. const FileFetcher = require('./file.js')
  3. const RemoteFetcher = require('./remote.js')
  4. const DirFetcher = require('./dir.js')
  5. const hashre = /^[a-f0-9]{40}$/
  6. const git = require('@npmcli/git')
  7. const pickManifest = require('npm-pick-manifest')
  8. const npa = require('npm-package-arg')
  9. const { Minipass } = require('minipass')
  10. const cacache = require('cacache')
  11. const log = require('proc-log')
  12. const npm = require('./util/npm.js')
  13. const _resolvedFromRepo = Symbol('_resolvedFromRepo')
  14. const _resolvedFromHosted = Symbol('_resolvedFromHosted')
  15. const _resolvedFromClone = Symbol('_resolvedFromClone')
  16. const _tarballFromResolved = Symbol.for('pacote.Fetcher._tarballFromResolved')
  17. const _addGitSha = Symbol('_addGitSha')
  18. const addGitSha = require('./util/add-git-sha.js')
  19. const _clone = Symbol('_clone')
  20. const _cloneHosted = Symbol('_cloneHosted')
  21. const _cloneRepo = Symbol('_cloneRepo')
  22. const _setResolvedWithSha = Symbol('_setResolvedWithSha')
  23. const _prepareDir = Symbol('_prepareDir')
  24. const _readPackageJson = Symbol.for('package.Fetcher._readPackageJson')
  25. // get the repository url.
  26. // prefer https if there's auth, since ssh will drop that.
  27. // otherwise, prefer ssh if available (more secure).
  28. // We have to add the git+ back because npa suppresses it.
  29. const repoUrl = (h, opts) =>
  30. h.sshurl && !(h.https && h.auth) && addGitPlus(h.sshurl(opts)) ||
  31. h.https && addGitPlus(h.https(opts))
  32. // add git+ to the url, but only one time.
  33. const addGitPlus = url => url && `git+${url}`.replace(/^(git\+)+/, 'git+')
  34. class GitFetcher extends Fetcher {
  35. constructor (spec, opts) {
  36. super(spec, opts)
  37. // we never want to compare integrity for git dependencies: npm/rfcs#525
  38. if (this.opts.integrity) {
  39. delete this.opts.integrity
  40. log.warn(`skipping integrity check for git dependency ${this.spec.fetchSpec}`)
  41. }
  42. this.resolvedRef = null
  43. if (this.spec.hosted) {
  44. this.from = this.spec.hosted.shortcut({ noCommittish: false })
  45. }
  46. // shortcut: avoid full clone when we can go straight to the tgz
  47. // if we have the full sha and it's a hosted git platform
  48. if (this.spec.gitCommittish && hashre.test(this.spec.gitCommittish)) {
  49. this.resolvedSha = this.spec.gitCommittish
  50. // use hosted.tarball() when we shell to RemoteFetcher later
  51. this.resolved = this.spec.hosted
  52. ? repoUrl(this.spec.hosted, { noCommittish: false })
  53. : this.spec.rawSpec
  54. } else {
  55. this.resolvedSha = ''
  56. }
  57. this.Arborist = opts.Arborist || null
  58. }
  59. // just exposed to make it easier to test all the combinations
  60. static repoUrl (hosted, opts) {
  61. return repoUrl(hosted, opts)
  62. }
  63. get types () {
  64. return ['git']
  65. }
  66. resolve () {
  67. // likely a hosted git repo with a sha, so get the tarball url
  68. // but in general, no reason to resolve() more than necessary!
  69. if (this.resolved) {
  70. return super.resolve()
  71. }
  72. // fetch the git repo and then look at the current hash
  73. const h = this.spec.hosted
  74. // try to use ssh, fall back to git.
  75. return h ? this[_resolvedFromHosted](h)
  76. : this[_resolvedFromRepo](this.spec.fetchSpec)
  77. }
  78. // first try https, since that's faster and passphrase-less for
  79. // public repos, and supports private repos when auth is provided.
  80. // Fall back to SSH to support private repos
  81. // NB: we always store the https url in resolved field if auth
  82. // is present, otherwise ssh if the hosted type provides it
  83. [_resolvedFromHosted] (hosted) {
  84. return this[_resolvedFromRepo](hosted.https && hosted.https())
  85. .catch(er => {
  86. // Throw early since we know pathspec errors will fail again if retried
  87. if (er instanceof git.errors.GitPathspecError) {
  88. throw er
  89. }
  90. const ssh = hosted.sshurl && hosted.sshurl()
  91. // no fallthrough if we can't fall through or have https auth
  92. if (!ssh || hosted.auth) {
  93. throw er
  94. }
  95. return this[_resolvedFromRepo](ssh)
  96. })
  97. }
  98. [_resolvedFromRepo] (gitRemote) {
  99. // XXX make this a custom error class
  100. if (!gitRemote) {
  101. return Promise.reject(new Error(`No git url for ${this.spec}`))
  102. }
  103. const gitRange = this.spec.gitRange
  104. const name = this.spec.name
  105. return git.revs(gitRemote, this.opts).then(remoteRefs => {
  106. return gitRange ? pickManifest({
  107. versions: remoteRefs.versions,
  108. 'dist-tags': remoteRefs['dist-tags'],
  109. name,
  110. }, gitRange, this.opts)
  111. : this.spec.gitCommittish ?
  112. remoteRefs.refs[this.spec.gitCommittish] ||
  113. remoteRefs.refs[remoteRefs.shas[this.spec.gitCommittish]]
  114. : remoteRefs.refs.HEAD // no git committish, get default head
  115. }).then(revDoc => {
  116. // the committish provided isn't in the rev list
  117. // things like HEAD~3 or @yesterday can land here.
  118. if (!revDoc || !revDoc.sha) {
  119. return this[_resolvedFromClone]()
  120. }
  121. this.resolvedRef = revDoc
  122. this.resolvedSha = revDoc.sha
  123. this[_addGitSha](revDoc.sha)
  124. return this.resolved
  125. })
  126. }
  127. [_setResolvedWithSha] (withSha) {
  128. // we haven't cloned, so a tgz download is still faster
  129. // of course, if it's not a known host, we can't do that.
  130. this.resolved = !this.spec.hosted ? withSha
  131. : repoUrl(npa(withSha).hosted, { noCommittish: false })
  132. }
  133. // when we get the git sha, we affix it to our spec to build up
  134. // either a git url with a hash, or a tarball download URL
  135. [_addGitSha] (sha) {
  136. this[_setResolvedWithSha](addGitSha(this.spec, sha))
  137. }
  138. [_resolvedFromClone] () {
  139. // do a full or shallow clone, then look at the HEAD
  140. // kind of wasteful, but no other option, really
  141. return this[_clone](dir => this.resolved)
  142. }
  143. [_prepareDir] (dir) {
  144. return this[_readPackageJson](dir + '/package.json').then(mani => {
  145. // no need if we aren't going to do any preparation.
  146. const scripts = mani.scripts
  147. if (!mani.workspaces && (!scripts || !(
  148. scripts.postinstall ||
  149. scripts.build ||
  150. scripts.preinstall ||
  151. scripts.install ||
  152. scripts.prepack ||
  153. scripts.prepare))) {
  154. return
  155. }
  156. // to avoid cases where we have an cycle of git deps that depend
  157. // on one another, we only ever do preparation for one instance
  158. // of a given git dep along the chain of installations.
  159. // Note that this does mean that a dependency MAY in theory end up
  160. // trying to run its prepare script using a dependency that has not
  161. // been properly prepared itself, but that edge case is smaller
  162. // and less hazardous than a fork bomb of npm and git commands.
  163. const noPrepare = !process.env._PACOTE_NO_PREPARE_ ? []
  164. : process.env._PACOTE_NO_PREPARE_.split('\n')
  165. if (noPrepare.includes(this.resolved)) {
  166. log.info('prepare', 'skip prepare, already seen', this.resolved)
  167. return
  168. }
  169. noPrepare.push(this.resolved)
  170. // the DirFetcher will do its own preparation to run the prepare scripts
  171. // All we have to do is put the deps in place so that it can succeed.
  172. return npm(
  173. this.npmBin,
  174. [].concat(this.npmInstallCmd).concat(this.npmCliConfig),
  175. dir,
  176. { ...process.env, _PACOTE_NO_PREPARE_: noPrepare.join('\n') },
  177. { message: 'git dep preparation failed' }
  178. )
  179. })
  180. }
  181. [_tarballFromResolved] () {
  182. const stream = new Minipass()
  183. stream.resolved = this.resolved
  184. stream.from = this.from
  185. // check it out and then shell out to the DirFetcher tarball packer
  186. this[_clone](dir => this[_prepareDir](dir)
  187. .then(() => new Promise((res, rej) => {
  188. if (!this.Arborist) {
  189. throw new Error('GitFetcher requires an Arborist constructor to pack a tarball')
  190. }
  191. const df = new DirFetcher(`file:${dir}`, {
  192. ...this.opts,
  193. Arborist: this.Arborist,
  194. resolved: null,
  195. integrity: null,
  196. })
  197. const dirStream = df[_tarballFromResolved]()
  198. dirStream.on('error', rej)
  199. dirStream.on('end', res)
  200. dirStream.pipe(stream)
  201. }))).catch(
  202. /* istanbul ignore next: very unlikely and hard to test */
  203. er => stream.emit('error', er)
  204. )
  205. return stream
  206. }
  207. // clone a git repo into a temp folder (or fetch and unpack if possible)
  208. // handler accepts a directory, and returns a promise that resolves
  209. // when we're done with it, at which point, cacache deletes it
  210. //
  211. // TODO: after cloning, create a tarball of the folder, and add to the cache
  212. // with cacache.put.stream(), using a key that's deterministic based on the
  213. // spec and repo, so that we don't ever clone the same thing multiple times.
  214. [_clone] (handler, tarballOk = true) {
  215. const o = { tmpPrefix: 'git-clone' }
  216. const ref = this.resolvedSha || this.spec.gitCommittish
  217. const h = this.spec.hosted
  218. const resolved = this.resolved
  219. // can be set manually to false to fall back to actual git clone
  220. tarballOk = tarballOk &&
  221. h && resolved === repoUrl(h, { noCommittish: false }) && h.tarball
  222. return cacache.tmp.withTmp(this.cache, o, async tmp => {
  223. // if we're resolved, and have a tarball url, shell out to RemoteFetcher
  224. if (tarballOk) {
  225. const nameat = this.spec.name ? `${this.spec.name}@` : ''
  226. return new RemoteFetcher(h.tarball({ noCommittish: false }), {
  227. ...this.opts,
  228. allowGitIgnore: true,
  229. pkgid: `git:${nameat}${this.resolved}`,
  230. resolved: this.resolved,
  231. integrity: null, // it'll always be different, if we have one
  232. }).extract(tmp).then(() => handler(tmp), er => {
  233. // fall back to ssh download if tarball fails
  234. if (er.constructor.name.match(/^Http/)) {
  235. return this[_clone](handler, false)
  236. } else {
  237. throw er
  238. }
  239. })
  240. }
  241. const sha = await (
  242. h ? this[_cloneHosted](ref, tmp)
  243. : this[_cloneRepo](this.spec.fetchSpec, ref, tmp)
  244. )
  245. this.resolvedSha = sha
  246. if (!this.resolved) {
  247. await this[_addGitSha](sha)
  248. }
  249. return handler(tmp)
  250. })
  251. }
  252. // first try https, since that's faster and passphrase-less for
  253. // public repos, and supports private repos when auth is provided.
  254. // Fall back to SSH to support private repos
  255. // NB: we always store the https url in resolved field if auth
  256. // is present, otherwise ssh if the hosted type provides it
  257. [_cloneHosted] (ref, tmp) {
  258. const hosted = this.spec.hosted
  259. return this[_cloneRepo](hosted.https({ noCommittish: true }), ref, tmp)
  260. .catch(er => {
  261. // Throw early since we know pathspec errors will fail again if retried
  262. if (er instanceof git.errors.GitPathspecError) {
  263. throw er
  264. }
  265. const ssh = hosted.sshurl && hosted.sshurl({ noCommittish: true })
  266. // no fallthrough if we can't fall through or have https auth
  267. if (!ssh || hosted.auth) {
  268. throw er
  269. }
  270. return this[_cloneRepo](ssh, ref, tmp)
  271. })
  272. }
  273. [_cloneRepo] (repo, ref, tmp) {
  274. const { opts, spec } = this
  275. return git.clone(repo, ref, tmp, { ...opts, spec })
  276. }
  277. manifest () {
  278. if (this.package) {
  279. return Promise.resolve(this.package)
  280. }
  281. return this.spec.hosted && this.resolved
  282. ? FileFetcher.prototype.manifest.apply(this)
  283. : this[_clone](dir =>
  284. this[_readPackageJson](dir + '/package.json')
  285. .then(mani => this.package = {
  286. ...mani,
  287. _resolved: this.resolved,
  288. _from: this.from,
  289. }))
  290. }
  291. packument () {
  292. return FileFetcher.prototype.packument.apply(this)
  293. }
  294. }
  295. module.exports = GitFetcher