x_nodejs.py (12558B)
1 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
2 #
3 # Modified by Anthony Fok on 2018-10-01 to add support for ppc64el and s390x
4 #
5 # Copyright (C) 2015-2017 Canonical Ltd
6 #
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License version 3 as
9 # published by the Free Software Foundation.
10 #
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
18
19 """The nodejs plugin is useful for node/npm based parts.
20
21 The plugin uses node to install dependencies from `package.json`. It
22 also sets up binaries defined in `package.json` into the `PATH`.
23
24 This plugin uses the common plugin keywords as well as those for "sources".
25 For more information check the 'plugins' topic for the former and the
26 'sources' topic for the latter.
27
28 Additionally, this plugin uses the following plugin-specific keywords:
29
30 - node-packages:
31 (list)
32 A list of dependencies to fetch using npm.
33 - node-engine:
34 (string)
35 The version of nodejs you want the snap to run on.
36 - npm-run:
37 (list)
38 A list of targets to `npm run`.
39 These targets will be run in order, after `npm install`
40 - npm-flags:
41 (list)
42 A list of flags for npm.
43 - node-package-manager
44 (string; default: npm)
45 The language package manager to use to drive installation
46 of node packages. Can be either `npm` (default) or `yarn`.
47 """
48
49 import collections
50 import contextlib
51 import json
52 import logging
53 import os
54 import shutil
55 import subprocess
56 import sys
57
58 import snapcraft
59 from snapcraft import sources
60 from snapcraft.file_utils import link_or_copy_tree
61 from snapcraft.internal import errors
62
63 logger = logging.getLogger(__name__)
64
65 _NODEJS_BASE = "node-v{version}-linux-{arch}"
66 _NODEJS_VERSION = "12.18.4"
67 _NODEJS_TMPL = "https://nodejs.org/dist/v{version}/{base}.tar.gz"
68 _NODEJS_ARCHES = {"i386": "x86", "amd64": "x64", "armhf": "armv7l", "arm64": "arm64", "ppc64el": "ppc64le", "s390x": "s390x"}
69 _YARN_URL = "https://yarnpkg.com/latest.tar.gz"
70
71
72 class NodePlugin(snapcraft.BasePlugin):
73 @classmethod
74 def schema(cls):
75 schema = super().schema()
76
77 schema["properties"]["node-packages"] = {
78 "type": "array",
79 "minitems": 1,
80 "uniqueItems": True,
81 "items": {"type": "string"},
82 "default": [],
83 }
84 schema["properties"]["node-engine"] = {
85 "type": "string",
86 "default": _NODEJS_VERSION,
87 }
88 schema["properties"]["node-package-manager"] = {
89 "type": "string",
90 "default": "npm",
91 "enum": ["npm", "yarn"],
92 }
93 schema["properties"]["npm-run"] = {
94 "type": "array",
95 "minitems": 1,
96 "uniqueItems": False,
97 "items": {"type": "string"},
98 "default": [],
99 }
100 schema["properties"]["npm-flags"] = {
101 "type": "array",
102 "minitems": 1,
103 "uniqueItems": False,
104 "items": {"type": "string"},
105 "default": [],
106 }
107
108 if "required" in schema:
109 del schema["required"]
110
111 return schema
112
113 @classmethod
114 def get_build_properties(cls):
115 # Inform Snapcraft of the properties associated with building. If these
116 # change in the YAML Snapcraft will consider the build step dirty.
117 return ["node-packages", "npm-run", "npm-flags"]
118
119 @classmethod
120 def get_pull_properties(cls):
121 # Inform Snapcraft of the properties associated with pulling. If these
122 # change in the YAML Snapcraft will consider the build step dirty.
123 return ["node-engine", "node-package-manager"]
124
125 @property
126 def _nodejs_tar(self):
127 if self._nodejs_tar_handle is None:
128 self._nodejs_tar_handle = sources.Tar(
129 self._nodejs_release_uri, self._npm_dir
130 )
131 return self._nodejs_tar_handle
132
133 @property
134 def _yarn_tar(self):
135 if self._yarn_tar_handle is None:
136 self._yarn_tar_handle = sources.Tar(_YARN_URL, self._npm_dir)
137 return self._yarn_tar_handle
138
139 def __init__(self, name, options, project):
140 super().__init__(name, options, project)
141 self._source_package_json = os.path.join(
142 os.path.abspath(self.options.source), "package.json"
143 )
144 self._npm_dir = os.path.join(self.partdir, "npm")
145 self._manifest = collections.OrderedDict()
146 self._nodejs_release_uri = get_nodejs_release(
147 self.options.node_engine, self.project.deb_arch
148 )
149 self._nodejs_tar_handle = None
150 self._yarn_tar_handle = None
151
152 def pull(self):
153 super().pull()
154 os.makedirs(self._npm_dir, exist_ok=True)
155 self._nodejs_tar.download()
156 if self.options.node_package_manager == "yarn":
157 self._yarn_tar.download()
158 # do the install in the pull phase to download all dependencies.
159 if self.options.node_package_manager == "npm":
160 self._npm_install(rootdir=self.sourcedir)
161 else:
162 self._yarn_install(rootdir=self.sourcedir)
163
164 def clean_pull(self):
165 super().clean_pull()
166
167 # Remove the npm directory (if any)
168 if os.path.exists(self._npm_dir):
169 shutil.rmtree(self._npm_dir)
170
171 def build(self):
172 super().build()
173 if self.options.node_package_manager == "npm":
174 installed_node_packages = self._npm_install(rootdir=self.builddir)
175 # Copy the content of the symlink to the build directory
176 # LP: #1702661
177 modules_dir = os.path.join(self.installdir, "lib", "node_modules")
178 _copy_symlinked_content(modules_dir)
179 else:
180 installed_node_packages = self._yarn_install(rootdir=self.builddir)
181 lock_file_path = os.path.join(self.sourcedir, "yarn.lock")
182 if os.path.isfile(lock_file_path):
183 with open(lock_file_path) as lock_file:
184 self._manifest["yarn-lock-contents"] = lock_file.read()
185
186 self._manifest["node-packages"] = [
187 "{}={}".format(name, installed_node_packages[name])
188 for name in installed_node_packages
189 ]
190
191 def _npm_install(self, rootdir):
192 self._nodejs_tar.provision(
193 self.installdir, clean_target=False, keep_tarball=True
194 )
195 npm_cmd = ["npm"] + self.options.npm_flags
196 npm_install = npm_cmd + ["--cache-min=Infinity", "install"]
197 for pkg in self.options.node_packages:
198 self.run(npm_install + ["--global"] + [pkg], cwd=rootdir)
199 if os.path.exists(os.path.join(rootdir, "package.json")):
200 self.run(npm_install, cwd=rootdir)
201 self.run(npm_install + ["--global"], cwd=rootdir)
202 for target in self.options.npm_run:
203 self.run(npm_cmd + ["run", target], cwd=rootdir)
204 return self._get_installed_node_packages("npm", self.installdir)
205
206 def _yarn_install(self, rootdir):
207 self._nodejs_tar.provision(
208 self.installdir, clean_target=False, keep_tarball=True
209 )
210 self._yarn_tar.provision(self._npm_dir, clean_target=False, keep_tarball=True)
211 yarn_cmd = [os.path.join(self._npm_dir, "bin", "yarn")]
212 yarn_cmd.extend(self.options.npm_flags)
213 if "http_proxy" in os.environ:
214 yarn_cmd.extend(["--proxy", os.environ["http_proxy"]])
215 if "https_proxy" in os.environ:
216 yarn_cmd.extend(["--https-proxy", os.environ["https_proxy"]])
217 flags = []
218 if rootdir == self.builddir:
219 yarn_add = yarn_cmd + ["global", "add"]
220 flags.extend(
221 [
222 "--offline",
223 "--prod",
224 "--global-folder",
225 self.installdir,
226 "--prefix",
227 self.installdir,
228 ]
229 )
230 else:
231 yarn_add = yarn_cmd + ["add"]
232 for pkg in self.options.node_packages:
233 self.run(yarn_add + [pkg] + flags, cwd=rootdir)
234
235 # local packages need to be added as if they were remote, we
236 # remove the local package.json so `yarn add` doesn't pollute it.
237 if os.path.exists(self._source_package_json):
238 with contextlib.suppress(FileNotFoundError):
239 os.unlink(os.path.join(rootdir, "package.json"))
240 shutil.copy(
241 self._source_package_json, os.path.join(rootdir, "package.json")
242 )
243 self.run(yarn_add + ["file:{}".format(rootdir)] + flags, cwd=rootdir)
244
245 # npm run would require to bring back package.json
246 if self.options.npm_run and os.path.exists(self._source_package_json):
247 # The current package.json is the yarn prefilled one.
248 with contextlib.suppress(FileNotFoundError):
249 os.unlink(os.path.join(rootdir, "package.json"))
250 os.link(self._source_package_json, os.path.join(rootdir, "package.json"))
251 for target in self.options.npm_run:
252 self.run(
253 yarn_cmd + ["run", target],
254 cwd=rootdir,
255 env=self._build_environment(rootdir),
256 )
257 return self._get_installed_node_packages("npm", self.installdir)
258
259 def _get_installed_node_packages(self, package_manager, cwd):
260 try:
261 output = self.run_output(
262 [package_manager, "ls", "--global", "--json"], cwd=cwd
263 )
264 except subprocess.CalledProcessError as error:
265 # XXX When dependencies have missing dependencies, an error like
266 # this is printed to stderr:
267 # npm ERR! peer dep missing: glob@*, required by glob-promise@3.1.0
268 # retcode is not 0, which raises an exception.
269 output = error.output.decode(sys.getfilesystemencoding()).strip()
270 packages = collections.OrderedDict()
271 dependencies = json.loads(output, object_pairs_hook=collections.OrderedDict)[
272 "dependencies"
273 ]
274 while dependencies:
275 key, value = dependencies.popitem(last=False)
276 # XXX Just as above, dependencies without version are the ones
277 # missing.
278 if "version" in value:
279 packages[key] = value["version"]
280 if "dependencies" in value:
281 dependencies.update(value["dependencies"])
282 return packages
283
284 def get_manifest(self):
285 return self._manifest
286
287 def _build_environment(self, rootdir):
288 env = os.environ.copy()
289 if rootdir.endswith("src"):
290 hidden_path = os.path.join(rootdir, "node_modules", ".bin")
291 if env.get("PATH"):
292 new_path = "{}:{}".format(hidden_path, env.get("PATH"))
293 else:
294 new_path = hidden_path
295 env["PATH"] = new_path
296 return env
297
298
299 def _get_nodejs_base(node_engine, machine):
300 if machine not in _NODEJS_ARCHES:
301 raise errors.SnapcraftEnvironmentError(
302 "architecture not supported ({})".format(machine)
303 )
304 return _NODEJS_BASE.format(version=node_engine, arch=_NODEJS_ARCHES[machine])
305
306
307 def get_nodejs_release(node_engine, arch):
308 return _NODEJS_TMPL.format(
309 version=node_engine, base=_get_nodejs_base(node_engine, arch)
310 )
311
312
313 def _copy_symlinked_content(modules_dir):
314 """Copy symlinked content.
315
316 When running newer versions of npm, symlinks to the local tree are
317 created from the part's installdir to the root of the builddir of the
318 part (this only affects some build configurations in some projects)
319 which is valid when running from the context of the part but invalid
320 as soon as the artifacts migrate across the steps,
321 i.e.; stage and prime.
322
323 If modules_dir does not exist we simply return.
324 """
325 if not os.path.exists(modules_dir):
326 return
327 modules = [os.path.join(modules_dir, d) for d in os.listdir(modules_dir)]
328 symlinks = [l for l in modules if os.path.islink(l)]
329 for link_path in symlinks:
330 link_target = os.path.realpath(link_path)
331 os.unlink(link_path)
332 link_or_copy_tree(link_target, link_path)