Update flatpak node generator and electron-builder version

flatpak-node-generator.py

@@ -9,7 +9,6 @@ from typing import *  # pyright: reportWildcardImportFromLibrary=false
 from typing import cast, IO
 
 from pathlib import Path
-from distutils.version import StrictVersion
 
 import argparse
 import asyncio
@@ -17,6 +16,7 @@ import base64
 import binascii
 import collections
 import contextlib
+import functools
 import hashlib
 import json
 import os
@@ -67,6 +67,68 @@ GIT_URL_HOSTS = ['github.com', 'gitlab.com', 'bitbucket.com', 'bitbucket.org']
 NPM_MIRROR = 'https://unpkg.com/'
 
 
+class SemVer(NamedTuple):
+    # Note that we ignore the metadata part, since all we do is version
+    # comparisons.
+    _SEMVER_RE = re.compile(r'(\d+)\.(\d+)\.(\d+)(?:-(?P<prerelease>[^+]+))?')
+
+    @functools.total_ordering
+    class Prerelease:
+        def __init__(self, parts: Tuple[Union[str, int]]) -> None:
+            self._parts = parts
+
+        @staticmethod
+        def parse(rel: str) -> Optional['SemVer.Prerelease']:
+            if not rel:
+                return None
+
+            parts: List[Union[str, int]] = []
+
+            for part in rel.split('.'):
+                try:
+                    part = int(part)
+                except ValueError:
+                    pass
+
+                parts.append(part)
+
+            return SemVer.Prerelease(tuple(parts))
+
+        @property
+        def parts(self) -> Tuple[Union[str, int]]:
+            return self._parts
+
+        def __lt__(self, other: 'SemVer.Prerelease'):
+            for our_part, other_part in zip(self._parts, other._parts):
+                if type(our_part) == type(other_part):
+                    if our_part < other_part:  # type: ignore
+                        return True
+                # Number parts are always less than strings.
+                elif isinstance(our_part, int):
+                    return True
+
+            return len(self._parts) < len(other._parts)
+
+        def __repr__(self) -> str:
+            return f'Prerelease(parts={self.parts})'
+
+    major: int
+    minor: int
+    patch: int
+    prerelease: Optional[Prerelease] = None
+
+    @staticmethod
+    def parse(version: str) -> 'SemVer':
+        match = SemVer._SEMVER_RE.match(version)
+        if match is None:
+            raise ValueError(f'Invalid semver version: {version}')
+
+        major, minor, patch = map(int, match.groups()[:3])
+        prerelease = SemVer.Prerelease.parse(match.group('prerelease'))
+
+        return SemVer(major, minor, patch, prerelease)
+
+
 class Cache:
     instance: 'Cache'
 
@@ -201,7 +263,8 @@ class FilesystemBasedCache(Cache):
             super().__init__(key)
             self._cache_root = cache_root
 
-            self._cache_path = self._cache_root / FilesystemBasedCache._escape_key(key)
+            self._cache_path = self._cache_root / \
+                FilesystemBasedCache._escape_key(key)
 
         def open_read(self) -> Optional[Cache.BucketReader]:
             try:
@@ -216,13 +279,15 @@ class FilesystemBasedCache(Cache):
             if not target.parent.exists():
                 target.parent.mkdir(exist_ok=True, parents=True)
 
-            fd, temp = tempfile.mkstemp(dir=self._cache_root, prefix='__temp__')
+            fd, temp = tempfile.mkstemp(
+                dir=self._cache_root, prefix='__temp__')
             return FilesystemBasedCache.FilesystemBucketWriter(os.fdopen(fd, 'wb'),
                                                                Path(temp), target)
 
     @property
     def _cache_root(self) -> Path:
-        xdg_cache_home = os.environ.get('XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
+        xdg_cache_home = os.environ.get(
+            'XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
         return Path(xdg_cache_home) / self._SUBDIR
 
     def get(self, key: str) -> Cache.BucketRef:
@@ -372,10 +437,6 @@ except ImportError:
     pass
 
 
-class CachedRequests(Requests):
-    pass
-
-
 class Integrity(NamedTuple):
     algorithm: str
     digest: str
@@ -464,7 +525,8 @@ class RemoteUrlMetadata(NamedTuple):
         metadata = RemoteUrlMetadata(integrity=builder.build(), size=size)
 
         with bucket.open_write() as bucket_writer:
-            bucket_writer.write(json.dumps(metadata.to_json_object()).encode('ascii'))
+            bucket_writer.write(json.dumps(
+                metadata.to_json_object()).encode('ascii'))
 
         return metadata
 
@@ -524,7 +586,40 @@ class Package(NamedTuple):
     lockfile: Path
 
 
-class ManifestGenerator(contextlib.AbstractContextManager):
+class NodeHeaders(NamedTuple):
+    target: str
+    runtime: str
+    disturl: str
+
+    @classmethod
+    def with_defaults(cls,
+                      target: str,
+                      runtime: Optional[str] = None,
+                      disturl: Optional[str] = None):
+        if runtime is None:
+            runtime = 'node'
+        if disturl is None:
+            if runtime == 'node':
+                disturl = 'http://nodejs.org/dist'
+            elif runtime == 'electron':
+                disturl = 'https://www.electronjs.org/headers'
+            else:
+                raise ValueError(
+                    f'Can\'t guess `disturl` for {runtime} version {target}')
+        return cls(target, runtime, disturl)
+
+    @property
+    def url(self) -> str:
+        # TODO it may be better to retrieve urls from disturl/index.json
+        return f'{self.disturl}/v{self.target}/node-v{self.target}-headers.tar.gz'
+
+    @property
+    def install_version(self) -> str:
+        # FIXME not sure if this static value will always work
+        return "9"
+
+
+class ManifestGenerator(ContextManager['ManifestGenerator']):
     MAX_GITHUB_SIZE = 49 * 1000 * 1000
     JSON_INDENT = 4
 
@@ -562,7 +657,8 @@ class ManifestGenerator(contextlib.AbstractContextManager):
         for source in self.ordered_sources():
             # Generate one source by itself, then check the length without the closing and
             # opening brackets.
-            source_json = json.dumps([source], indent=ManifestGenerator.JSON_INDENT)
+            source_json = json.dumps(
+                [source], indent=ManifestGenerator.JSON_INDENT)
             source_json_len = len('\n'.join(source_json.splitlines()[1:-1]))
             if current_size + source_json_len >= ManifestGenerator.MAX_GITHUB_SIZE:
                 yield current
@@ -630,10 +726,18 @@ class ManifestGenerator(contextlib.AbstractContextManager):
                                           only_arches=only_arches)
 
     def add_data_source(self, data: Union[str, bytes], destination: Path) -> None:
-        # Note that safe is empty so that slashes are escaped, to work around
+        if isinstance(data, bytes):
-        # https://gitlab.gnome.org/GNOME/libsoup/-/merge_requests/194
+            source = {
-        quoted = urllib.parse.quote(data, safe='')
+                'type': 'inline',
-        source = {'type': 'file', 'url': 'data:' + quoted}
+                'contents': base64.b64encode(data).decode('ascii'),
+                'base64': True,
+            }
+        else:
+            assert isinstance(data, str)
+            source = {
+                'type': 'inline',
+                'contents': data,
+            }
         self._add_source_with_destination(source, destination, is_dir=False)
 
     def add_git_source(self,
@@ -663,7 +767,8 @@ class ManifestGenerator(contextlib.AbstractContextManager):
 
     def _finalize(self) -> None:
         if self._commands:
-            self._add_source({'type': 'shell', 'commands': tuple(self._commands)})
+            self._add_source(
+                {'type': 'shell', 'commands': tuple(self._commands)})
 
 
 class LockfileProvider:
@@ -690,7 +795,33 @@ class LockfileProvider:
         raise NotImplementedError()
 
 
-class ModuleProvider(contextlib.AbstractContextManager):
+class RCFileProvider:
+    RCFILE_NAME: str
+
+    def parse_rcfile(self, rcfile: Path) -> Dict[str, str]:
+        with open(rcfile, 'r') as r:
+            rcfile_text = r.read()
+        parser_re = re.compile(r'^(?!#|;)(\S+)(?:\s+|\s*=\s*)(?:"(.+)"|(\S+))$',
+                               re.MULTILINE)
+        result: Dict[str, str] = {}
+        for key, quoted_val, val in parser_re.findall(rcfile_text):
+            result[key] = quoted_val or val
+        return result
+
+    def get_node_headers(self, rcfile: Path) -> Optional[NodeHeaders]:
+        rc_data = self.parse_rcfile(rcfile)
+        if 'target' not in rc_data:
+            return None
+        target = rc_data['target']
+        runtime = rc_data.get('runtime')
+        disturl = rc_data.get('disturl')
+
+        assert isinstance(runtime, str) and isinstance(disturl, str)
+
+        return NodeHeaders.with_defaults(target, runtime, disturl)
+
+
+class ModuleProvider(ContextManager['ModuleProvider']):
     async def generate_package(self, package: Package) -> None:
         raise NotImplementedError()
 
@@ -756,7 +887,8 @@ class ElectronBinaryManager:
         for line in integrity_data.splitlines():
             digest, star_filename = line.split()
             filename = star_filename.strip('*')
-            integrities[filename] = Integrity(algorithm='sha256', digest=digest)
+            integrities[filename] = Integrity(
+                algorithm='sha256', digest=digest)
 
         integrities[ElectronBinaryManager.INTEGRITY_BASE_FILENAME] = (
             Integrity.generate(integrity_data))
@@ -771,6 +903,7 @@ class SpecialSourceProvider:
         node_chromedriver_from_electron: str
         electron_ffmpeg: str
         electron_node_headers: bool
+        electron_from_rcfile: bool
         nwjs_version: str
         nwjs_node_headers: bool
         nwjs_ffmpeg: bool
@@ -781,6 +914,7 @@ class SpecialSourceProvider:
         self.node_chromedriver_from_electron = options.node_chromedriver_from_electron
         self.electron_ffmpeg = options.electron_ffmpeg
         self.electron_node_headers = options.electron_node_headers
+        self.electron_bins_for_headers = options.electron_from_rcfile
         self.nwjs_version = options.nwjs_version
         self.nwjs_node_headers = options.nwjs_node_headers
         self.nwjs_ffmpeg = options.nwjs_ffmpeg
@@ -792,6 +926,10 @@ class SpecialSourceProvider:
             return self.gen.data_root / 'cache' / 'electron'
         return self.gen.data_root / 'electron-cache'
 
+    @property
+    def gyp_dir(self) -> Path:
+        return self.gen.data_root / 'cache' / 'node-gyp'
+
     def _add_electron_cache_downloads(self,
                                       manager: ElectronBinaryManager,
                                       binary_name: str,
@@ -805,11 +943,11 @@ class SpecialSourceProvider:
                                     binary.integrity,
                                     electron_cache_dir / binary.filename,
                                     only_arches=[binary.arch.flatpak])
-            #Symlinks for @electron/get, which stores electron zips in a subdir
+            # Symlinks for @electron/get, which stores electron zips in a subdir
             if self.xdg_layout:
                 sanitized_url = ''.join(c for c in binary.url if c not in '/:')
 
-                #And for @electron/get >= 1.12.4 its sha256 hash of url dirname
+                # And for @electron/get >= 1.12.4 its sha256 hash of url dirname
                 url = urllib.parse.urlparse(binary.url)
                 url_dir = urllib.parse.urlunparse(
                     url._replace(path=os.path.dirname(url.path)))
@@ -821,16 +959,16 @@ class SpecialSourceProvider:
                     f'mkdir -p "{url_hash}"',
                     f'ln -s "../{binary.filename}" "{url_hash}/{binary.filename}"'
                 ],
-                                          destination=electron_cache_dir,
+                    destination=electron_cache_dir,
-                                          only_arches=[binary.arch.flatpak])
+                    only_arches=[binary.arch.flatpak])
 
         if add_integrities:
             integrity_file = manager.integrity_file
             self.gen.add_url_source(integrity_file.url, integrity_file.integrity,
                                     electron_cache_dir / integrity_file.filename)
 
-    async def _handle_electron(self, package: Package) -> None:
+    async def __add_electron(self, version: str) -> None:
-        manager = await ElectronBinaryManager.for_version(package.version)
+        manager = await ElectronBinaryManager.for_version(version)
         self._add_electron_cache_downloads(manager, 'electron')
 
         if self.electron_ffmpeg is not None:
@@ -848,32 +986,33 @@ class SpecialSourceProvider:
             else:
                 assert False, self.electron_ffmpeg
 
+    async def _handle_electron(self, package: Package) -> None:
+        await self.__add_electron(package.version)
+
     def _handle_gulp_atom_electron(self, package: Package) -> None:
         # Versions after 1.22.0 use @electron/get and don't need this
-        if StrictVersion(package.version) <= StrictVersion('1.22.0'):
+        if SemVer.parse(package.version) <= SemVer.parse('1.22.0'):
-            cache_path = self.gen.data_root / 'tmp' / 'gulp-electron-cache' / 'atom' / 'electron'
+            cache_path = self.gen.data_root / 'tmp' / \
+                'gulp-electron-cache' / 'atom' / 'electron'
             self.gen.add_command(f'mkdir -p "{cache_path.parent}"')
-            self.gen.add_command(f'ln -sfTr "{self.electron_cache_dir}" "{cache_path}"')
+            self.gen.add_command(
+                f'ln -sfTr "{self.electron_cache_dir}" "{cache_path}"')
 
     async def _handle_electron_headers(self, package: Package) -> None:
+        node_headers = NodeHeaders.with_defaults(runtime='electron',
+                                                 target=package.version)
         if self.xdg_layout:
-            node_gyp_headers_dir = self.gen.data_root / 'cache' / 'node-gyp' / package.version
+            node_gyp_headers_dir = self.gen.data_root / \
+                'cache' / 'node-gyp' / package.version
         else:
             node_gyp_headers_dir = self.gen.data_root / 'node-gyp' / 'electron-current'
-        url = f'https://www.electronjs.org/headers/v{package.version}/node-v{package.version}-headers.tar.gz'
+        await self.generate_node_headers(node_headers, dest=node_gyp_headers_dir)
-        metadata = await RemoteUrlMetadata.get(url, cachable=True)
-        self.gen.add_archive_source(url,
-                                    metadata.integrity,
-                                    destination=node_gyp_headers_dir)
-        if self.xdg_layout:
-            install_version_data = "9"
-            self.gen.add_data_source(install_version_data,
-                                     destination=node_gyp_headers_dir / 'installVersion')
 
     async def _get_chromedriver_binary_version(self, package: Package) -> str:
         # Note: node-chromedriver seems to not have tagged all releases on GitHub, so
         # just use unpkg instead.
-        url = urllib.parse.urljoin(NPM_MIRROR, f'chromedriver@{package.version}/lib/chromedriver')
+        url = urllib.parse.urljoin(NPM_MIRROR,
+                                   f'chromedriver@{package.version}/lib/chromedriver')
         js = await Requests.instance.read_all(url, cachable=True)
         # XXX: a tad ugly
         match = re.search(r"exports\.version = '([^']+)'", js.decode())
@@ -944,7 +1083,8 @@ class SpecialSourceProvider:
 
             if self.nwjs_ffmpeg:
                 ffmpeg_dl_url = f'{ffmpeg_dl_base}/{version}/{version}-{nwjs_arch}.zip'
-                ffmpeg_metadata = await RemoteUrlMetadata.get(ffmpeg_dl_url, cachable=True)
+                ffmpeg_metadata = await RemoteUrlMetadata.get(ffmpeg_dl_url,
+                                                              cachable=True)
                 self.gen.add_archive_source(ffmpeg_dl_url,
                                             ffmpeg_metadata.integrity,
                                             destination=dest,
@@ -959,7 +1099,8 @@ class SpecialSourceProvider:
                 'https://nwjs.io/versions.json', cachable=False))
             version = versions_json['latest'].lstrip('v')
         await self._add_nwjs_cache_downloads(version)
-        self.gen.add_data_source(version, destination=self.gen.data_root / 'nwjs-version')
+        self.gen.add_data_source(
+            version, destination=self.gen.data_root / 'nwjs-version')
 
     async def _handle_dugite_native(self, package: Package) -> None:
         dl_json_url = urllib.parse.urljoin(
@@ -996,7 +1137,8 @@ class SpecialSourceProvider:
             'arm': 'arm-unknown-linux-gnueabihf',
             'aarch64': 'aarch64-unknown-linux-gnu'
         }
-        destdir = self.gen.data_root / 'tmp' / f'vscode-ripgrep-cache-{package.version}'
+        destdir = self.gen.data_root / 'tmp' / \
+            f'vscode-ripgrep-cache-{package.version}'
         for arch, ripgrep_arch in ripgrep_arch_map.items():
             filename = f'ripgrep-{tag}-{ripgrep_arch}.tar.gz'
             url = f'https://github.com/microsoft/ripgrep-prebuilt/releases/download/{tag}/{filename}'
@@ -1008,7 +1150,10 @@ class SpecialSourceProvider:
 
     async def _handle_playwright(self, package: Package) -> None:
         base_url = f'https://github.com/microsoft/playwright/raw/v{package.version}/'
-        browsers_json_url = base_url + 'browsers.json'
+        if SemVer.parse(package.version) >= SemVer.parse('1.16.0'):
+            browsers_json_url = base_url + 'packages/playwright-core/browsers.json'
+        else:
+            browsers_json_url = base_url + 'browsers.json'
         browsers_json = json.loads(await Requests.instance.read_all(browsers_json_url,
                                                                     cachable=True))
         for browser in browsers_json['browsers']:
@@ -1044,7 +1189,8 @@ class SpecialSourceProvider:
 
             dl_url = url_tp % (revision, dl_file)
             metadata = await RemoteUrlMetadata.get(dl_url, cachable=True)
-            destdir = self.gen.data_root / 'cache' / 'ms-playwright' / f'{name}-{revision}'
+            destdir = self.gen.data_root / 'cache' / \
+                'ms-playwright' / f'{name}-{revision}'
             self.gen.add_archive_source(dl_url,
                                         metadata.integrity,
                                         destination=destdir,
@@ -1066,7 +1212,8 @@ class SpecialSourceProvider:
             metadata = await RemoteUrlMetadata.get(dl_url, cachable=True)
 
             cache_dst = self.gen.data_root / 'cache' / 'esbuild'
-            archive_dst = cache_dst / '.package' / f'{pkg_name}@{package.version}'
+            archive_dst = cache_dst / '.package' / \
+                f'{pkg_name}@{package.version}'
             bin_src = archive_dst / 'bin' / 'esbuild'
             bin_dst = cache_dst / 'bin' / f'{pkg_name}@{package.version}'
 
@@ -1094,13 +1241,29 @@ class SpecialSourceProvider:
         for electron_arch, flatpak_arch in (
                 ElectronBinaryManager.ELECTRON_ARCHES_TO_FLATPAK.items()):
             script.append(f'"{flatpak_arch}")')
-            script.append(f'  export ELECTRON_BUILDER_ARCH_ARGS="--{electron_arch}"')
+            script.append(
+                f'  export ELECTRON_BUILDER_ARCH_ARGS="--{electron_arch}"')
             script.append('  ;;')
 
         script.append('esac')
 
         self.gen.add_script_source(script, destination)
 
+    async def generate_node_headers(self,
+                                    node_headers: NodeHeaders,
+                                    dest: Optional[Path] = None):
+        url = node_headers.url
+        install_version = node_headers.install_version
+        if dest is None:
+            dest = self.gyp_dir / node_headers.target
+        metadata = await RemoteUrlMetadata.get(url, cachable=True)
+        self.gen.add_archive_source(url, metadata.integrity, destination=dest)
+        self.gen.add_data_source(
+            install_version, destination=dest / 'installVersion')
+
+        if self.electron_bins_for_headers and node_headers.runtime == "electron":
+            await self.__add_electron(node_headers.target)
+
     async def generate_special_sources(self, package: Package) -> None:
         if isinstance(Requests.instance, StubRequests):
             # This is going to crash and burn.
@@ -1108,7 +1271,7 @@ class SpecialSourceProvider:
 
         if package.name == 'electron':
             await self._handle_electron(package)
-            if self.electron_node_headers or self.xdg_layout:
+            if self.electron_node_headers:
                 await self._handle_electron_headers(package)
         elif package.name == 'electron-chromedriver':
             await self._handle_electron_chromedriver(package)
@@ -1122,7 +1285,7 @@ class SpecialSourceProvider:
             await self._handle_nw_builder(package)
         elif package.name in {'dugite', '@shiftkey/dugite'}:
             await self._handle_dugite_native(package)
-        elif package.name == 'vscode-ripgrep':
+        elif package.name in {'vscode-ripgrep', '@vscode/ripgrep'}:
             await self._handle_ripgrep_prebuilt(package)
         elif package.name == 'playwright':
             await self._handle_playwright(package)
@@ -1131,6 +1294,8 @@ class SpecialSourceProvider:
 
 
 class NpmLockfileProvider(LockfileProvider):
+    _ALIAS_RE = re.compile(r'^npm:(.[^@]*)@(.*)$')
+
     class Options(NamedTuple):
         no_devel: bool
 
@@ -1147,6 +1312,9 @@ class NpmLockfileProvider(LockfileProvider):
                 continue
 
             version: str = info['version']
+            alias_match = self._ALIAS_RE.match(version)
+            if alias_match is not None:
+                name, version = alias_match.groups()
 
             source: PackageSource
             if info.get('from'):
@@ -1171,6 +1339,10 @@ class NpmLockfileProvider(LockfileProvider):
         yield from self.process_dependencies(lockfile, data.get('dependencies', {}))
 
 
+class NpmRCFileProvider(RCFileProvider):
+    RCFILE_NAME = '.npmrc'
+
+
 class NpmModuleProvider(ModuleProvider):
     class Options(NamedTuple):
         registry: str
@@ -1204,7 +1376,9 @@ class NpmModuleProvider(ModuleProvider):
     def __exit__(self, exc_type: Optional[Type[BaseException]],
                  exc_value: Optional[BaseException],
                  tb: Optional[types.TracebackType]) -> None:
-        self._finalize()
+        # Don't bother finalizing if an exception was thrown.
+        if exc_type is None:
+            self._finalize()
 
     def get_cacache_integrity_path(self, integrity: Integrity) -> Path:
         digest = integrity.digest
@@ -1334,7 +1508,8 @@ class NpmModuleProvider(ModuleProvider):
             self.gen.add_data_source(raw_data, content_path)
             self.add_index_entry(index.url, metadata)
 
-        patch_commands: DefaultDict[Path, List[str]] = collections.defaultdict(lambda: [])
+        patch_commands: DefaultDict[Path, List[str]
+                                    ] = collections.defaultdict(lambda: [])
 
         if self.git_sources:
             # Generate jq scripts to patch the package*.json files.
@@ -1381,7 +1556,8 @@ class NpmModuleProvider(ModuleProvider):
                     data['package-lock.json'][original_version] = new_version
 
                 for filename, script in scripts.items():
-                    target = Path('$FLATPAK_BUILDER_BUILDDIR') / prefix / filename
+                    target = Path('$FLATPAK_BUILDER_BUILDDIR') / \
+                        prefix / filename
                     script = textwrap.dedent(script.lstrip('\n')).strip().replace(
                         '\n', '')
                     json_data = json.dumps(data[filename])
@@ -1401,39 +1577,20 @@ class NpmModuleProvider(ModuleProvider):
             patch_dest = patch_dest.with_name(patch_dest.name + '.sh')
 
             self.gen.add_script_source(patch_commands[lockfile], patch_dest)
-            patch_all_commands.append(f'$FLATPAK_BUILDER_BUILDDIR/{patch_dest}')
+            patch_all_commands.append(
+                f'$FLATPAK_BUILDER_BUILDDIR/{patch_dest}')
 
         patch_all_dest = self.gen.data_root / 'patch-all.sh'
         self.gen.add_script_source(patch_all_commands, patch_all_dest)
 
         if not self.no_autopatch:
             # FLATPAK_BUILDER_BUILDDIR isn't defined yet for script sources.
-            self.gen.add_command(f'FLATPAK_BUILDER_BUILDDIR=$PWD {patch_all_dest}')
+            self.gen.add_command(
+                f'FLATPAK_BUILDER_BUILDDIR=$PWD {patch_all_dest}')
 
         if self.index_entries:
-            # (ab-)use a "script" module to generate the index.
-            parents: Set[str] = set()
-
-            for path in self.index_entries:
-                for parent in map(str, path.relative_to(self.cacache_dir).parents):
-                    if parent != '.':
-                        parents.add(parent)
-
-            index_commands: List[str] = []
-            index_commands.append('import os')
-            index_commands.append(f'os.chdir({str(self.cacache_dir)!r})')
-
-            for parent in sorted(parents, key=len):
-                index_commands.append(f'os.makedirs({parent!r}, exist_ok=True)')
-
             for path, entry in self.index_entries.items():
-                path = path.relative_to(self.cacache_dir)
+                self.gen.add_data_source(entry, path)
-                index_commands.append(f'with open({str(path)!r}, "w") as fp:')
-                index_commands.append(f'    fp.write({entry!r})')
-
-            script_dest = self.gen.data_root / 'generate-index.py'
-            self.gen.add_script_source(index_commands, script_dest)
-            self.gen.add_command(f'python3 {script_dest}')
 
 
 class YarnLockfileProvider(LockfileProvider):
@@ -1521,6 +1678,10 @@ class YarnLockfileProvider(LockfileProvider):
             yield self.parse_package_section(lockfile, section)
 
 
+class YarnRCFileProvider(RCFileProvider):
+    RCFILE_NAME = '.yarnrc'
+
+
 class YarnModuleProvider(ModuleProvider):
     # From https://github.com/yarnpkg/yarn/blob/v1.22.4/src/fetchers/tarball-fetcher.js
     _PACKAGE_TARBALL_URL_RE = re.compile(
@@ -1570,6 +1731,9 @@ class ProviderFactory:
     def create_lockfile_provider(self) -> LockfileProvider:
         raise NotImplementedError()
 
+    def create_rcfile_providers(self) -> List[RCFileProvider]:
+        raise NotImplementedError()
+
     def create_module_provider(self, gen: ManifestGenerator,
                                special: SpecialSourceProvider) -> ModuleProvider:
         raise NotImplementedError()
@@ -1587,6 +1751,9 @@ class NpmProviderFactory(ProviderFactory):
     def create_lockfile_provider(self) -> NpmLockfileProvider:
         return NpmLockfileProvider(self.options.lockfile)
 
+    def create_rcfile_providers(self) -> List[RCFileProvider]:
+        return [NpmRCFileProvider()]
+
     def create_module_provider(self, gen: ManifestGenerator,
                                special: SpecialSourceProvider) -> NpmModuleProvider:
         return NpmModuleProvider(gen, special, self.lockfile_root, self.options.module)
@@ -1599,12 +1766,15 @@ class YarnProviderFactory(ProviderFactory):
     def create_lockfile_provider(self) -> YarnLockfileProvider:
         return YarnLockfileProvider()
 
+    def create_rcfile_providers(self) -> List[RCFileProvider]:
+        return [YarnRCFileProvider(), NpmRCFileProvider()]
+
     def create_module_provider(self, gen: ManifestGenerator,
                                special: SpecialSourceProvider) -> YarnModuleProvider:
         return YarnModuleProvider(gen, special)
 
 
-class GeneratorProgress(contextlib.AbstractContextManager):
+class GeneratorProgress(ContextManager['GeneratorProgress']):
     def __init__(self, packages: Collection[Package],
                  module_provider: ModuleProvider) -> None:
         self.finished = 0
@@ -1653,8 +1823,19 @@ class GeneratorProgress(contextlib.AbstractContextManager):
 
     async def run(self) -> None:
         self._update()
-        await asyncio.wait(
+
-            [asyncio.create_task(self._generate(pkg)) for pkg in self.packages])
+        tasks = [asyncio.create_task(self._generate(pkg))
+                 for pkg in self.packages]
+        for coro in asyncio.as_completed(tasks):
+            try:
+                await coro
+            except:
+                # If an exception occurred, make sure to cancel all the other
+                # tasks.
+                for task in tasks:
+                    task.cancel()
+
+                raise
 
 
 def scan_for_lockfiles(base: Path, patterns: List[str]) -> Iterator[Path]:
@@ -1723,18 +1904,24 @@ async def main() -> None:
     parser.add_argument('--electron-node-headers',
                         action='store_true',
                         help='Download the electron node headers')
+    parser.add_argument('--electron-from-rcfile',
+                        action='store_true',
+                        help='Download electron version corresponding to '
+                        'the node headers version(s) from .yarnrc/.npmrc')
     parser.add_argument('--nwjs-version',
                         help='Specify NW.js version (will use latest otherwise)')
     parser.add_argument('--nwjs-node-headers',
                         action='store_true',
                         help='Download the NW.js node headers')
-    parser.add_argument('--nwjs-ffmpeg', action='store_true',
+    parser.add_argument('--nwjs-ffmpeg',
+                        action='store_true',
                         help='Download prebuilt ffmpeg for current NW.js version')
     parser.add_argument('--xdg-layout',
                         action='store_true',
                         help='Use XDG layout for caches')
     # Internal option, useful for testing.
-    parser.add_argument('--stub-requests', action='store_true', help=argparse.SUPPRESS)
+    parser.add_argument('--stub-requests',
+                        action='store_true', help=argparse.SUPPRESS)
 
     args = parser.parse_args()
 
@@ -1753,7 +1940,8 @@ async def main() -> None:
         if Requests.instance.is_async:
             Requests.instance = UrllibRequests()
     elif not Requests.instance.is_async:
-        print('WARNING: aiohttp is not found, performance will suffer.', file=sys.stderr)
+        print('WARNING: aiohttp is not found, performance will suffer.',
+              file=sys.stderr)
         print('  (Pass --no-aiohttp to silence this warning.)', file=sys.stderr)
 
     if not args.no_requests_cache:
@@ -1761,7 +1949,8 @@ async def main() -> None:
 
     lockfiles: List[Path]
     if args.recursive or args.recursive_pattern:
-        lockfiles = list(scan_for_lockfiles(Path(args.lockfile), args.recursive_pattern))
+        lockfiles = list(scan_for_lockfiles(
+            Path(args.lockfile), args.recursive_pattern))
         if not lockfiles:
             sys.exit('No lockfiles found.')
         print(f'Found {len(lockfiles)} lockfiles.')
@@ -1785,11 +1974,21 @@ async def main() -> None:
 
     print('Reading packages from lockfiles...')
     packages: Set[Package] = set()
+    rcfile_node_headers: Set[NodeHeaders] = set()
 
     for lockfile in lockfiles:
         lockfile_provider = provider_factory.create_lockfile_provider()
+        rcfile_providers = provider_factory.create_rcfile_providers()
+
         packages.update(lockfile_provider.process_lockfile(lockfile))
 
+        for rcfile_provider in rcfile_providers:
+            rcfile = lockfile.parent / rcfile_provider.RCFILE_NAME
+            if rcfile.is_file():
+                nh = rcfile_provider.get_node_headers(rcfile)
+                if nh is not None:
+                    rcfile_node_headers.add(nh)
+
     print(f'{len(packages)} packages read.')
 
     gen = ManifestGenerator()
@@ -1802,12 +2001,16 @@ async def main() -> None:
             nwjs_ffmpeg=args.nwjs_ffmpeg,
             xdg_layout=args.xdg_layout,
             electron_ffmpeg=args.electron_ffmpeg,
-            electron_node_headers=args.electron_node_headers)
+            electron_node_headers=args.electron_node_headers,
+            electron_from_rcfile=args.electron_from_rcfile)
         special = SpecialSourceProvider(gen, options)
 
         with provider_factory.create_module_provider(gen, special) as module_provider:
             with GeneratorProgress(packages, module_provider) as progress:
                 await progress.run()
+        for headers in rcfile_node_headers:
+            print(f'Generating headers {headers.runtime} @ {headers.target}')
+            await special.generate_node_headers(headers)
 
         if args.xdg_layout:
             script_name = "setup_sdk_node_headers.sh"
@@ -1819,7 +2022,7 @@ async def main() -> None:
                 f'ln -s "$nodedir/include" "{node_gyp_dir}/$version/include"',
                 f'echo 9 > "{node_gyp_dir}/$version/installVersion"',
             ],
-                                  destination=gen.data_root / script_name)
+                destination=gen.data_root / script_name)
             gen.add_command(f"bash {gen.data_root / script_name}")
 
     if args.split:
@@ -1846,4 +2049,4 @@ async def main() -> None:
 
 
 if __name__ == '__main__':
-    asyncio.run(main())
+    asyncio.run(main())

package-append.json

@@ -22,8 +22,8 @@
     "build-7z-x86": "electron-builder --ia32 --linux 7z"
   },
   "devDependencies": {
-    "electron": "^13.5.1",
+    "electron": "^13.6.9",
-    "electron-builder": "^22.13.1"
+    "electron-builder": "^23.0.3"
   },
   "build": {
     "files": [