Skip to content

Commit e9dc45a

Browse files
devversionandrewseguin
devversion
authored and
andrewseguin
committed
build: option to automatically retry postinstall patches (#19615)
Sometimes we need to change existing postinstall patches. This requires a cleanup of the currently installed node modules, but there is no good warning/messaging. We improve this with this commit by detecting such stale patches and prompting for automatic retry with cleaned up node modulesbuild: option to automatically retry postinstall patches Sometimes we need to change existing postinstall patches. This requires a cleanup of the currently installed node modules, but there is no good warning/messaging. We improve this with this commit by detecting such stale patches and prompting for automatic retry with cleaned up node modules. Note: Please clean up your node modules after this change. This needs to still happen manually once, so that we can keep track of applied patch versions and report/prompt automatically in the future.
1 parent 2b5dcc2 commit e9dc45a

File tree

1 file changed

+213
-144
lines changed

1 file changed

+213
-144
lines changed

‎tools/postinstall/apply-patches.js‎

Lines changed: 213 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
const shelljs = require('shelljs');
88
const path = require('path');
99
const fs = require('fs');
10+
const inquirer = require('inquirer');
11+
const chalk = require('chalk');
1012

1113
/**
1214
* Version of the post install patch. Needs to be incremented when
@@ -23,154 +25,172 @@ const projectDir = path.join(__dirname, '../..');
2325
*/
2426
const PATCHES_PER_FILE = {};
2527

26-
shelljs.set('-e');
27-
shelljs.cd(projectDir);
28-
29-
// Workaround for https://github.com/angular/angular/issues/18810.
30-
shelljs.exec('ngc -p angular-tsconfig.json');
31-
32-
// Workaround for: https://github.com/angular/angular/issues/32651. We just do not
33-
// generate re-exports for secondary entry-points. Similar to what "ng-packagr" does.
34-
searchAndReplace(
35-
/(?!function\s+)createMetadataReexportFile\([^)]+\);/, '',
36-
'node_modules/@angular/bazel/src/ng_package/packager.js');
37-
searchAndReplace(
38-
/(?!function\s+)createTypingsReexportFile\([^)]+\);/, '',
39-
'node_modules/@angular/bazel/src/ng_package/packager.js');
40-
41-
// Workaround for: https://github.com/angular/angular/pull/32650
42-
searchAndReplace(
43-
'var indexFile;', `
44-
var indexFile = files.find(f => f.endsWith('/public-api.ts'));
45-
`,
46-
'node_modules/@angular/compiler-cli/src/metadata/bundle_index_host.js');
47-
searchAndReplace(
48-
'var resolvedEntryPoint = null;', `
49-
var resolvedEntryPoint = tsFiles.find(f => f.endsWith('/public-api.ts')) || null;
50-
`,
51-
'node_modules/@angular/compiler-cli/src/ngtsc/entry_point/src/logic.js');
52-
53-
// Workaround for: https://hackmd.io/MlqFp-yrSx-0mw4rD7dnQQ?both. We only want to discard
54-
// the metadata of files in the bazel managed node modules. That way we keep the default
55-
// behavior of ngc-wrapped except for dependencies between sources of the library. This makes
56-
// the "generateCodeForLibraries" flag more accurate in the Bazel environment where previous
57-
// compilations should not be treated as external libraries. Read more about this in the document.
58-
searchAndReplace(
59-
/if \((this\.options\.generateCodeForLibraries === false)/, `
60-
const fs = require('fs');
61-
const hasFlatModuleBundle = fs.existsSync(filePath.replace('.d.ts', '.metadata.json'));
62-
if ((filePath.includes('node_modules/') || !hasFlatModuleBundle) && $1`,
63-
'node_modules/@angular/compiler-cli/src/transformers/compiler_host.js');
64-
applyPatch(path.join(__dirname, './flat_module_factory_resolution.patch'));
65-
// The three replacements below ensure that metadata files can be read by NGC and
66-
// that metadata files are collected as Bazel action inputs.
67-
searchAndReplace(
68-
/(const NGC_ASSETS = \/[^(]+\()([^)]*)(\).*\/;)/, '$1$2|metadata.json$3',
69-
'node_modules/@angular/bazel/src/ngc-wrapped/index.js');
70-
searchAndReplace(
71-
/^((\s*)results = depset\(dep.angular.summaries, transitive = \[results]\))$/m,
72-
`$1#\n$2results = depset(dep.angular.metadata, transitive = [results])`,
73-
'node_modules/@angular/bazel/src/ng_module.bzl');
74-
searchAndReplace(
75-
/^((\s*)results = depset\(target.angular\.summaries if _has_target_angular_summaries\(target\) else \[]\))$/m,
76-
`$1#\n$2results = depset(target.angular.metadata if _has_target_angular_summaries(target) else [], transitive = [results])`,
77-
'node_modules/@angular/bazel/src/ng_module.bzl');
78-
// Ensure that "metadata" of transitive dependencies can be collected.
79-
searchAndReplace(
80-
/providers\["angular"]\["metadata"] = outs\.metadata/,
81-
`$& + [m for dep in ctx.attr.deps if (hasattr(dep, "angular") and hasattr(dep.angular, "metadata")) for m in dep.angular.metadata]`,
82-
'node_modules/@angular/bazel/src/ng_module.bzl');
83-
84-
// Workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1208.
85-
applyPatch(path.join(__dirname, './manifest_externs_hermeticity.patch'));
86-
87-
try {
88-
// Temporary patch pre-req for https://github.com/angular/angular/pull/36333.
89-
// Can be removed once @angular/bazel is updated here to include this patch.
90-
// try/catch needed for this the material CI tests to work in angular/repo
91-
applyPatch(path.join(__dirname, './@angular_bazel_ng_module.patch'));
92-
} catch {}
93-
94-
try {
95-
// Temporary patch pre-req for https://github.com/angular/angular/pull/36971.
96-
// Can be removed once @angular/bazel is updated here to include this patch.
97-
// try/catch needed for this as the framework repo has this patch already applied,
98-
// and re-applying again causes an error.
99-
applyPatch(path.join(__dirname, './@angular_bazel_ivy_flat_module.patch'));
100-
} catch {}
101-
102-
// Workaround for https://github.com/angular/angular/issues/33452:
103-
searchAndReplace(/angular_compiler_options = {/, `$&
104-
"strictTemplates": True,`, 'node_modules/@angular/bazel/src/ng_module.bzl');
105-
106-
// More info in https://github.com/angular/angular/pull/33786
107-
shelljs.rm('-rf', [
108-
'node_modules/rxjs/add/',
109-
'node_modules/rxjs/observable/',
110-
'node_modules/rxjs/operator/',
111-
// rxjs/operators is a public entry point that also contains files to support legacy deep import
112-
// paths, so we need to preserve index.* and package.json files that are required for module
113-
// resolution.
114-
'node_modules/rxjs/operators/!(index.*|package.json)',
115-
'node_modules/rxjs/scheduler/',
116-
'node_modules/rxjs/symbol/',
117-
'node_modules/rxjs/util/',
118-
'node_modules/rxjs/internal/Rx.d.ts',
119-
'node_modules/rxjs/AsyncSubject.*',
120-
'node_modules/rxjs/BehaviorSubject.*',
121-
'node_modules/rxjs/InnerSubscriber.*',
122-
'node_modules/rxjs/interfaces.*',
123-
'node_modules/rxjs/Notification.*',
124-
'node_modules/rxjs/Observable.*',
125-
'node_modules/rxjs/Observer.*',
126-
'node_modules/rxjs/Operator.*',
127-
'node_modules/rxjs/OuterSubscriber.*',
128-
'node_modules/rxjs/ReplaySubject.*',
129-
'node_modules/rxjs/Rx.*',
130-
'node_modules/rxjs/Scheduler.*',
131-
'node_modules/rxjs/Subject.*',
132-
'node_modules/rxjs/SubjectSubscription.*',
133-
'node_modules/rxjs/Subscriber.*',
134-
'node_modules/rxjs/Subscription.*',
135-
]);
136-
137-
// Apply all collected patches on a per-file basis. This is necessary because
138-
// multiple edits might apply to the same file, and we only want to mark a given
139-
// file as patched once all edits have been made.
140-
Object.keys(PATCHES_PER_FILE).forEach(filePath => {
141-
if (hasFileBeenPatched(filePath)) {
142-
console.info('File ' + filePath + ' is already patched. Skipping..');
143-
return;
28+
const PATCH_MARKER_FILE_PATH = path.join(projectDir, 'node_modules/_ng-comp-patch-marker.json');
29+
30+
/** Registry of applied patches. */
31+
let registry = null;
32+
33+
main();
34+
35+
async function main() {
36+
shelljs.set('-e');
37+
shelljs.cd(projectDir);
38+
39+
registry = await readAndValidatePatchMarker();
40+
41+
// Apply all patches synchronously.
42+
applyPatches();
43+
44+
// Write the patch marker file so that we don't accidentally re-apply patches
45+
// in subsequent Yarn installations.
46+
fs.writeFileSync(PATCH_MARKER_FILE_PATH, JSON.stringify(registry, null, 2));
47+
}
48+
49+
function applyPatches() {
50+
// Workaround for https://github.com/angular/angular/issues/18810.
51+
shelljs.exec('ngc -p angular-tsconfig.json');
52+
53+
// Workaround for: https://github.com/angular/angular/issues/32651. We just do not
54+
// generate re-exports for secondary entry-points. Similar to what "ng-packagr" does.
55+
searchAndReplace(
56+
/(?!function\s+)createMetadataReexportFile\([^)]+\);/, '',
57+
'node_modules/@angular/bazel/src/ng_package/packager.js');
58+
searchAndReplace(
59+
/(?!function\s+)createTypingsReexportFile\([^)]+\);/, '',
60+
'node_modules/@angular/bazel/src/ng_package/packager.js');
61+
62+
// Workaround for: https://github.com/angular/angular/pull/32650
63+
searchAndReplace(
64+
'var indexFile;', `
65+
var indexFile = files.find(f => f.endsWith('/public-api.ts'));
66+
`,
67+
'node_modules/@angular/compiler-cli/src/metadata/bundle_index_host.js');
68+
searchAndReplace(
69+
'var resolvedEntryPoint = null;', `
70+
var resolvedEntryPoint = tsFiles.find(f => f.endsWith('/public-api.ts')) || null;
71+
`,
72+
'node_modules/@angular/compiler-cli/src/ngtsc/entry_point/src/logic.js');
73+
74+
// Workaround for: https://hackmd.io/MlqFp-yrSx-0mw4rD7dnQQ?both. We only want to discard
75+
// the metadata of files in the bazel managed node modules. That way we keep the default
76+
// behavior of ngc-wrapped except for dependencies between sources of the library. This makes
77+
// the "generateCodeForLibraries" flag more accurate in the Bazel environment where previous
78+
// compilations should not be treated as external libraries. Read more about this in the document.
79+
searchAndReplace(
80+
/if \((this\.options\.generateCodeForLibraries === false)/, `
81+
const fs = require('fs');
82+
const hasFlatModuleBundle = fs.existsSync(filePath.replace('.d.ts', '.metadata.json'));
83+
if ((filePath.includes('node_modules/') || !hasFlatModuleBundle) && $1`,
84+
'node_modules/@angular/compiler-cli/src/transformers/compiler_host.js');
85+
applyPatch(path.join(__dirname, './flat_module_factory_resolution.patch'));
86+
// The three replacements below ensure that metadata files can be read by NGC and
87+
// that metadata files are collected as Bazel action inputs.
88+
searchAndReplace(
89+
/(const NGC_ASSETS = \/[^(]+\()([^)]*)(\).*\/;)/, '$1$2|metadata.json$3',
90+
'node_modules/@angular/bazel/src/ngc-wrapped/index.js');
91+
searchAndReplace(
92+
/^((\s*)results = depset\(dep.angular.summaries, transitive = \[results]\))$/m,
93+
`$1#\n$2results = depset(dep.angular.metadata, transitive = [results])`,
94+
'node_modules/@angular/bazel/src/ng_module.bzl');
95+
searchAndReplace(
96+
/^((\s*)results = depset\(target.angular\.summaries if _has_target_angular_summaries\(target\) else \[]\))$/m,
97+
`$1#\n$2results = depset(target.angular.metadata if _has_target_angular_summaries(target) else [], transitive = [results])`,
98+
'node_modules/@angular/bazel/src/ng_module.bzl');
99+
// Ensure that "metadata" of transitive dependencies can be collected.
100+
searchAndReplace(
101+
/providers\["angular"]\["metadata"] = outs\.metadata/,
102+
`$& + [m for dep in ctx.attr.deps if (hasattr(dep, "angular") and hasattr(dep.angular, "metadata")) for m in dep.angular.metadata]`,
103+
'node_modules/@angular/bazel/src/ng_module.bzl');
104+
105+
// Workaround for: https://github.com/bazelbuild/rules_nodejs/issues/1208.
106+
applyPatch(path.join(__dirname, './manifest_externs_hermeticity.patch'));
107+
108+
try {
109+
// Temporary patch pre-req for https://github.com/angular/angular/pull/36333.
110+
// Can be removed once @angular/bazel is updated here to include this patch.
111+
// try/catch needed for this the material CI tests to work in angular/repo
112+
applyPatch(path.join(__dirname, './@angular_bazel_ng_module.patch'));
113+
} catch {
144114
}
145115

146-
let content = fs.readFileSync(filePath, 'utf8');
147-
const patchFunctions = PATCHES_PER_FILE[filePath];
116+
try {
117+
// Temporary patch pre-req for https://github.com/angular/angular/pull/36971.
118+
// Can be removed once @angular/bazel is updated here to include this patch.
119+
// try/catch needed for this as the framework repo has this patch already applied,
120+
// and re-applying again causes an error.
121+
applyPatch(path.join(__dirname, './@angular_bazel_ivy_flat_module.patch'));
122+
} catch {
123+
}
124+
125+
// Workaround for https://github.com/angular/angular/issues/33452:
126+
searchAndReplace(
127+
/angular_compiler_options = {/, `$&
128+
"strictTemplates": True,`,
129+
'node_modules/@angular/bazel/src/ng_module.bzl');
148130

149-
console.info(`Patching file ${filePath} with ${patchFunctions.length} edits..`);
150-
patchFunctions.forEach(patchFn => content = patchFn(content));
131+
// More info in https://github.com/angular/angular/pull/33786
132+
shelljs.rm('-rf', [
133+
'node_modules/rxjs/add/',
134+
'node_modules/rxjs/observable/',
135+
'node_modules/rxjs/operator/',
136+
// rxjs/operators is a public entry point that also contains files to support legacy deep import
137+
// paths, so we need to preserve index.* and package.json files that are required for module
138+
// resolution.
139+
'node_modules/rxjs/operators/!(index.*|package.json)',
140+
'node_modules/rxjs/scheduler/',
141+
'node_modules/rxjs/symbol/',
142+
'node_modules/rxjs/util/',
143+
'node_modules/rxjs/internal/Rx.d.ts',
144+
'node_modules/rxjs/AsyncSubject.*',
145+
'node_modules/rxjs/BehaviorSubject.*',
146+
'node_modules/rxjs/InnerSubscriber.*',
147+
'node_modules/rxjs/interfaces.*',
148+
'node_modules/rxjs/Notification.*',
149+
'node_modules/rxjs/Observable.*',
150+
'node_modules/rxjs/Observer.*',
151+
'node_modules/rxjs/Operator.*',
152+
'node_modules/rxjs/OuterSubscriber.*',
153+
'node_modules/rxjs/ReplaySubject.*',
154+
'node_modules/rxjs/Rx.*',
155+
'node_modules/rxjs/Scheduler.*',
156+
'node_modules/rxjs/Subject.*',
157+
'node_modules/rxjs/SubjectSubscription.*',
158+
'node_modules/rxjs/Subscriber.*',
159+
'node_modules/rxjs/Subscription.*',
160+
]);
161+
162+
// Apply all collected patches on a per-file basis. This is necessary because
163+
// multiple edits might apply to the same file, and we only want to mark a given
164+
// file as patched once all edits have been made.
165+
Object.keys(PATCHES_PER_FILE).forEach(filePath => {
166+
if (isFilePatched(filePath)) {
167+
console.info('File ' + filePath + ' is already patched. Skipping..');
168+
return;
169+
}
151170

152-
fs.writeFileSync(filePath, content, 'utf8');
153-
writePatchMarker(filePath);
154-
});
171+
let content = fs.readFileSync(filePath, 'utf8');
172+
const patchFunctions = PATCHES_PER_FILE[filePath];
173+
174+
console.info(`Patching file ${filePath} with ${patchFunctions.length} edits..`);
175+
patchFunctions.forEach(patchFn => content = patchFn(content));
176+
177+
fs.writeFileSync(filePath, content, 'utf8');
178+
captureFileAsPatched(filePath);
179+
});
180+
}
155181

156182
/**
157-
* Applies the given patch if not done already. Throws if the patch does
158-
* not apply cleanly.
183+
* Applies the given patch if not done already. Throws if the patch
184+
* does not apply cleanly.
159185
*/
160186
function applyPatch(patchFile) {
161-
// Note: We replace non-word characters from the patch marker file name.
162-
// This is necessary because Yarn throws if cached node modules are restored
163-
// which contain files with special characters. Below is an example error:
164-
// ENOTDIR: not a directory, scandir '/<...>/node_modules/@angular_bazel_ng_module.<..>'".
165-
const patchMarkerBasename = `${path.basename(patchFile).replace(/[^\w]/, '_')}`;
166-
const patchMarkerPath = path.join(projectDir, 'node_modules/', patchMarkerBasename);
167-
168-
if (hasFileBeenPatched(patchMarkerPath)) {
187+
if (isFilePatched(patchFile)) {
188+
console.info('Patch: ' + patchFile + ' has been applied already. Skipping..');
169189
return;
170190
}
171191

172192
shelljs.cat(patchFile).exec('patch -p0');
173-
writePatchMarker(patchMarkerPath);
193+
captureFileAsPatched(patchFile);
174194
}
175195

176196
/**
@@ -185,21 +205,70 @@ function searchAndReplace(search, replacement, relativeFilePath) {
185205
fileEdits.push(originalContent => {
186206
const newFileContent = originalContent.replace(search, replacement);
187207
if (originalContent === newFileContent) {
188-
throw Error(`Could not perform replacement in: ${filePath}.\n` +
208+
throw Error(
209+
`Could not perform replacement in: ${filePath}.\n` +
189210
`Searched for pattern: ${search}`);
190211
}
191212
return newFileContent;
192213
});
193214
}
194215

216+
/** Gets a project unique id for a given file path. */
217+
function getIdForFile(filePath) {
218+
return path.relative(projectDir, filePath).replace(/\\/g, '/');
219+
}
220+
195221
/** Marks the specified file as patched. */
196-
function writePatchMarker(filePath) {
197-
new shelljs.ShellString(PATCH_VERSION).to(`${filePath}.patch_marker`);
222+
function captureFileAsPatched(filePath) {
223+
registry.patched[getIdForFile(filePath)] = true;
224+
}
225+
226+
/** Checks whether the given file is patched. */
227+
function isFilePatched(filePath) {
228+
return registry.patched[getIdForFile(filePath)] === true;
198229
}
199230

200-
/** Checks if the given file has been patched. */
201-
function hasFileBeenPatched(filePath) {
202-
const markerFilePath = `${filePath}.patch_marker`;
203-
return shelljs.test('-e', markerFilePath) &&
204-
shelljs.cat(markerFilePath).toString().trim() === `${PATCH_VERSION}`;
231+
/**
232+
* Reads the patch marker from the node modules if present. Validates that applied
233+
* patches are up-to-date. If not, an error will be reported with a prompt that
234+
* allows convenient clean up of node modules in case those need to be cleaned up.
235+
*/
236+
async function readAndValidatePatchMarker() {
237+
if (!shelljs.test('-e', PATCH_MARKER_FILE_PATH)) {
238+
return {version: PATCH_VERSION, patched: {}};
239+
}
240+
const registry = JSON.parse(shelljs.cat(PATCH_MARKER_FILE_PATH));
241+
// If the node modules are up-to-date, return the parsed patch registry.
242+
if (registry.version === PATCH_VERSION) {
243+
return registry;
244+
}
245+
// Print errors that explain the current situation where patches from another
246+
// postinstall patch revision are applied in the current node modules.
247+
if (registry.version < PATCH_VERSION) {
248+
console.error(chalk.red('Your node modules have been patched by a previous Yarn install.'));
249+
console.error(chalk.red('The postinstall patches have changed since then, and in order to'));
250+
console.error(chalk.red('apply the most recent patches, your node modules need to be cleaned'));
251+
console.error(chalk.red('up from past changes.'));
252+
} else {
253+
console.error(chalk.red('Your node modules already have patches applied from a more recent.'));
254+
console.error(chalk.red('revision of the components repository. In order to be able to apply'));
255+
console.error(chalk.red('patches for the current revision, your node modules need to be'));
256+
console.error(chalk.red('cleaned up.'));
257+
}
258+
259+
const {cleanupModules} = await inquirer.prompt({
260+
name: 'cleanupModules',
261+
type: 'confirm',
262+
message: 'Clean up node modules automatically?',
263+
default: false
264+
});
265+
266+
if (cleanupModules) {
267+
// This re-runs Yarn with `--check-files` mode. The postinstall will rerun afterwards,
268+
// so we can exit with a zero exit-code here.
269+
shelljs.exec('yarn --check-files --frozen-lockfile', {cwd: projectDir});
270+
process.exit(0);
271+
} else {
272+
process.exit(1);
273+
}
205274
}

0 commit comments

Comments
 (0)