Skip to content

Commit

Permalink
Add config key invalidation (#9597)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattcompiles
mattcompiles authored Mar 26, 2024
1 parent ec3f4a5 commit 84fc9a9
Show file tree
Hide file tree
Showing 13 changed files with 424 additions and 58 deletions.
6 changes: 6 additions & 0 deletions packages/core/core/src/InternalConfig.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ type ConfigOpts = {|
env?: Environment,
result?: ConfigResult,
invalidateOnFileChange?: Set<ProjectPath>,
invalidateOnConfigKeyChange?: Array<{|
filePath: ProjectPath,
configKey: string,
|}>,
invalidateOnFileCreate?: Array<InternalFileCreateInvalidation>,
invalidateOnEnvChange?: Set<string>,
invalidateOnOptionChange?: Set<string>,
Expand All @@ -35,6 +39,7 @@ export function createConfig({
env,
result,
invalidateOnFileChange,
invalidateOnConfigKeyChange,
invalidateOnFileCreate,
invalidateOnEnvChange,
invalidateOnOptionChange,
Expand All @@ -56,6 +61,7 @@ export function createConfig({
result: result ?? null,
cacheKey: null,
invalidateOnFileChange: invalidateOnFileChange ?? new Set(),
invalidateOnConfigKeyChange: invalidateOnConfigKeyChange ?? [],
invalidateOnFileCreate: invalidateOnFileCreate ?? [],
invalidateOnEnvChange: invalidateOnEnvChange ?? new Set(),
invalidateOnOptionChange: invalidateOnOptionChange ?? new Set(),
Expand Down
5 changes: 2 additions & 3 deletions packages/core/core/src/Parcel.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,15 +417,14 @@ export default class Parcel {
let opts = getWatcherOptions(resolvedOptions);
let sub = await resolvedOptions.inputFS.watch(
resolvedOptions.watchDir,
(err, events) => {
async (err, events) => {
if (err) {
this.#watchEvents.emit({error: err});
return;
}

let isInvalid = this.#requestTracker.respondToFSEvents(
let isInvalid = await this.#requestTracker.respondToFSEvents(
events,
resolvedOptions.projectRoot,
Number.POSITIVE_INFINITY,
);
if (isInvalid && this.#watchQueue.getNumWaiting() === 0) {
Expand Down
135 changes: 121 additions & 14 deletions packages/core/core/src/RequestTracker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import type {
InternalGlob,
} from './types';
import logger from '@parcel/logger';
import type {Deferred} from '@parcel/utils';
import {type Deferred} from '@parcel/utils';

import invariant from 'assert';
import nullthrows from 'nullthrows';
Expand Down Expand Up @@ -53,6 +53,7 @@ import {
import {report} from './ReporterRunner';
import {PromiseQueue} from '@parcel/utils';
import type {Cache} from '@parcel/cache';
import {getConfigKeyContentHash} from './requests/ConfigRequest';

export const requestGraphEdgeTypes = {
subrequest: 2,
Expand All @@ -75,6 +76,7 @@ type RequestGraphOpts = {|
unpredicatableNodeIds: Set<NodeId>,
invalidateOnBuildNodeIds: Set<NodeId>,
cachedRequestChunks: Set<number>,
configKeyNodes: Map<ProjectPath, Set<NodeId>>,
|};

type SerializedRequestGraph = {|
Expand All @@ -87,6 +89,7 @@ type SerializedRequestGraph = {|
unpredicatableNodeIds: Set<NodeId>,
invalidateOnBuildNodeIds: Set<NodeId>,
cachedRequestChunks: Set<number>,
configKeyNodes: Map<ProjectPath, Set<NodeId>>,
|};

const FILE: 0 = 0;
Expand All @@ -95,6 +98,8 @@ const FILE_NAME: 2 = 2;
const ENV: 3 = 3;
const OPTION: 4 = 4;
const GLOB: 5 = 5;
const CONFIG_KEY: 6 = 6;

type FileNode = {|id: ContentKey, +type: typeof FILE|};

type GlobNode = {|id: ContentKey, +type: typeof GLOB, value: InternalGlob|};
Expand All @@ -116,6 +121,13 @@ type OptionNode = {|
hash: string,
|};

type ConfigKeyNode = {|
id: ContentKey,
+type: typeof CONFIG_KEY,
configKey: string,
contentHash: string,
|};

type Request<TInput, TResult> = {|
id: string,
+type: RequestType,
Expand Down Expand Up @@ -159,12 +171,18 @@ type RequestGraphNode =
| GlobNode
| FileNameNode
| EnvNode
| OptionNode;
| OptionNode
| ConfigKeyNode;

export type RunAPI<TResult> = {|
invalidateOnFileCreate: InternalFileCreateInvalidation => void,
invalidateOnFileDelete: ProjectPath => void,
invalidateOnFileUpdate: ProjectPath => void,
invalidateOnConfigKeyChange: (
filePath: ProjectPath,
configKey: string,
contentHash: string,
) => void,
invalidateOnStartup: () => void,
invalidateOnBuild: () => void,
invalidateOnEnvChange: string => void,
Expand Down Expand Up @@ -226,6 +244,17 @@ const nodeFromOption = (option: string, value: mixed): RequestGraphNode => ({
hash: hashFromOption(value),
});

const nodeFromConfigKey = (
fileName: ProjectPath,
configKey: string,
contentHash: string,
): RequestGraphNode => ({
id: `config_key:${fromProjectPathRelative(fileName)}:${configKey}`,
type: CONFIG_KEY,
configKey,
contentHash,
});

const keyFromEnvContentKey = (contentKey: ContentKey): string =>
contentKey.slice('env:'.length);

Expand All @@ -247,6 +276,7 @@ export class RequestGraph extends ContentGraph<
unpredicatableNodeIds: Set<NodeId> = new Set();
invalidateOnBuildNodeIds: Set<NodeId> = new Set();
cachedRequestChunks: Set<number> = new Set();
configKeyNodes: Map<ProjectPath, Set<NodeId>> = new Map();

// $FlowFixMe[prop-missing]
static deserialize(opts: RequestGraphOpts): RequestGraph {
Expand All @@ -260,6 +290,7 @@ export class RequestGraph extends ContentGraph<
deserialized.unpredicatableNodeIds = opts.unpredicatableNodeIds;
deserialized.invalidateOnBuildNodeIds = opts.invalidateOnBuildNodeIds;
deserialized.cachedRequestChunks = opts.cachedRequestChunks;
deserialized.configKeyNodes = opts.configKeyNodes;
return deserialized;
}

Expand All @@ -275,6 +306,7 @@ export class RequestGraph extends ContentGraph<
unpredicatableNodeIds: this.unpredicatableNodeIds,
invalidateOnBuildNodeIds: this.invalidateOnBuildNodeIds,
cachedRequestChunks: this.cachedRequestChunks,
configKeyNodes: this.configKeyNodes,
};
}

Expand Down Expand Up @@ -310,6 +342,10 @@ export class RequestGraph extends ContentGraph<
this.envNodeIds.delete(nodeId);
} else if (node.type === OPTION) {
this.optionNodeIds.delete(nodeId);
} else if (node.type === CONFIG_KEY) {
for (let configKeyNodes of this.configKeyNodes.values()) {
configKeyNodes.delete(nodeId);
}
}
return super.removeNode(nodeId);
}
Expand Down Expand Up @@ -407,6 +443,40 @@ export class RequestGraph extends ContentGraph<
}
}

invalidateOnConfigKeyChange(
requestNodeId: NodeId,
filePath: ProjectPath,
configKey: string,
contentHash: string,
) {
let configKeyNodeId = this.addNode(
nodeFromConfigKey(filePath, configKey, contentHash),
);
let nodes = this.configKeyNodes.get(filePath);

if (!nodes) {
nodes = new Set();
this.configKeyNodes.set(filePath, nodes);
}

nodes.add(configKeyNodeId);

if (
!this.hasEdge(
requestNodeId,
configKeyNodeId,
requestGraphEdgeTypes.invalidated_by_update,
)
) {
this.addEdge(
requestNodeId,
configKeyNodeId,
// Store as an update edge, but file deletes are handled too
requestGraphEdgeTypes.invalidated_by_update,
);
}
}

invalidateOnFileUpdate(requestNodeId: NodeId, filePath: ProjectPath) {
let fileNodeId = this.addNode(nodeFromFilePath(filePath));

Expand Down Expand Up @@ -748,11 +818,11 @@ export class RequestGraph extends ContentGraph<
}
}

respondToFSEvents(
async respondToFSEvents(
events: Array<Event>,
projectRoot: string,
options: ParcelOptions,
threshold: number,
): boolean {
): Async<boolean> {
let didInvalidate = false;
let count = 0;
let predictedTime = 0;
Expand All @@ -779,7 +849,7 @@ export class RequestGraph extends ContentGraph<
}
}

let _filePath = toProjectPath(projectRoot, _path);
let _filePath = toProjectPath(options.projectRoot, _path);
let filePath = fromProjectPathRelative(_filePath);
let hasFileRequest = this.hasContentKey(filePath);

Expand Down Expand Up @@ -885,6 +955,40 @@ export class RequestGraph extends ContentGraph<
// to requests as invalidations for future requests.
this.removeNode(nodeId);
}

let configKeyNodes = this.configKeyNodes.get(_filePath);
if (configKeyNodes && (type === 'delete' || type === 'update')) {
for (let nodeId of configKeyNodes) {
let isInvalid = type === 'delete';

if (type === 'update') {
let node = this.getNode(nodeId);
invariant(node && node.type === CONFIG_KEY);

let contentHash = await getConfigKeyContentHash(
_filePath,
node.configKey,
options,
);

isInvalid = node.contentHash !== contentHash;
}

if (isInvalid) {
for (let connectedNode of this.getNodeIdsConnectedTo(
nodeId,
requestGraphEdgeTypes.invalidated_by_update,
)) {
this.invalidateNode(
connectedNode,
type === 'delete' ? FILE_DELETE : FILE_UPDATE,
);
}
didInvalidate = true;
this.removeNode(nodeId);
}
}
}
}

let duration = Date.now() - startTime;
Expand Down Expand Up @@ -1031,12 +1135,8 @@ export default class RequestTracker {
}
}

respondToFSEvents(
events: Array<Event>,
projectRoot: string,
threshold: number,
): boolean {
return this.graph.respondToFSEvents(events, projectRoot, threshold);
respondToFSEvents(events: Array<Event>, threshold: number): Async<boolean> {
return this.graph.respondToFSEvents(events, this.options, threshold);
}

hasInvalidRequests(): boolean {
Expand Down Expand Up @@ -1135,6 +1235,13 @@ export default class RequestTracker {
let api: RunAPI<TResult> = {
invalidateOnFileCreate: input =>
this.graph.invalidateOnFileCreate(requestId, input),
invalidateOnConfigKeyChange: (filePath, configKey, contentHash) =>
this.graph.invalidateOnConfigKeyChange(
requestId,
filePath,
configKey,
contentHash,
),
invalidateOnFileDelete: filePath =>
this.graph.invalidateOnFileDelete(requestId, filePath),
invalidateOnFileUpdate: filePath =>
Expand Down Expand Up @@ -1434,9 +1541,9 @@ async function loadRequestGraph(options): Async<RequestGraph> {
requestGraph.invalidateOptionNodes(options);

try {
requestGraph.respondToFSEvents(
await requestGraph.respondToFSEvents(
options.unstableFileInvalidations || events,
options.projectRoot,
options,
10000,
);
return requestGraph;
Expand Down
17 changes: 16 additions & 1 deletion packages/core/core/src/public/Config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,13 @@ export default class PublicConfig implements IConfig {
);
}

invalidateOnConfigKeyChange(filePath: FilePath, configKey: string) {
this.#config.invalidateOnConfigKeyChange.push({
filePath: toProjectPath(this.#options.projectRoot, filePath),
configKey,
});
}

addDevDependency(devDep: DevDepOptions) {
this.#config.devDeps.push({
...devDep,
Expand Down Expand Up @@ -133,8 +140,16 @@ export default class PublicConfig implements IConfig {
): Promise<?ConfigResultWithFilePath<T>> {
let packageKey = options?.packageKey;
if (packageKey != null) {
let pkg = await this.getConfigFrom(searchPath, ['package.json']);
let pkg = await this.getConfigFrom(searchPath, ['package.json'], {
exclude: this.#options.featureFlags.configKeyInvalidation,
});

if (pkg && pkg.contents[packageKey]) {
if (this.#options.featureFlags.configKeyInvalidation) {
// Invalidate only when the package key changes
this.invalidateOnConfigKeyChange(pkg.filePath, packageKey);
}

return {
contents: pkg.contents[packageKey],
filePath: pkg.filePath,
Expand Down
Loading

0 comments on commit 84fc9a9

Please sign in to comment.