Skip to content

Commit 7230d7f

Browse files
brotheroftuxbrotheroftux
committed
feat: liquid markdown front matter separately from the rest of the content
1 parent 324d200 commit 7230d7f

File tree

13 files changed

+480
-42
lines changed

13 files changed

+480
-42
lines changed

‎src/transform/frontmatter/common.ts

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
export type FrontMatter = {
2+
[key: string]: unknown;
3+
metadata?: Record<string, unknown>[];
4+
};
5+
6+
export const frontMatterFence = '---';
7+
8+
/**
9+
* Temporary workaround to enable parsing YAML metadata from potentially
10+
* Liquid-aware source files
11+
* @param content Input string which could contain Liquid-style substitution syntax (which clashes with YAML
12+
* object syntax)
13+
* @returns String with `{}` escaped, ready to be parsed with `js-yaml`
14+
*/
15+
export const escapeLiquidSubstitutionSyntax = (content: string): string =>
16+
content.replace(/{{/g, '(({{').replace(/}}/g, '}}))');
17+
18+
/**
19+
* Inverse of a workaround defined above.
20+
* @see `escapeLiquidSubstitutionSyntax`
21+
* @param escapedContent Input string with `{}` escaped with backslashes
22+
* @returns Unescaped string
23+
*/
24+
export const unescapeLiquidSubstitutionSyntax = (escapedContent: string): string =>
25+
escapedContent.replace(/\(\({{/g, '{{').replace(/}}\)\)/g, '}}');
26+
27+
export const countLineAmount = (str: string) => str.split(/\r?\n/).length;

‎src/transform/frontmatter/emplace.ts

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import {dump} from 'js-yaml';
2+
3+
import {FrontMatter, frontMatterFence} from './common';
4+
5+
export const serializeFrontMatter = (frontMatter: FrontMatter) => {
6+
const dumped = dump(frontMatter, {lineWidth: -1}).trim();
7+
8+
// This empty object check is a bit naive
9+
// The other option would be to check if all own fields are `undefined`,
10+
// since we exploit passing in `undefined` to remove a field quite a bit
11+
if (dumped === '{}') {
12+
return '';
13+
}
14+
15+
return `${frontMatterFence}\n${dumped}\n${frontMatterFence}\n`;
16+
};
17+
18+
export const emplaceSerializedFrontMatter = (
19+
frontMatterStrippedContent: string,
20+
frontMatter: string,
21+
) => `${frontMatter}${frontMatterStrippedContent}`;
22+
23+
export const emplaceFrontMatter = (frontMatterStrippedContent: string, frontMatter: FrontMatter) =>
24+
emplaceSerializedFrontMatter(frontMatterStrippedContent, serializeFrontMatter(frontMatter));

‎src/transform/frontmatter/extract.ts

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import {YAMLException, load} from 'js-yaml';
2+
3+
import {log} from '../log';
4+
5+
import {
6+
FrontMatter,
7+
countLineAmount,
8+
escapeLiquidSubstitutionSyntax,
9+
frontMatterFence,
10+
unescapeLiquidSubstitutionSyntax,
11+
} from './common';
12+
import {transformFrontMatterValues} from './transformValues';
13+
14+
type ParseExistingMetadataReturn = {
15+
frontMatter: FrontMatter;
16+
frontMatterStrippedContent: string;
17+
frontMatterLineCount: number;
18+
};
19+
20+
const matchMetadata = (fileContent: string) => {
21+
if (!fileContent.startsWith(frontMatterFence)) {
22+
return null;
23+
}
24+
25+
// Search by format:
26+
// ---
27+
// metaName1: metaValue1
28+
// metaName2: meta value2
29+
// incorrectMetadata
30+
// ---
31+
const regexpMetadata = '(?<=-{3}\\r?\\n)((.*\\r?\\n)*?)(?=-{3}\\r?\\n)';
32+
// Search by format:
33+
// ---
34+
// main content 123
35+
const regexpFileContent = '-{3}\\r?\\n((.*[\r?\n]*)*)';
36+
37+
const regexpParseFileContent = new RegExp(`${regexpMetadata}${regexpFileContent}`, 'gm');
38+
39+
return regexpParseFileContent.exec(fileContent);
40+
};
41+
42+
const duplicateKeysCompatibleLoad = (yaml: string, filePath: string | undefined) => {
43+
try {
44+
return load(yaml);
45+
} catch (e) {
46+
if (e instanceof YAMLException) {
47+
const duplicateKeysDeprecationWarning = `
48+
In ${filePath ?? '(unknown)'}: Encountered a YAML parsing exception when processing file metadata: ${e.reason}.
49+
It's highly possible the input file contains duplicate mapping keys.
50+
Will retry processing with necessary compatibility flags.
51+
Please note that this behaviour is DEPRECATED and WILL be removed in a future version
52+
without further notice, so the build WILL fail when supplied with YAML-incompatible meta.
53+
`
54+
.replace(/^\s+/gm, '')
55+
.replace(/\n/g, ' ')
56+
.trim();
57+
58+
log.warn(duplicateKeysDeprecationWarning);
59+
60+
return load(yaml, {json: true});
61+
}
62+
63+
throw e;
64+
}
65+
};
66+
67+
export const separateAndExtractFrontMatter = (
68+
fileContent: string,
69+
filePath?: string,
70+
): ParseExistingMetadataReturn => {
71+
const matches = matchMetadata(fileContent);
72+
73+
if (matches && matches.length > 0) {
74+
const [, metadata, , metadataStrippedContent] = matches;
75+
76+
return {
77+
frontMatter: transformFrontMatterValues(
78+
duplicateKeysCompatibleLoad(
79+
escapeLiquidSubstitutionSyntax(metadata),
80+
filePath,
81+
) as FrontMatter,
82+
(v) => (typeof v === 'string' ? unescapeLiquidSubstitutionSyntax(v) : v),
83+
),
84+
frontMatterStrippedContent: metadataStrippedContent,
85+
frontMatterLineCount: countLineAmount(metadata),
86+
};
87+
}
88+
89+
return {
90+
frontMatter: {},
91+
frontMatterStrippedContent: fileContent,
92+
frontMatterLineCount: 0,
93+
};
94+
};

‎src/transform/frontmatter/index.ts

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
export * from './extract';
2+
export * from './emplace';
3+
export * from './transformValues';
4+
export {countLineAmount} from './common';

‎src/transform/frontmatter/transformValues.ts

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import {FrontMatter} from './common';
2+
3+
export const transformFrontMatterValues = (
4+
frontMatter: FrontMatter,
5+
valueMapper: (v: unknown) => unknown,
6+
): FrontMatter => {
7+
const transformInner = (something: unknown): unknown => {
8+
if (typeof something === 'object' && something !== null) {
9+
return Object.fromEntries(
10+
Object.entries(something).map(([k, v]) => [k, transformInner(v)]),
11+
);
12+
}
13+
14+
if (Array.isArray(something)) {
15+
return something.map((el) => transformInner(el));
16+
}
17+
18+
return valueMapper(something);
19+
};
20+
21+
return transformInner(frontMatter) as FrontMatter;
22+
};

‎src/transform/index.ts

+4-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import type {EnvType, OptionsType, OutputType} from './typings';
33
import {bold} from 'chalk';
44

55
import {log} from './log';
6-
import liquid from './liquid';
6+
import liquidSnippet from './liquid';
77
import initMarkdownit from './md';
88

99
function applyLiquid(input: string, options: OptionsType) {
@@ -15,7 +15,9 @@ function applyLiquid(input: string, options: OptionsType) {
1515
isLiquided = false,
1616
} = options;
1717

18-
return disableLiquid || isLiquided ? input : liquid(input, vars, path, {conditionsInCode});
18+
return disableLiquid || isLiquided
19+
? input
20+
: liquidSnippet(input, vars, path, {conditionsInCode});
1921
}
2022

2123
function handleError(error: unknown, path?: string): never {

‎src/transform/liquid/index.ts

+73-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11
import type {Dictionary} from 'lodash';
22

3+
import {
4+
countLineAmount,
5+
emplaceSerializedFrontMatter,
6+
separateAndExtractFrontMatter,
7+
serializeFrontMatter,
8+
transformFrontMatterValues,
9+
} from '../frontmatter';
10+
311
import applySubstitutions from './substitutions';
412
import {prepareSourceMap} from './sourceMap';
513
import applyCycles from './cycles';
@@ -66,7 +74,7 @@ function repairCode(str: string, codes: string[]) {
6674
return replace(fence, fence, (code) => codes[Number(code)], str);
6775
}
6876

69-
function liquid<
77+
function liquidSnippet<
7078
B extends boolean = false,
7179
C = B extends false ? string : {output: string; sourceMap: Dictionary<string>},
7280
>(
@@ -141,6 +149,67 @@ function liquid<
141149
return output as unknown as C;
142150
}
143151

144-
// 'export default' instead of 'export = ' because of circular dependency with './cycles.ts'.
145-
// somehow it breaks import in './cycles.ts' and imports nothing
146-
export default liquid;
152+
type TransformSourceMapOptions = {
153+
emplacedResultOffset: number;
154+
emplacedSourceOffset: number;
155+
};
156+
157+
function transformSourceMap(
158+
sourceMap: Dictionary<string>,
159+
{emplacedResultOffset, emplacedSourceOffset}: TransformSourceMapOptions,
160+
) {
161+
return Object.fromEntries(
162+
Object.entries(sourceMap).map(([lineInResult, lineInSource]) => [
163+
(Number(lineInResult) + emplacedResultOffset).toString(),
164+
(Number(lineInSource) + emplacedSourceOffset).toString(),
165+
]),
166+
);
167+
}
168+
169+
function liquidDocument<
170+
B extends boolean = false,
171+
C = B extends false ? string : {output: string; sourceMap: Dictionary<string>},
172+
>(
173+
originInput: string,
174+
vars: Record<string, unknown>,
175+
path?: string,
176+
settings?: ArgvSettings & {withSourceMap?: B},
177+
): C {
178+
const {frontMatter, frontMatterStrippedContent, frontMatterLineCount} =
179+
separateAndExtractFrontMatter(originInput, path);
180+
181+
const transformedFrontMatter = transformFrontMatterValues(frontMatter, (v) =>
182+
typeof v === 'string'
183+
? liquidSnippet(v, vars, path, {...settings, withSourceMap: false})
184+
: v,
185+
);
186+
const transformedAndSerialized = serializeFrontMatter(transformedFrontMatter);
187+
188+
// -1 comes from the fact that the last line in serialized FM is the same as the first line in stripped content
189+
const resultFrontMatterOffset = Math.max(0, countLineAmount(transformedAndSerialized) - 1);
190+
const sourceFrontMatterOffset = Math.max(0, frontMatterLineCount - 1);
191+
192+
const liquidProcessedContent = liquidSnippet(frontMatterStrippedContent, vars, path, settings);
193+
194+
// typeof check for better inference; the catch is that return of liquidSnippet can be an
195+
// object even with source maps off, see `substitutions.test.ts`
196+
return (settings?.withSourceMap && typeof liquidProcessedContent === 'object'
197+
? {
198+
output: emplaceSerializedFrontMatter(
199+
liquidProcessedContent.output,
200+
transformedAndSerialized,
201+
),
202+
sourceMap: transformSourceMap(liquidProcessedContent.sourceMap, {
203+
emplacedResultOffset: resultFrontMatterOffset,
204+
emplacedSourceOffset: sourceFrontMatterOffset,
205+
}),
206+
}
207+
: emplaceSerializedFrontMatter(
208+
liquidProcessedContent as string,
209+
transformedAndSerialized,
210+
)) as unknown as C;
211+
}
212+
213+
// both default and named exports for convenience
214+
export {liquidDocument, liquidSnippet};
215+
export default liquidDocument;

‎src/transform/utilsFS.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {readFileSync, statSync} from 'fs';
44
import escapeRegExp from 'lodash/escapeRegExp';
55
import {join, parse, relative, resolve, sep} from 'path';
66

7-
import liquid from './liquid';
7+
import liquidSnippet from './liquid';
88
import {StateCore} from './typings';
99
import {defaultTransformLink} from './utils';
1010

@@ -68,7 +68,7 @@ export function getFileTokens(path: string, state: StateCore, options: GetFileTo
6868
let sourceMap;
6969

7070
if (!disableLiquid) {
71-
const liquidResult = liquid(content, builtVars, path, {
71+
const liquidResult = liquidSnippet(content, builtVars, path, {
7272
withSourceMap: true,
7373
conditionsInCode,
7474
});

‎test/liquid/__snapshots__/frontmatter.test.ts.snap

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Jest Snapshot v1, https://goo.gl/fbAQLP
2+
3+
exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (YAML multiline syntax) 1`] = `
4+
"---
5+
multiline: \\">- This should break, right?\\\\n\\\\tRight?\\"
6+
---
7+
Content"
8+
`;
9+
10+
exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (curly braces) 1`] = `
11+
"---
12+
braces: '{}'
13+
---
14+
Content"
15+
`;
16+
17+
exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (double quotes) 1`] = `
18+
"---
19+
quotes: '\\"When you arise in the morning, think of what a precious privilege it is to be alive - to breathe, to think, to enjoy, to love.\\" — Marcus Aurelius (allegedly)'
20+
---
21+
Content"
22+
`;
23+
24+
exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (single quotes) 1`] = `
25+
"---
26+
quotes: This isn't your typical substitution. It has single quotes.
27+
---
28+
Content"
29+
`;
30+
31+
exports[`Liquid substitutions in front matter (formerly metadata) should not fail even when variables contain reserved characters (square brackets) 1`] = `
32+
"---
33+
brackets: '[]'
34+
---
35+
Content"
36+
`;

0 commit comments

Comments
 (0)