From f528d6139ee095f0e20320e851d5b4f4021f260c Mon Sep 17 00:00:00 2001 From: Jacky Zhao Date: Fri, 14 Mar 2025 09:18:21 -0700 Subject: [PATCH] checkpoint --- docs/authoring content.md | 2 + package-lock.json | 20 +- package.json | 3 +- quartz.config.ts | 2 +- quartz/build.ts | 423 ++++++------------ quartz/cli/args.js | 4 +- quartz/cli/handlers.js | 28 +- quartz/components/renderPage.tsx | 24 +- quartz/depgraph.test.ts | 118 ----- quartz/depgraph.ts | 228 ---------- quartz/plugins/emitters/404.tsx | 4 - quartz/plugins/emitters/aliases.ts | 71 +-- quartz/plugins/emitters/assets.ts | 60 +-- quartz/plugins/emitters/cname.ts | 6 +- quartz/plugins/emitters/componentResources.ts | 32 +- quartz/plugins/emitters/contentIndex.tsx | 25 +- quartz/plugins/emitters/contentPage.tsx | 54 +-- quartz/plugins/emitters/folderPage.tsx | 17 - quartz/plugins/emitters/helpers.ts | 2 + quartz/plugins/emitters/static.ts | 17 +- quartz/plugins/emitters/tagPage.tsx | 22 - quartz/plugins/transformers/frontmatter.ts | 42 +- quartz/plugins/transformers/lastmod.ts | 10 +- quartz/plugins/types.ts | 19 +- quartz/processors/emit.ts | 6 +- quartz/processors/parse.ts | 44 +- quartz/util/ctx.ts | 8 +- quartz/util/log.ts | 15 +- quartz/util/path.ts | 2 +- quartz/worker.ts | 25 +- 30 files changed, 389 insertions(+), 944 deletions(-) delete mode 100644 quartz/depgraph.test.ts delete mode 100644 quartz/depgraph.ts diff --git a/docs/authoring content.md b/docs/authoring content.md index 623357fc3..ea2ef9415 100644 --- a/docs/authoring content.md +++ b/docs/authoring content.md @@ -1,5 +1,7 @@ --- title: Authoring Content +aliases: +- test/author 1 --- All of the content in your Quartz should go in the `/content` folder. The content for the home page of your Quartz lives in `content/index.md`. If you've [[index#🪴 Get Started|setup Quartz]] already, this folder should already be initialized. Any Markdown in this folder will get processed by Quartz. diff --git a/package-lock.json b/package-lock.json index 588820535..73f3f1148 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@jackyzha0/quartz", - "version": "4.4.1", + "version": "4.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@jackyzha0/quartz", - "version": "4.4.1", + "version": "4.5.0", "license": "MIT", "dependencies": { "@clack/prompts": "^0.10.0", @@ -14,6 +14,7 @@ "@myriaddreamin/rehype-typst": "^0.5.4", "@napi-rs/simple-git": "0.1.19", "@tweenjs/tween.js": "^25.0.0", + "ansi-truncate": "^1.2.0", "async-mutex": "^0.5.0", "chalk": "^5.4.1", "chokidar": "^4.0.3", @@ -2032,6 +2033,15 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/ansi-truncate": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/ansi-truncate/-/ansi-truncate-1.2.0.tgz", + "integrity": "sha512-/SLVrxNIP8o8iRHjdK3K9s2hDqdvb86NEjZOAB6ecWFsOo+9obaby97prnvAPn6j7ExXCpbvtlJFYPkkspg4BQ==", + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^1.2.0" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -3058,6 +3068,12 @@ "node": ">=8.6.0" } }, + "node_modules/fast-string-truncated-width": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-1.2.1.tgz", + "integrity": "sha512-Q9acT/+Uu3GwGj+5w/zsGuQjh9O1TyywhIwAxHudtWrgF09nHOPrvTLhQevPbttcxjr/SNN7mJmfOw/B1bXgow==", + "license": "MIT" + }, "node_modules/fastq": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.0.tgz", diff --git a/package.json b/package.json index 1fb31eec1..f7857b9a0 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@jackyzha0/quartz", "description": "🌱 publish your digital garden and notes as a website", "private": true, - "version": "4.4.1", + "version": "4.5.0", "type": "module", "author": "jackyzha0 ", "license": "MIT", @@ -40,6 +40,7 @@ "@myriaddreamin/rehype-typst": "^0.5.4", "@napi-rs/simple-git": "0.1.19", "@tweenjs/tween.js": "^25.0.0", + "ansi-truncate": "^1.2.0", "async-mutex": "^0.5.0", "chalk": "^5.4.1", "chokidar": "^4.0.3", diff --git a/quartz.config.ts b/quartz.config.ts index f54060908..03ef0d7f8 100644 --- a/quartz.config.ts +++ b/quartz.config.ts @@ -57,7 +57,7 @@ const config: QuartzConfig = { transformers: [ Plugin.FrontMatter(), Plugin.CreatedModifiedDate({ - priority: ["frontmatter", "filesystem"], + priority: ["git", "frontmatter", "filesystem"], }), Plugin.SyntaxHighlighting({ theme: { diff --git a/quartz/build.ts b/quartz/build.ts index 91a5a5ab4..326654571 100644 --- a/quartz/build.ts +++ b/quartz/build.ts @@ -17,34 +17,38 @@ import { glob, toPosixPath } from "./util/glob" import { trace } from "./util/trace" import { options } from "./util/sourcemap" import { Mutex } from "async-mutex" -import DepGraph from "./depgraph" import { getStaticResourcesFromPlugins } from "./plugins" import { randomIdNonSecure } from "./util/random" +import { ChangeEvent } from "./plugins/types" -type Dependencies = Record | null> +type ContentMap = Map< + FilePath, + | { + type: "markdown" + content: ProcessedContent + } + | { + type: "other" + } +> type BuildData = { ctx: BuildCtx ignored: GlobbyFilterFunction mut: Mutex - initialSlugs: FullSlug[] - // TODO merge contentMap and trackedAssets - contentMap: Map - trackedAssets: Set - toRebuild: Set - toRemove: Set + contentMap: ContentMap + changesSinceLastBuild: Record lastBuildMs: number - dependencies: Dependencies } -type FileEvent = "add" | "change" | "delete" - async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) { const ctx: BuildCtx = { buildId: randomIdNonSecure(), argv, cfg, allSlugs: [], + allFiles: [], + incremental: false, } const perf = new PerfTimer() @@ -67,64 +71,58 @@ async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) { perf.addEvent("glob") const allFiles = await glob("**/*.*", argv.directory, cfg.configuration.ignorePatterns) - const fps = allFiles.filter((fp) => fp.endsWith(".md")).sort() + const markdownPaths = allFiles.filter((fp) => fp.endsWith(".md")).sort() console.log( - `Found ${fps.length} input files from \`${argv.directory}\` in ${perf.timeSince("glob")}`, + `Found ${markdownPaths.length} input files from \`${argv.directory}\` in ${perf.timeSince("glob")}`, ) - const filePaths = fps.map((fp) => joinSegments(argv.directory, fp) as FilePath) + const filePaths = markdownPaths.map((fp) => joinSegments(argv.directory, fp) as FilePath) + ctx.allFiles = allFiles ctx.allSlugs = allFiles.map((fp) => slugifyFilePath(fp as FilePath)) const parsedFiles = await parseMarkdown(ctx, filePaths) const filteredContent = filterContent(ctx, parsedFiles) - const dependencies: Record | null> = {} - - // Only build dependency graphs if we're doing a fast rebuild - if (argv.fastRebuild) { - const staticResources = getStaticResourcesFromPlugins(ctx) - for (const emitter of cfg.plugins.emitters) { - dependencies[emitter.name] = - (await emitter.getDependencyGraph?.(ctx, filteredContent, staticResources)) ?? null - } - } - await emitContent(ctx, filteredContent) - console.log(chalk.green(`Done processing ${fps.length} files in ${perf.timeSince()}`)) + console.log(chalk.green(`Done processing ${markdownPaths.length} files in ${perf.timeSince()}`)) release() - if (argv.serve) { - return startServing(ctx, mut, parsedFiles, clientRefresh, dependencies) + if (argv.watch) { + ctx.incremental = true + return startWatching(ctx, mut, parsedFiles, clientRefresh) } } // setup watcher for rebuilds -async function startServing( +async function startWatching( ctx: BuildCtx, mut: Mutex, initialContent: ProcessedContent[], clientRefresh: () => void, - dependencies: Dependencies, // emitter name: dep graph ) { - const { argv } = ctx + const { argv, allFiles } = ctx + + const contentMap: ContentMap = new Map() + for (const filePath of allFiles) { + contentMap.set(filePath, { + type: "other", + }) + } - // cache file parse results - const contentMap = new Map() for (const content of initialContent) { const [_tree, vfile] = content - contentMap.set(vfile.data.filePath!, content) + contentMap.set(vfile.data.relativePath!, { + type: "markdown", + content, + }) } const buildData: BuildData = { ctx, mut, - dependencies, contentMap, ignored: await isGitIgnored(), - initialSlugs: ctx.allSlugs, - toRebuild: new Set(), - toRemove: new Set(), - trackedAssets: new Set(), + changesSinceLastBuild: {}, lastBuildMs: 0, } @@ -134,31 +132,33 @@ async function startServing( ignoreInitial: true, }) - const buildFromEntry = argv.fastRebuild ? partialRebuildFromEntrypoint : rebuildFromEntrypoint + const changes: ChangeEvent[] = [] watcher - .on("add", (fp) => buildFromEntry(fp as string, "add", clientRefresh, buildData)) - .on("change", (fp) => buildFromEntry(fp as string, "change", clientRefresh, buildData)) - .on("unlink", (fp) => buildFromEntry(fp as string, "delete", clientRefresh, buildData)) + .on("add", (fp) => { + if (buildData.ignored(fp)) return + changes.push({ path: fp as FilePath, type: "add" }) + rebuild(changes, clientRefresh, buildData) + }) + .on("change", (fp) => { + if (buildData.ignored(fp)) return + changes.push({ path: fp as FilePath, type: "change" }) + rebuild(changes, clientRefresh, buildData) + }) + .on("unlink", (fp) => { + if (buildData.ignored(fp)) return + changes.push({ path: fp as FilePath, type: "delete" }) + rebuild(changes, clientRefresh, buildData) + }) return async () => { await watcher.close() } } -async function partialRebuildFromEntrypoint( - filepath: string, - action: FileEvent, - clientRefresh: () => void, - buildData: BuildData, // note: this function mutates buildData -) { - const { ctx, ignored, dependencies, contentMap, mut, toRemove } = buildData +async function rebuild(changes: ChangeEvent[], clientRefresh: () => void, buildData: BuildData) { + const { ctx, contentMap, mut, changesSinceLastBuild } = buildData const { argv, cfg } = ctx - // don't do anything for gitignored files - if (ignored(filepath)) { - return - } - const buildId = randomIdNonSecure() ctx.buildId = buildId buildData.lastBuildMs = new Date().getTime() @@ -171,261 +171,104 @@ async function partialRebuildFromEntrypoint( } const perf = new PerfTimer() + perf.addEvent("rebuild") console.log(chalk.yellow("Detected change, rebuilding...")) + console.log(changes) - // UPDATE DEP GRAPH - const fp = joinSegments(argv.directory, toPosixPath(filepath)) as FilePath + // update changesSinceLastBuild + for (const change of changes) { + changesSinceLastBuild[change.path] = change.type + } const staticResources = getStaticResourcesFromPlugins(ctx) - let processedFiles: ProcessedContent[] = [] - - switch (action) { - case "add": - // add to cache when new file is added - processedFiles = await parseMarkdown(ctx, [fp]) - processedFiles.forEach(([tree, vfile]) => contentMap.set(vfile.data.filePath!, [tree, vfile])) - - // update the dep graph by asking all emitters whether they depend on this file - for (const emitter of cfg.plugins.emitters) { - const emitterGraph = - (await emitter.getDependencyGraph?.(ctx, processedFiles, staticResources)) ?? null - - if (emitterGraph) { - const existingGraph = dependencies[emitter.name] - if (existingGraph !== null) { - existingGraph.mergeGraph(emitterGraph) - } else { - // might be the first time we're adding a mardown file - dependencies[emitter.name] = emitterGraph - } - } - } - break - case "change": - // invalidate cache when file is changed - processedFiles = await parseMarkdown(ctx, [fp]) - processedFiles.forEach(([tree, vfile]) => contentMap.set(vfile.data.filePath!, [tree, vfile])) - - // only content files can have added/removed dependencies because of transclusions - if (path.extname(fp) === ".md") { - for (const emitter of cfg.plugins.emitters) { - // get new dependencies from all emitters for this file - const emitterGraph = - (await emitter.getDependencyGraph?.(ctx, processedFiles, staticResources)) ?? null - - // only update the graph if the emitter plugin uses the changed file - // eg. Assets plugin ignores md files, so we skip updating the graph - if (emitterGraph?.hasNode(fp)) { - // merge the new dependencies into the dep graph - dependencies[emitter.name]?.updateIncomingEdgesForNode(emitterGraph, fp) - } - } - } - break - case "delete": - toRemove.add(fp) - break - } - - if (argv.verbose) { - console.log(`Updated dependency graphs in ${perf.timeSince()}`) - } - - // EMIT - perf.addEvent("rebuild") - let emittedFiles = 0 - - for (const emitter of cfg.plugins.emitters) { - const depGraph = dependencies[emitter.name] - - // emitter hasn't defined a dependency graph. call it with all processed files - if (depGraph === null) { - if (argv.verbose) { - console.log( - `Emitter ${emitter.name} doesn't define a dependency graph. Calling it with all files...`, + const processedFiles = await Promise.all( + Object.entries(changesSinceLastBuild) + .filter(([fp, type]) => type !== "delete" && path.extname(fp) === ".md") + .map(async ([fp, _type]) => { + const fullPath = joinSegments(argv.directory, toPosixPath(fp)) as FilePath + const parsed = await parseMarkdown(ctx, [fullPath]) + parsed.forEach((content) => + contentMap.set(content[1].data.relativePath!, { + type: "markdown", + content, + }), ) + return parsed + }), + ).then((results) => results.flat()) + + // update state using changesSinceLastBuild + // we do this weird play of add => compute change events => remove + // so that partialEmitters can do appropriate cleanup based on the content of deleted files + for (const [file, change] of Object.entries(changesSinceLastBuild)) { + if (change === "delete") { + // universal delete case + contentMap.delete(file as FilePath) + } + + // manually track non-markdown files as processed files only + // contains markdown files + if (change === "add" && path.extname(file) !== ".md") { + contentMap.set(file as FilePath, { + type: "other", + }) + } + } + + const changeEvents: ChangeEvent[] = Object.entries(changesSinceLastBuild).map(([fp, type]) => { + const path = fp as FilePath + const processedContent = contentMap.get(path) + if (processedContent?.type === "markdown") { + const [_tree, file] = processedContent.content + return { + type, + path, + file, } + } + + return { + type, + path, + } + }) - const files = [...contentMap.values()].filter( - ([_node, vfile]) => !toRemove.has(vfile.data.filePath!), - ) - - const emitted = await emitter.emit(ctx, files, staticResources) - if (Symbol.asyncIterator in emitted) { - // Async generator case - for await (const file of emitted) { - emittedFiles++ - if (ctx.argv.verbose) { - console.log(`[emit:${emitter.name}] ${file}`) - } - } - } else { - // Array case - emittedFiles += emitted.length - if (ctx.argv.verbose) { - for (const file of emitted) { - console.log(`[emit:${emitter.name}] ${file}`) - } - } - } + // update allFiles and then allSlugs with the consistent view of content map + ctx.allFiles = Array.from(contentMap.keys()) + ctx.allSlugs = ctx.allFiles.map((fp) => slugifyFilePath(fp as FilePath)) + let emittedFiles = 0 + for (const emitter of cfg.plugins.emitters) { + // Try to use partialEmit if available, otherwise assume the output is static + const emitFn = emitter.partialEmit + if (!emitFn) { continue } - // only call the emitter if it uses this file - if (depGraph.hasNode(fp)) { - // re-emit using all files that are needed for the downstream of this file - // eg. for ContentIndex, the dep graph could be: - // a.md --> contentIndex.json - // b.md ------^ - // - // if a.md changes, we need to re-emit contentIndex.json, - // and supply [a.md, b.md] to the emitter - const upstreams = [...depGraph.getLeafNodeAncestors(fp)] as FilePath[] - - const upstreamContent = upstreams - // filter out non-markdown files - .filter((file) => contentMap.has(file)) - // if file was deleted, don't give it to the emitter - .filter((file) => !toRemove.has(file)) - .map((file) => contentMap.get(file)!) - - const emitted = await emitter.emit(ctx, upstreamContent, staticResources) - if (Symbol.asyncIterator in emitted) { - // Async generator case - for await (const file of emitted) { - emittedFiles++ - if (ctx.argv.verbose) { - console.log(`[emit:${emitter.name}] ${file}`) - } - } - } else { - // Array case - emittedFiles += emitted.length + const emitted = await emitFn(ctx, processedFiles, staticResources, changeEvents) + if (Symbol.asyncIterator in emitted) { + // Async generator case + for await (const file of emitted) { + emittedFiles++ if (ctx.argv.verbose) { - for (const file of emitted) { - console.log(`[emit:${emitter.name}] ${file}`) - } + console.log(`[emit:${emitter.name}] ${file}`) + } + } + } else { + // Array case + emittedFiles += emitted.length + if (ctx.argv.verbose) { + for (const file of emitted) { + console.log(`[emit:${emitter.name}] ${file}`) } } } } console.log(`Emitted ${emittedFiles} files to \`${argv.output}\` in ${perf.timeSince("rebuild")}`) - - // CLEANUP - const destinationsToDelete = new Set() - for (const file of toRemove) { - // remove from cache - contentMap.delete(file) - Object.values(dependencies).forEach((depGraph) => { - // remove the node from dependency graphs - depGraph?.removeNode(file) - // remove any orphan nodes. eg if a.md is deleted, a.html is orphaned and should be removed - const orphanNodes = depGraph?.removeOrphanNodes() - orphanNodes?.forEach((node) => { - // only delete files that are in the output directory - if (node.startsWith(argv.output)) { - destinationsToDelete.add(node) - } - }) - }) - } - await rimraf([...destinationsToDelete]) - console.log(chalk.green(`Done rebuilding in ${perf.timeSince()}`)) - - toRemove.clear() - release() + changes.length = 0 clientRefresh() -} - -async function rebuildFromEntrypoint( - fp: string, - action: FileEvent, - clientRefresh: () => void, - buildData: BuildData, // note: this function mutates buildData -) { - const { ctx, ignored, mut, initialSlugs, contentMap, toRebuild, toRemove, trackedAssets } = - buildData - - const { argv } = ctx - - // don't do anything for gitignored files - if (ignored(fp)) { - return - } - - // dont bother rebuilding for non-content files, just track and refresh - fp = toPosixPath(fp) - const filePath = joinSegments(argv.directory, fp) as FilePath - if (path.extname(fp) !== ".md") { - if (action === "add" || action === "change") { - trackedAssets.add(filePath) - } else if (action === "delete") { - trackedAssets.delete(filePath) - } - clientRefresh() - return - } - - if (action === "add" || action === "change") { - toRebuild.add(filePath) - } else if (action === "delete") { - toRemove.add(filePath) - } - - const buildId = randomIdNonSecure() - ctx.buildId = buildId - buildData.lastBuildMs = new Date().getTime() - const release = await mut.acquire() - - // there's another build after us, release and let them do it - if (ctx.buildId !== buildId) { - release() - return - } - - const perf = new PerfTimer() - console.log(chalk.yellow("Detected change, rebuilding...")) - - try { - const filesToRebuild = [...toRebuild].filter((fp) => !toRemove.has(fp)) - const parsedContent = await parseMarkdown(ctx, filesToRebuild) - for (const content of parsedContent) { - const [_tree, vfile] = content - contentMap.set(vfile.data.filePath!, content) - } - - for (const fp of toRemove) { - contentMap.delete(fp) - } - - const parsedFiles = [...contentMap.values()] - const filteredContent = filterContent(ctx, parsedFiles) - - // re-update slugs - const trackedSlugs = [...new Set([...contentMap.keys(), ...toRebuild, ...trackedAssets])] - .filter((fp) => !toRemove.has(fp)) - .map((fp) => slugifyFilePath(path.posix.relative(argv.directory, fp) as FilePath)) - - ctx.allSlugs = [...new Set([...initialSlugs, ...trackedSlugs])] - - // TODO: we can probably traverse the link graph to figure out what's safe to delete here - // instead of just deleting everything - await rimraf(path.join(argv.output, ".*"), { glob: true }) - await emitContent(ctx, filteredContent) - console.log(chalk.green(`Done rebuilding in ${perf.timeSince()}`)) - } catch (err) { - console.log(chalk.yellow(`Rebuild failed. Waiting on a change to fix the error...`)) - if (argv.verbose) { - console.log(chalk.red(err)) - } - } - - clientRefresh() - toRebuild.clear() - toRemove.clear() release() } diff --git a/quartz/cli/args.js b/quartz/cli/args.js index 123d0ac55..d2408e94b 100644 --- a/quartz/cli/args.js +++ b/quartz/cli/args.js @@ -71,10 +71,10 @@ export const BuildArgv = { default: false, describe: "run a local server to live-preview your Quartz", }, - fastRebuild: { + watch: { boolean: true, default: false, - describe: "[experimental] rebuild only the changed files", + describe: "watch for changes and rebuild automatically", }, baseDir: { string: true, diff --git a/quartz/cli/handlers.js b/quartz/cli/handlers.js index 6ef380596..4ac2a4ab9 100644 --- a/quartz/cli/handlers.js +++ b/quartz/cli/handlers.js @@ -225,6 +225,10 @@ See the [documentation](https://quartz.jzhao.xyz) for how to get started. * @param {*} argv arguments for `build` */ export async function handleBuild(argv) { + if (argv.serve) { + argv.watch = true + } + console.log(chalk.bgGreen.black(`\n Quartz v${version} \n`)) const ctx = await esbuild.context({ entryPoints: [fp], @@ -331,9 +335,10 @@ export async function handleBuild(argv) { clientRefresh() } + let clientRefresh = () => {} if (argv.serve) { const connections = [] - const clientRefresh = () => connections.forEach((conn) => conn.send("rebuild")) + clientRefresh = () => connections.forEach((conn) => conn.send("rebuild")) if (argv.baseDir !== "" && !argv.baseDir.startsWith("/")) { argv.baseDir = "/" + argv.baseDir @@ -433,6 +438,7 @@ export async function handleBuild(argv) { return serve() }) + server.listen(argv.port) const wss = new WebSocketServer({ port: argv.wsPort }) wss.on("connection", (ws) => connections.push(ws)) @@ -441,16 +447,26 @@ export async function handleBuild(argv) { `Started a Quartz server listening at http://localhost:${argv.port}${argv.baseDir}`, ), ) - console.log("hint: exit with ctrl+c") - const paths = await globby(["**/*.ts", "**/*.tsx", "**/*.scss", "package.json"]) + } else { + await build(clientRefresh) + ctx.dispose() + } + + if (argv.watch) { + const paths = await globby([ + "**/*.ts", + "quartz/cli/*.js", + "**/*.tsx", + "**/*.scss", + "package.json", + ]) chokidar .watch(paths, { ignoreInitial: true }) .on("add", () => build(clientRefresh)) .on("change", () => build(clientRefresh)) .on("unlink", () => build(clientRefresh)) - } else { - await build(() => {}) - ctx.dispose() + + console.log(chalk.grey("hint: exit with ctrl+c")) } } diff --git a/quartz/components/renderPage.tsx b/quartz/components/renderPage.tsx index a43b66cb7..c3f10df04 100644 --- a/quartz/components/renderPage.tsx +++ b/quartz/components/renderPage.tsx @@ -65,17 +65,12 @@ export function pageResources( return resources } -export function renderPage( +function renderTranscludes( + root: Root, cfg: GlobalConfiguration, slug: FullSlug, componentData: QuartzComponentProps, - components: RenderComponents, - pageResources: StaticResources, -): string { - // make a deep copy of the tree so we don't remove the transclusion references - // for the file cached in contentMap in build.ts - const root = clone(componentData.tree) as Root - +) { // process transcludes in componentData visit(root, "element", (node, _index, _parent) => { if (node.tagName === "blockquote") { @@ -191,6 +186,19 @@ export function renderPage( } } }) +} + +export function renderPage( + cfg: GlobalConfiguration, + slug: FullSlug, + componentData: QuartzComponentProps, + components: RenderComponents, + pageResources: StaticResources, +): string { + // make a deep copy of the tree so we don't remove the transclusion references + // for the file cached in contentMap in build.ts + const root = clone(componentData.tree) as Root + renderTranscludes(root, cfg, slug, componentData) // set componentData.tree to the edited html that has transclusions rendered componentData.tree = root diff --git a/quartz/depgraph.test.ts b/quartz/depgraph.test.ts deleted file mode 100644 index 062f13e35..000000000 --- a/quartz/depgraph.test.ts +++ /dev/null @@ -1,118 +0,0 @@ -import test, { describe } from "node:test" -import DepGraph from "./depgraph" -import assert from "node:assert" - -describe("DepGraph", () => { - test("getLeafNodes", () => { - const graph = new DepGraph() - graph.addEdge("A", "B") - graph.addEdge("B", "C") - graph.addEdge("D", "C") - assert.deepStrictEqual(graph.getLeafNodes("A"), new Set(["C"])) - assert.deepStrictEqual(graph.getLeafNodes("B"), new Set(["C"])) - assert.deepStrictEqual(graph.getLeafNodes("C"), new Set(["C"])) - assert.deepStrictEqual(graph.getLeafNodes("D"), new Set(["C"])) - }) - - describe("getLeafNodeAncestors", () => { - test("gets correct ancestors in a graph without cycles", () => { - const graph = new DepGraph() - graph.addEdge("A", "B") - graph.addEdge("B", "C") - graph.addEdge("D", "B") - assert.deepStrictEqual(graph.getLeafNodeAncestors("A"), new Set(["A", "B", "D"])) - assert.deepStrictEqual(graph.getLeafNodeAncestors("B"), new Set(["A", "B", "D"])) - assert.deepStrictEqual(graph.getLeafNodeAncestors("C"), new Set(["A", "B", "D"])) - assert.deepStrictEqual(graph.getLeafNodeAncestors("D"), new Set(["A", "B", "D"])) - }) - - test("gets correct ancestors in a graph with cycles", () => { - const graph = new DepGraph() - graph.addEdge("A", "B") - graph.addEdge("B", "C") - graph.addEdge("C", "A") - graph.addEdge("C", "D") - assert.deepStrictEqual(graph.getLeafNodeAncestors("A"), new Set(["A", "B", "C"])) - assert.deepStrictEqual(graph.getLeafNodeAncestors("B"), new Set(["A", "B", "C"])) - assert.deepStrictEqual(graph.getLeafNodeAncestors("C"), new Set(["A", "B", "C"])) - assert.deepStrictEqual(graph.getLeafNodeAncestors("D"), new Set(["A", "B", "C"])) - }) - }) - - describe("mergeGraph", () => { - test("merges two graphs", () => { - const graph = new DepGraph() - graph.addEdge("A.md", "A.html") - - const other = new DepGraph() - other.addEdge("B.md", "B.html") - - graph.mergeGraph(other) - - const expected = { - nodes: ["A.md", "A.html", "B.md", "B.html"], - edges: [ - ["A.md", "A.html"], - ["B.md", "B.html"], - ], - } - - assert.deepStrictEqual(graph.export(), expected) - }) - }) - - describe("updateIncomingEdgesForNode", () => { - test("merges when node exists", () => { - // A.md -> B.md -> B.html - const graph = new DepGraph() - graph.addEdge("A.md", "B.md") - graph.addEdge("B.md", "B.html") - - // B.md is edited so it removes the A.md transclusion - // and adds C.md transclusion - // C.md -> B.md - const other = new DepGraph() - other.addEdge("C.md", "B.md") - other.addEdge("B.md", "B.html") - - // A.md -> B.md removed, C.md -> B.md added - // C.md -> B.md -> B.html - graph.updateIncomingEdgesForNode(other, "B.md") - - const expected = { - nodes: ["A.md", "B.md", "B.html", "C.md"], - edges: [ - ["B.md", "B.html"], - ["C.md", "B.md"], - ], - } - - assert.deepStrictEqual(graph.export(), expected) - }) - - test("adds node if it does not exist", () => { - // A.md -> B.md - const graph = new DepGraph() - graph.addEdge("A.md", "B.md") - - // Add a new file C.md that transcludes B.md - // B.md -> C.md - const other = new DepGraph() - other.addEdge("B.md", "C.md") - - // B.md -> C.md added - // A.md -> B.md -> C.md - graph.updateIncomingEdgesForNode(other, "C.md") - - const expected = { - nodes: ["A.md", "B.md", "C.md"], - edges: [ - ["A.md", "B.md"], - ["B.md", "C.md"], - ], - } - - assert.deepStrictEqual(graph.export(), expected) - }) - }) -}) diff --git a/quartz/depgraph.ts b/quartz/depgraph.ts deleted file mode 100644 index 3d048cd83..000000000 --- a/quartz/depgraph.ts +++ /dev/null @@ -1,228 +0,0 @@ -export default class DepGraph { - // node: incoming and outgoing edges - _graph = new Map; outgoing: Set }>() - - constructor() { - this._graph = new Map() - } - - export(): Object { - return { - nodes: this.nodes, - edges: this.edges, - } - } - - toString(): string { - return JSON.stringify(this.export(), null, 2) - } - - // BASIC GRAPH OPERATIONS - - get nodes(): T[] { - return Array.from(this._graph.keys()) - } - - get edges(): [T, T][] { - let edges: [T, T][] = [] - this.forEachEdge((edge) => edges.push(edge)) - return edges - } - - hasNode(node: T): boolean { - return this._graph.has(node) - } - - addNode(node: T): void { - if (!this._graph.has(node)) { - this._graph.set(node, { incoming: new Set(), outgoing: new Set() }) - } - } - - // Remove node and all edges connected to it - removeNode(node: T): void { - if (this._graph.has(node)) { - // first remove all edges so other nodes don't have references to this node - for (const target of this._graph.get(node)!.outgoing) { - this.removeEdge(node, target) - } - for (const source of this._graph.get(node)!.incoming) { - this.removeEdge(source, node) - } - this._graph.delete(node) - } - } - - forEachNode(callback: (node: T) => void): void { - for (const node of this._graph.keys()) { - callback(node) - } - } - - hasEdge(from: T, to: T): boolean { - return Boolean(this._graph.get(from)?.outgoing.has(to)) - } - - addEdge(from: T, to: T): void { - this.addNode(from) - this.addNode(to) - - this._graph.get(from)!.outgoing.add(to) - this._graph.get(to)!.incoming.add(from) - } - - removeEdge(from: T, to: T): void { - if (this._graph.has(from) && this._graph.has(to)) { - this._graph.get(from)!.outgoing.delete(to) - this._graph.get(to)!.incoming.delete(from) - } - } - - // returns -1 if node does not exist - outDegree(node: T): number { - return this.hasNode(node) ? this._graph.get(node)!.outgoing.size : -1 - } - - // returns -1 if node does not exist - inDegree(node: T): number { - return this.hasNode(node) ? this._graph.get(node)!.incoming.size : -1 - } - - forEachOutNeighbor(node: T, callback: (neighbor: T) => void): void { - this._graph.get(node)?.outgoing.forEach(callback) - } - - forEachInNeighbor(node: T, callback: (neighbor: T) => void): void { - this._graph.get(node)?.incoming.forEach(callback) - } - - forEachEdge(callback: (edge: [T, T]) => void): void { - for (const [source, { outgoing }] of this._graph.entries()) { - for (const target of outgoing) { - callback([source, target]) - } - } - } - - // DEPENDENCY ALGORITHMS - - // Add all nodes and edges from other graph to this graph - mergeGraph(other: DepGraph): void { - other.forEachEdge(([source, target]) => { - this.addNode(source) - this.addNode(target) - this.addEdge(source, target) - }) - } - - // For the node provided: - // If node does not exist, add it - // If an incoming edge was added in other, it is added in this graph - // If an incoming edge was deleted in other, it is deleted in this graph - updateIncomingEdgesForNode(other: DepGraph, node: T): void { - this.addNode(node) - - // Add edge if it is present in other - other.forEachInNeighbor(node, (neighbor) => { - this.addEdge(neighbor, node) - }) - - // For node provided, remove incoming edge if it is absent in other - this.forEachEdge(([source, target]) => { - if (target === node && !other.hasEdge(source, target)) { - this.removeEdge(source, target) - } - }) - } - - // Remove all nodes that do not have any incoming or outgoing edges - // A node may be orphaned if the only node pointing to it was removed - removeOrphanNodes(): Set { - let orphanNodes = new Set() - - this.forEachNode((node) => { - if (this.inDegree(node) === 0 && this.outDegree(node) === 0) { - orphanNodes.add(node) - } - }) - - orphanNodes.forEach((node) => { - this.removeNode(node) - }) - - return orphanNodes - } - - // Get all leaf nodes (i.e. destination paths) reachable from the node provided - // Eg. if the graph is A -> B -> C - // D ---^ - // and the node is B, this function returns [C] - getLeafNodes(node: T): Set { - let stack: T[] = [node] - let visited = new Set() - let leafNodes = new Set() - - // DFS - while (stack.length > 0) { - let node = stack.pop()! - - // If the node is already visited, skip it - if (visited.has(node)) { - continue - } - visited.add(node) - - // Check if the node is a leaf node (i.e. destination path) - if (this.outDegree(node) === 0) { - leafNodes.add(node) - } - - // Add all unvisited neighbors to the stack - this.forEachOutNeighbor(node, (neighbor) => { - if (!visited.has(neighbor)) { - stack.push(neighbor) - } - }) - } - - return leafNodes - } - - // Get all ancestors of the leaf nodes reachable from the node provided - // Eg. if the graph is A -> B -> C - // D ---^ - // and the node is B, this function returns [A, B, D] - getLeafNodeAncestors(node: T): Set { - const leafNodes = this.getLeafNodes(node) - let visited = new Set() - let upstreamNodes = new Set() - - // Backwards DFS for each leaf node - leafNodes.forEach((leafNode) => { - let stack: T[] = [leafNode] - - while (stack.length > 0) { - let node = stack.pop()! - - if (visited.has(node)) { - continue - } - visited.add(node) - // Add node if it's not a leaf node (i.e. destination path) - // Assumes destination file cannot depend on another destination file - if (this.outDegree(node) !== 0) { - upstreamNodes.add(node) - } - - // Add all unvisited parents to the stack - this.forEachInNeighbor(node, (parentNode) => { - if (!visited.has(parentNode)) { - stack.push(parentNode) - } - }) - } - }) - - return upstreamNodes - } -} diff --git a/quartz/plugins/emitters/404.tsx b/quartz/plugins/emitters/404.tsx index 90c9d58c3..b0acae79c 100644 --- a/quartz/plugins/emitters/404.tsx +++ b/quartz/plugins/emitters/404.tsx @@ -9,7 +9,6 @@ import { NotFound } from "../../components" import { defaultProcessedContent } from "../vfile" import { write } from "./helpers" import { i18n } from "../../i18n" -import DepGraph from "../../depgraph" export const NotFoundPage: QuartzEmitterPlugin = () => { const opts: FullPageLayout = { @@ -28,9 +27,6 @@ export const NotFoundPage: QuartzEmitterPlugin = () => { getQuartzComponents() { return [Head, Body, pageBody, Footer] }, - async getDependencyGraph(_ctx, _content, _resources) { - return new DepGraph() - }, async *emit(ctx, _content, resources) { const cfg = ctx.cfg.configuration const slug = "404" as FullSlug diff --git a/quartz/plugins/emitters/aliases.ts b/quartz/plugins/emitters/aliases.ts index a16093b90..a26d7bc03 100644 --- a/quartz/plugins/emitters/aliases.ts +++ b/quartz/plugins/emitters/aliases.ts @@ -1,46 +1,47 @@ -import { FilePath, joinSegments, resolveRelative, simplifySlug } from "../../util/path" +import { resolveRelative, simplifySlug } from "../../util/path" import { QuartzEmitterPlugin } from "../types" import { write } from "./helpers" -import DepGraph from "../../depgraph" -import { getAliasSlugs } from "../transformers/frontmatter" +import { BuildCtx } from "../../util/ctx" +import { VFile } from "vfile" + +async function *processFile(ctx: BuildCtx, file: VFile) { + const ogSlug = simplifySlug(file.data.slug!) + + for (const slug of file.data.aliases ?? []) { + const redirUrl = resolveRelative(slug, file.data.slug!) + yield write({ + ctx, + content: ` + + + + ${ogSlug} + + + + + + + `, + slug, + ext: ".html", + }) + } +} export const AliasRedirects: QuartzEmitterPlugin = () => ({ name: "AliasRedirects", - async getDependencyGraph(ctx, content, _resources) { - const graph = new DepGraph() - - const { argv } = ctx + async *emit(ctx, content) { for (const [_tree, file] of content) { - for (const slug of getAliasSlugs(file.data.frontmatter?.aliases ?? [], argv, file)) { - graph.addEdge(file.data.filePath!, joinSegments(argv.output, slug + ".html") as FilePath) - } + yield* processFile(ctx, file) } - - return graph }, - async *emit(ctx, content, _resources) { - for (const [_tree, file] of content) { - const ogSlug = simplifySlug(file.data.slug!) - - for (const slug of file.data.aliases ?? []) { - const redirUrl = resolveRelative(slug, file.data.slug!) - yield write({ - ctx, - content: ` - - - - ${ogSlug} - - - - - - - `, - slug, - ext: ".html", - }) + async *partialEmit(ctx, _content, _resources, changeEvents) { + for (const changeEvent of changeEvents) { + if (!changeEvent.file) continue + if (changeEvent.type === "add" || changeEvent.type === "change") { + // add new ones if this file still exists + yield* processFile(ctx, changeEvent.file) } } }, diff --git a/quartz/plugins/emitters/assets.ts b/quartz/plugins/emitters/assets.ts index 120d168f5..a10ba41b7 100644 --- a/quartz/plugins/emitters/assets.ts +++ b/quartz/plugins/emitters/assets.ts @@ -3,7 +3,6 @@ import { QuartzEmitterPlugin } from "../types" import path from "path" import fs from "fs" import { glob } from "../../util/glob" -import DepGraph from "../../depgraph" import { Argv } from "../../util/ctx" import { QuartzConfig } from "../../cfg" @@ -12,41 +11,42 @@ const filesToCopy = async (argv: Argv, cfg: QuartzConfig) => { return await glob("**", argv.directory, ["**/*.md", ...cfg.configuration.ignorePatterns]) } +const copyFile = async (argv: Argv, fp: FilePath) => { + const src = joinSegments(argv.directory, fp) as FilePath + + const name = slugifyFilePath(fp) + const dest = joinSegments(argv.output, name) as FilePath + + // ensure dir exists + const dir = path.dirname(dest) as FilePath + await fs.promises.mkdir(dir, { recursive: true }) + + await fs.promises.copyFile(src, dest) + return dest +} + export const Assets: QuartzEmitterPlugin = () => { return { name: "Assets", - async getDependencyGraph(ctx, _content, _resources) { - const { argv, cfg } = ctx - const graph = new DepGraph() - - const fps = await filesToCopy(argv, cfg) - - for (const fp of fps) { - const ext = path.extname(fp) - const src = joinSegments(argv.directory, fp) as FilePath - const name = (slugifyFilePath(fp as FilePath, true) + ext) as FilePath - - const dest = joinSegments(argv.output, name) as FilePath - - graph.addEdge(src, dest) - } - - return graph - }, - async *emit({ argv, cfg }, _content, _resources) { - const assetsPath = argv.output + async *emit({ argv, cfg }) { const fps = await filesToCopy(argv, cfg) for (const fp of fps) { - const ext = path.extname(fp) - const src = joinSegments(argv.directory, fp) as FilePath - const name = (slugifyFilePath(fp as FilePath, true) + ext) as FilePath - - const dest = joinSegments(assetsPath, name) as FilePath - const dir = path.dirname(dest) as FilePath - await fs.promises.mkdir(dir, { recursive: true }) // ensure dir exists - await fs.promises.copyFile(src, dest) - yield dest + yield copyFile(argv, fp) } }, + async *partialEmit(ctx, _content, _resources, changeEvents) { + for (const changeEvent of changeEvents) { + const ext = path.extname(changeEvent.path) + if (ext === ".md") continue + + if (changeEvent.type === "add" || changeEvent.type === "change") { + yield copyFile(ctx.argv, changeEvent.path) + } else if (changeEvent.type === 'delete') { + const name = slugifyFilePath(changeEvent.path) + const dest = joinSegments(ctx.argv.output, name) as FilePath + await fs.promises.unlink(dest) + } + } + } } } diff --git a/quartz/plugins/emitters/cname.ts b/quartz/plugins/emitters/cname.ts index 897d8516d..a18574aaf 100644 --- a/quartz/plugins/emitters/cname.ts +++ b/quartz/plugins/emitters/cname.ts @@ -2,7 +2,6 @@ import { FilePath, joinSegments } from "../../util/path" import { QuartzEmitterPlugin } from "../types" import fs from "fs" import chalk from "chalk" -import DepGraph from "../../depgraph" export function extractDomainFromBaseUrl(baseUrl: string) { const url = new URL(`https://${baseUrl}`) @@ -11,10 +10,7 @@ export function extractDomainFromBaseUrl(baseUrl: string) { export const CNAME: QuartzEmitterPlugin = () => ({ name: "CNAME", - async getDependencyGraph(_ctx, _content, _resources) { - return new DepGraph() - }, - async emit({ argv, cfg }, _content, _resources) { + async emit({ argv, cfg }) { if (!cfg.configuration.baseUrl) { console.warn(chalk.yellow("CNAME emitter requires `baseUrl` to be set in your configuration")) return [] diff --git a/quartz/plugins/emitters/componentResources.ts b/quartz/plugins/emitters/componentResources.ts index fe855ba20..889e5070a 100644 --- a/quartz/plugins/emitters/componentResources.ts +++ b/quartz/plugins/emitters/componentResources.ts @@ -13,7 +13,6 @@ import { googleFontHref, joinStyles, processGoogleFonts } from "../../util/theme import { Features, transform } from "lightningcss" import { transform as transpile } from "esbuild" import { write } from "./helpers" -import DepGraph from "../../depgraph" type ComponentResources = { css: string[] @@ -203,9 +202,6 @@ function addGlobalPageResources(ctx: BuildCtx, componentResources: ComponentReso export const ComponentResources: QuartzEmitterPlugin = () => { return { name: "ComponentResources", - async getDependencyGraph(_ctx, _content, _resources) { - return new DepGraph() - }, async *emit(ctx, _content, _resources) { const cfg = ctx.cfg.configuration // component specific scripts and styles @@ -281,19 +277,21 @@ export const ComponentResources: QuartzEmitterPlugin = () => { }, include: Features.MediaQueries, }).code.toString(), - }), - yield write({ - ctx, - slug: "prescript" as FullSlug, - ext: ".js", - content: prescript, - }), - yield write({ - ctx, - slug: "postscript" as FullSlug, - ext: ".js", - content: postscript, - }) + }) + + yield write({ + ctx, + slug: "prescript" as FullSlug, + ext: ".js", + content: prescript, + }) + + yield write({ + ctx, + slug: "postscript" as FullSlug, + ext: ".js", + content: postscript, + }) }, } } diff --git a/quartz/plugins/emitters/contentIndex.tsx b/quartz/plugins/emitters/contentIndex.tsx index 6f43bad4a..01d2e0034 100644 --- a/quartz/plugins/emitters/contentIndex.tsx +++ b/quartz/plugins/emitters/contentIndex.tsx @@ -7,7 +7,6 @@ import { QuartzEmitterPlugin } from "../types" import { toHtml } from "hast-util-to-html" import { write } from "./helpers" import { i18n } from "../../i18n" -import DepGraph from "../../depgraph" export type ContentIndexMap = Map export type ContentDetails = { @@ -97,27 +96,7 @@ export const ContentIndex: QuartzEmitterPlugin> = (opts) => { opts = { ...defaultOptions, ...opts } return { name: "ContentIndex", - async getDependencyGraph(ctx, content, _resources) { - const graph = new DepGraph() - - for (const [_tree, file] of content) { - const sourcePath = file.data.filePath! - - graph.addEdge( - sourcePath, - joinSegments(ctx.argv.output, "static/contentIndex.json") as FilePath, - ) - if (opts?.enableSiteMap) { - graph.addEdge(sourcePath, joinSegments(ctx.argv.output, "sitemap.xml") as FilePath) - } - if (opts?.enableRSS) { - graph.addEdge(sourcePath, joinSegments(ctx.argv.output, "index.xml") as FilePath) - } - } - - return graph - }, - async *emit(ctx, content, _resources) { + async *emit(ctx, content) { const cfg = ctx.cfg.configuration const linkIndex: ContentIndexMap = new Map() for (const [tree, file] of content) { @@ -126,7 +105,7 @@ export const ContentIndex: QuartzEmitterPlugin> = (opts) => { if (opts?.includeEmptyFiles || (file.data.text && file.data.text !== "")) { linkIndex.set(slug, { slug, - filePath: file.data.filePath!, + filePath: file.data.relativePath!, title: file.data.frontmatter?.title!, links: file.data.links ?? [], tags: file.data.frontmatter?.tags ?? [], diff --git a/quartz/plugins/emitters/contentPage.tsx b/quartz/plugins/emitters/contentPage.tsx index f83393054..277d2e56b 100644 --- a/quartz/plugins/emitters/contentPage.tsx +++ b/quartz/plugins/emitters/contentPage.tsx @@ -14,43 +14,8 @@ import { defaultContentPageLayout, sharedPageComponents } from "../../../quartz. import { Content } from "../../components" import chalk from "chalk" import { write } from "./helpers" -import DepGraph from "../../depgraph" - -// get all the dependencies for the markdown file -// eg. images, scripts, stylesheets, transclusions -const parseDependencies = (argv: Argv, hast: Root, file: VFile): string[] => { - const dependencies: string[] = [] - - visit(hast, "element", (elem): void => { - let ref: string | null = null - - if ( - ["script", "img", "audio", "video", "source", "iframe"].includes(elem.tagName) && - elem?.properties?.src - ) { - ref = elem.properties.src.toString() - } else if (["a", "link"].includes(elem.tagName) && elem?.properties?.href) { - // transclusions will create a tags with relative hrefs - ref = elem.properties.href.toString() - } - - // if it is a relative url, its a local file and we need to add - // it to the dependency graph. otherwise, ignore - if (ref === null || !isRelativeURL(ref)) { - return - } - - let fp = path.join(file.data.filePath!, path.relative(argv.directory, ref)).replace(/\\/g, "/") - // markdown files have the .md extension stripped in hrefs, add it back here - if (!fp.split("/").pop()?.includes(".")) { - fp += ".md" - } - dependencies.push(fp) - }) - - return dependencies -} +// TODO check for transclusions in partial rebuild export const ContentPage: QuartzEmitterPlugin> = (userOpts) => { const opts: FullPageLayout = { ...sharedPageComponents, @@ -79,21 +44,6 @@ export const ContentPage: QuartzEmitterPlugin> = (userOp Footer, ] }, - async getDependencyGraph(ctx, content, _resources) { - const graph = new DepGraph() - - for (const [tree, file] of content) { - const sourcePath = file.data.filePath! - const slug = file.data.slug! - graph.addEdge(sourcePath, joinSegments(ctx.argv.output, slug + ".html") as FilePath) - - parseDependencies(ctx.argv, tree as Root, file).forEach((dep) => { - graph.addEdge(dep as FilePath, sourcePath) - }) - } - - return graph - }, async *emit(ctx, content, resources) { const cfg = ctx.cfg.configuration const allFiles = content.map((c) => c[1].data) @@ -129,7 +79,7 @@ export const ContentPage: QuartzEmitterPlugin> = (userOp }) } - if (!containsIndex && !ctx.argv.fastRebuild) { + if (!containsIndex) { console.log( chalk.yellow( `\nWarning: you seem to be missing an \`index.md\` home page file at the root of your \`${ctx.argv.directory}\` folder (\`${path.join(ctx.argv.directory, "index.md")} does not exist\`). This may cause errors when deploying.`, diff --git a/quartz/plugins/emitters/folderPage.tsx b/quartz/plugins/emitters/folderPage.tsx index 1c812077f..cdd2c728c 100644 --- a/quartz/plugins/emitters/folderPage.tsx +++ b/quartz/plugins/emitters/folderPage.tsx @@ -19,7 +19,6 @@ import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.lay import { FolderContent } from "../../components" import { write } from "./helpers" import { i18n } from "../../i18n" -import DepGraph from "../../depgraph" interface FolderPageOptions extends FullPageLayout { sort?: (f1: QuartzPluginData, f2: QuartzPluginData) => number @@ -53,22 +52,6 @@ export const FolderPage: QuartzEmitterPlugin> = (user Footer, ] }, - async getDependencyGraph(_ctx, content, _resources) { - // Example graph: - // nested/file.md --> nested/index.html - // nested/file2.md ------^ - const graph = new DepGraph() - - content.map(([_tree, vfile]) => { - const slug = vfile.data.slug - const folderName = path.dirname(slug ?? "") as SimpleSlug - if (slug && folderName !== "." && folderName !== "tags") { - graph.addEdge(vfile.data.filePath!, joinSegments(folderName, "index.html") as FilePath) - } - }) - - return graph - }, async *emit(ctx, content, resources) { const allFiles = content.map((c) => c[1].data) const cfg = ctx.cfg.configuration diff --git a/quartz/plugins/emitters/helpers.ts b/quartz/plugins/emitters/helpers.ts index 6218178a4..fc14bd97e 100644 --- a/quartz/plugins/emitters/helpers.ts +++ b/quartz/plugins/emitters/helpers.ts @@ -11,6 +11,8 @@ type WriteOptions = { content: string | Buffer | Readable } +type DeleteOptions = Omit + export const write = async ({ ctx, slug, ext, content }: WriteOptions): Promise => { const pathToPage = joinSegments(ctx.argv.output, slug + ext) as FilePath const dir = path.dirname(pathToPage) diff --git a/quartz/plugins/emitters/static.ts b/quartz/plugins/emitters/static.ts index 7c0ecfd0e..380ee2893 100644 --- a/quartz/plugins/emitters/static.ts +++ b/quartz/plugins/emitters/static.ts @@ -2,26 +2,11 @@ import { FilePath, QUARTZ, joinSegments } from "../../util/path" import { QuartzEmitterPlugin } from "../types" import fs from "fs" import { glob } from "../../util/glob" -import DepGraph from "../../depgraph" import { dirname } from "path" export const Static: QuartzEmitterPlugin = () => ({ name: "Static", - async getDependencyGraph({ argv, cfg }, _content, _resources) { - const graph = new DepGraph() - - const staticPath = joinSegments(QUARTZ, "static") - const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns) - for (const fp of fps) { - graph.addEdge( - joinSegments("static", fp) as FilePath, - joinSegments(argv.output, "static", fp) as FilePath, - ) - } - - return graph - }, - async *emit({ argv, cfg }, _content) { + async *emit({ argv, cfg }) { const staticPath = joinSegments(QUARTZ, "static") const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns) const outputStaticPath = joinSegments(argv.output, "static") diff --git a/quartz/plugins/emitters/tagPage.tsx b/quartz/plugins/emitters/tagPage.tsx index 41cf09145..44fe24662 100644 --- a/quartz/plugins/emitters/tagPage.tsx +++ b/quartz/plugins/emitters/tagPage.tsx @@ -16,7 +16,6 @@ import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.lay import { TagContent } from "../../components" import { write } from "./helpers" import { i18n } from "../../i18n" -import DepGraph from "../../depgraph" interface TagPageOptions extends FullPageLayout { sort?: (f1: QuartzPluginData, f2: QuartzPluginData) => number @@ -50,27 +49,6 @@ export const TagPage: QuartzEmitterPlugin> = (userOpts) Footer, ] }, - async getDependencyGraph(ctx, content, _resources) { - const graph = new DepGraph() - - for (const [_tree, file] of content) { - const sourcePath = file.data.filePath! - const tags = (file.data.frontmatter?.tags ?? []).flatMap(getAllSegmentPrefixes) - // if the file has at least one tag, it is used in the tag index page - if (tags.length > 0) { - tags.push("index") - } - - for (const tag of tags) { - graph.addEdge( - sourcePath, - joinSegments(ctx.argv.output, "tags", tag + ".html") as FilePath, - ) - } - } - - return graph - }, async *emit(ctx, content, resources) { const allFiles = content.map((c) => c[1].data) const cfg = ctx.cfg.configuration diff --git a/quartz/plugins/transformers/frontmatter.ts b/quartz/plugins/transformers/frontmatter.ts index b3a916a65..d4f31c204 100644 --- a/quartz/plugins/transformers/frontmatter.ts +++ b/quartz/plugins/transformers/frontmatter.ts @@ -3,12 +3,9 @@ import remarkFrontmatter from "remark-frontmatter" import { QuartzTransformerPlugin } from "../types" import yaml from "js-yaml" import toml from "toml" -import { FilePath, FullSlug, joinSegments, slugifyFilePath, slugTag } from "../../util/path" +import { FilePath, FullSlug, getFileExtension, joinSegments, slugifyFilePath, slugTag, trimSuffix } from "../../util/path" import { QuartzPluginData } from "../vfile" import { i18n } from "../../i18n" -import { Argv } from "../../util/ctx" -import { VFile } from "vfile" -import path from "path" export interface Options { delimiters: string | [string, string] @@ -43,26 +40,24 @@ function coerceToArray(input: string | string[]): string[] | undefined { .map((tag: string | number) => tag.toString()) } -export function getAliasSlugs(aliases: string[], argv: Argv, file: VFile): FullSlug[] { - const dir = path.posix.relative(argv.directory, path.dirname(file.data.filePath!)) - const slugs: FullSlug[] = aliases.map( - (alias) => path.posix.join(dir, slugifyFilePath(alias as FilePath)) as FullSlug, - ) - const permalink = file.data.frontmatter?.permalink - if (typeof permalink === "string") { - slugs.push(permalink as FullSlug) +function getAliasSlugs(aliases: string[]): FullSlug[] { + const res: FullSlug[] = [] + for (const alias of aliases) { + const isMd = getFileExtension(alias) === "md" + const mockFp = isMd ? alias : alias + ".md" + const slug = slugifyFilePath(mockFp as FilePath) + res.push(slug) } - // fix any slugs that have trailing slash - return slugs.map((slug) => - slug.endsWith("/") ? (joinSegments(slug, "index") as FullSlug) : slug, - ) + + return res } export const FrontMatter: QuartzTransformerPlugin> = (userOpts) => { const opts = { ...defaultOptions, ...userOpts } return { name: "FrontMatter", - markdownPlugins({ cfg, allSlugs, argv }) { + markdownPlugins(ctx) { + const { cfg, allSlugs } = ctx return [ [remarkFrontmatter, ["yaml", "toml"]], () => { @@ -88,9 +83,18 @@ export const FrontMatter: QuartzTransformerPlugin> = (userOpts) const aliases = coerceToArray(coalesceAliases(data, ["aliases", "alias"])) if (aliases) { data.aliases = aliases // frontmatter - const slugs = (file.data.aliases = getAliasSlugs(aliases, argv, file)) - allSlugs.push(...slugs) + file.data.aliases = getAliasSlugs(aliases) + allSlugs.push(...file.data.aliases) } + + if (data.permalink != null && data.permalink.toString() !== "") { + data.permalink = data.permalink.toString() as FullSlug + const aliases = file.data.aliases ?? [] + aliases.push(data.permalink) + file.data.aliases = aliases + allSlugs.push(data.permalink) + } + const cssclasses = coerceToArray(coalesceAliases(data, ["cssclasses", "cssclass"])) if (cssclasses) data.cssclasses = cssclasses diff --git a/quartz/plugins/transformers/lastmod.ts b/quartz/plugins/transformers/lastmod.ts index fd5769263..aeabad19e 100644 --- a/quartz/plugins/transformers/lastmod.ts +++ b/quartz/plugins/transformers/lastmod.ts @@ -31,7 +31,7 @@ export const CreatedModifiedDate: QuartzTransformerPlugin> = (u const opts = { ...defaultOptions, ...userOpts } return { name: "CreatedModifiedDate", - markdownPlugins() { + markdownPlugins(ctx) { return [ () => { let repo: Repository | undefined = undefined @@ -40,8 +40,8 @@ export const CreatedModifiedDate: QuartzTransformerPlugin> = (u let modified: MaybeDate = undefined let published: MaybeDate = undefined - const fp = file.data.filePath! - const fullFp = path.isAbsolute(fp) ? fp : path.posix.join(file.cwd, fp) + const fp = file.data.relativePath! + const fullFp = path.posix.join(ctx.argv.directory, fp) for (const source of opts.priority) { if (source === "filesystem") { const st = await fs.promises.stat(fullFp) @@ -56,11 +56,11 @@ export const CreatedModifiedDate: QuartzTransformerPlugin> = (u // Get a reference to the main git repo. // It's either the same as the workdir, // or 1+ level higher in case of a submodule/subtree setup - repo = Repository.discover(file.cwd) + repo = Repository.discover(ctx.argv.directory) } try { - modified ||= await repo.getFileLatestModifiedDateAsync(file.data.filePath!) + modified ||= await repo.getFileLatestModifiedDateAsync(fullFp) } catch { console.log( chalk.yellow( diff --git a/quartz/plugins/types.ts b/quartz/plugins/types.ts index 166ec5dfa..f3627c826 100644 --- a/quartz/plugins/types.ts +++ b/quartz/plugins/types.ts @@ -4,7 +4,7 @@ import { ProcessedContent } from "./vfile" import { QuartzComponent } from "../components/types" import { FilePath } from "../util/path" import { BuildCtx } from "../util/ctx" -import DepGraph from "../depgraph" +import { VFile } from "vfile" export interface PluginTypes { transformers: QuartzTransformerPluginInstance[] @@ -33,6 +33,12 @@ export type QuartzFilterPluginInstance = { shouldPublish(ctx: BuildCtx, content: ProcessedContent): boolean } +export type ChangeEvent = { + type: "add" | "change" | "delete" + path: FilePath + file?: VFile +} + export type QuartzEmitterPlugin = ( opts?: Options, ) => QuartzEmitterPluginInstance @@ -43,16 +49,17 @@ export type QuartzEmitterPluginInstance = { content: ProcessedContent[], resources: StaticResources, ): Promise | AsyncGenerator + partialEmit?( + ctx: BuildCtx, + content: ProcessedContent[], + resources: StaticResources, + changeEvents: ChangeEvent[], + ): Promise | AsyncGenerator /** * Returns the components (if any) that are used in rendering the page. * This helps Quartz optimize the page by only including necessary resources * for components that are actually used. */ getQuartzComponents?: (ctx: BuildCtx) => QuartzComponent[] - getDependencyGraph?( - ctx: BuildCtx, - content: ProcessedContent[], - resources: StaticResources, - ): Promise> externalResources?: ExternalResourcesFn } diff --git a/quartz/processors/emit.ts b/quartz/processors/emit.ts index aae119dfb..00bc9c82c 100644 --- a/quartz/processors/emit.ts +++ b/quartz/processors/emit.ts @@ -11,7 +11,7 @@ export async function emitContent(ctx: BuildCtx, content: ProcessedContent[]) { const perf = new PerfTimer() const log = new QuartzLogger(ctx.argv.verbose) - log.start(`Emitting output files`) + log.start(`Emitting files`) let emittedFiles = 0 const staticResources = getStaticResourcesFromPlugins(ctx) @@ -26,7 +26,7 @@ export async function emitContent(ctx: BuildCtx, content: ProcessedContent[]) { if (ctx.argv.verbose) { console.log(`[emit:${emitter.name}] ${file}`) } else { - log.updateText(`Emitting output files: ${emitter.name} -> ${chalk.gray(file)}`) + log.updateText(`${emitter.name} -> ${chalk.gray(file)}`) } } } else { @@ -36,7 +36,7 @@ export async function emitContent(ctx: BuildCtx, content: ProcessedContent[]) { if (ctx.argv.verbose) { console.log(`[emit:${emitter.name}] ${file}`) } else { - log.updateText(`Emitting output files: ${emitter.name} -> ${chalk.gray(file)}`) + log.updateText(`${emitter.name} -> ${chalk.gray(file)}`) } } } diff --git a/quartz/processors/parse.ts b/quartz/processors/parse.ts index 479313f49..56421c8f8 100644 --- a/quartz/processors/parse.ts +++ b/quartz/processors/parse.ts @@ -12,7 +12,8 @@ import path from "path" import workerpool, { Promise as WorkerPromise } from "workerpool" import { QuartzLogger } from "../util/log" import { trace } from "../util/trace" -import { BuildCtx } from "../util/ctx" +import { BuildCtx, WorkerSerializableBuildCtx } from "../util/ctx" +import chalk from "chalk" export type QuartzMdProcessor = Processor export type QuartzHtmlProcessor = Processor @@ -175,21 +176,42 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise[] = [] - for (const chunk of chunks(fps, CHUNK_SIZE)) { - mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, argv, chunk])) + const serializableCtx: WorkerSerializableBuildCtx = { + buildId: ctx.buildId, + argv: ctx.argv, + allSlugs: ctx.allSlugs, + allFiles: ctx.allFiles, + incremental: ctx.incremental, } - const mdResults: [MarkdownContent[], FullSlug[]][] = - await WorkerPromise.all(mdPromises).catch(errorHandler) - const childPromises: WorkerPromise[] = [] - for (const [_, extraSlugs] of mdResults) { - ctx.allSlugs.push(...extraSlugs) + const textToMarkdownPromises: WorkerPromise[] = [] + let processedFiles = 0 + for (const chunk of chunks(fps, CHUNK_SIZE)) { + textToMarkdownPromises.push(pool.exec("parseMarkdown", [serializableCtx, chunk])) } + + const mdResults: Array = await Promise.all( + textToMarkdownPromises.map(async (promise) => { + const result = await promise + processedFiles += result.length + log.updateText(`text->markdown ${chalk.gray(`${processedFiles}/${fps.length}`)}`) + return result + }), + ).catch(errorHandler) + + const markdownToHtmlPromises: WorkerPromise[] = [] + processedFiles = 0 for (const [mdChunk, _] of mdResults) { - childPromises.push(pool.exec("processHtml", [ctx.buildId, argv, mdChunk, ctx.allSlugs])) + markdownToHtmlPromises.push(pool.exec("processHtml", [serializableCtx, mdChunk])) } - const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch(errorHandler) + const results: ProcessedContent[][] = await Promise.all( + markdownToHtmlPromises.map(async (promise) => { + const result = await promise + processedFiles += result.length + log.updateText(`markdown->html ${chalk.gray(`${processedFiles}/${fps.length}`)}`) + return result + }), + ).catch(errorHandler) res = results.flat() await pool.terminate() diff --git a/quartz/util/ctx.ts b/quartz/util/ctx.ts index 044d21f68..b3e7a37f5 100644 --- a/quartz/util/ctx.ts +++ b/quartz/util/ctx.ts @@ -1,12 +1,12 @@ import { QuartzConfig } from "../cfg" -import { FullSlug } from "./path" +import { FilePath, FullSlug } from "./path" export interface Argv { directory: string verbose: boolean output: string serve: boolean - fastRebuild: boolean + watch: boolean port: number wsPort: number remoteDevHost?: string @@ -18,4 +18,8 @@ export interface BuildCtx { argv: Argv cfg: QuartzConfig allSlugs: FullSlug[] + allFiles: FilePath[] + incremental: boolean } + +export type WorkerSerializableBuildCtx = Omit diff --git a/quartz/util/log.ts b/quartz/util/log.ts index 4fcc2404f..7584b83ef 100644 --- a/quartz/util/log.ts +++ b/quartz/util/log.ts @@ -1,9 +1,11 @@ +import truncate from "ansi-truncate" import readline from "readline" export class QuartzLogger { verbose: boolean private spinnerInterval: NodeJS.Timeout | undefined private spinnerText: string = "" + private updateSuffix: string = "" private spinnerIndex: number = 0 private readonly spinnerChars = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"] @@ -13,6 +15,7 @@ export class QuartzLogger { start(text: string) { this.spinnerText = text + if (this.verbose) { console.log(text) } else { @@ -20,14 +23,22 @@ export class QuartzLogger { this.spinnerInterval = setInterval(() => { readline.clearLine(process.stdout, 0) readline.cursorTo(process.stdout, 0) - process.stdout.write(`${this.spinnerChars[this.spinnerIndex]} ${this.spinnerText}`) + + const columns = process.stdout.columns || 80 + let output = `${this.spinnerChars[this.spinnerIndex]} ${this.spinnerText}` + if (this.updateSuffix) { + output += `: ${this.updateSuffix}` + } + + const truncated = truncate(output, columns) + process.stdout.write(truncated) this.spinnerIndex = (this.spinnerIndex + 1) % this.spinnerChars.length }, 20) } } updateText(text: string) { - this.spinnerText = text + this.updateSuffix = text } end(text?: string) { diff --git a/quartz/util/path.ts b/quartz/util/path.ts index 6d99c364e..6a5e20adf 100644 --- a/quartz/util/path.ts +++ b/quartz/util/path.ts @@ -260,7 +260,7 @@ export function endsWith(s: string, suffix: string): boolean { return s === suffix || s.endsWith("/" + suffix) } -function trimSuffix(s: string, suffix: string): string { +export function trimSuffix(s: string, suffix: string): string { if (endsWith(s, suffix)) { s = s.slice(0, -suffix.length) } diff --git a/quartz/worker.ts b/quartz/worker.ts index c9cd98055..f47726441 100644 --- a/quartz/worker.ts +++ b/quartz/worker.ts @@ -1,7 +1,7 @@ import sourceMapSupport from "source-map-support" sourceMapSupport.install(options) import cfg from "../quartz.config" -import { Argv, BuildCtx } from "./util/ctx" +import { Argv, BuildCtx, WorkerSerializableBuildCtx } from "./util/ctx" import { FilePath, FullSlug } from "./util/path" import { createFileParser, @@ -14,35 +14,24 @@ import { MarkdownContent, ProcessedContent } from "./plugins/vfile" // only called from worker thread export async function parseMarkdown( - buildId: string, - argv: Argv, + partialCtx: WorkerSerializableBuildCtx, fps: FilePath[], -): Promise<[MarkdownContent[], FullSlug[]]> { - // this is a hack - // we assume markdown parsers can add to `allSlugs`, - // but don't actually use them - const allSlugs: FullSlug[] = [] +): Promise { const ctx: BuildCtx = { - buildId, + ...partialCtx, cfg, - argv, - allSlugs, } - return [await createFileParser(ctx, fps)(createMdProcessor(ctx)), allSlugs] + return await createFileParser(ctx, fps)(createMdProcessor(ctx)) } // only called from worker thread export function processHtml( - buildId: string, - argv: Argv, + partialCtx: WorkerSerializableBuildCtx, mds: MarkdownContent[], - allSlugs: FullSlug[], ): Promise { const ctx: BuildCtx = { - buildId, + ...partialCtx, cfg, - argv, - allSlugs, } return createMarkdownParser(ctx, mds)(createHtmlProcessor(ctx)) }