From 2e299c67ccbd48552273b9bae1d4e9afecbd0715 Mon Sep 17 00:00:00 2001 From: Anton Bulakh Date: Sat, 18 Jan 2025 22:40:07 +0200 Subject: [PATCH] feat: untangle quartz from local configs in least amount of changes For the current setup where people have to fork or at least clone quartz this changes nothing - but it allows you to install quartz as a devDependency via npm and have it actually work. One real change is switch from `.quartz-cache` to `node_modules/.cache/quartz` for transpilation results, this is an artifact from my previous attempts, I guess with this one I can change it back - but `node_modules/.cache` feels more better imo idk. edit: OTOH if you want to have quartz be a _completely_ separate binary (which this also enables I think), having it create a node_modules folder is weird, so I made a quick hack for that for now. Example: ```bash $ mkdir my-repo && cd my-repo $ npm i quartz@necauqua/quartz#untangled # quartz@ prefix is important $ cp node_modules/quartz/quartz.*.ts . # copy the default configs $ mkdir content && echo "# Hello World!" > content/index.md $ npx quartz build --serve # this just works! $ echo 'body { background: red !important; }' > styles.scss ``` Notice how I used my branch in the `npm i` line, ideally it'd be `npm i quartz@jackyzho0/quartz`, or maybe we can somehow get the quartz package on npm and it'll just be `npm i quartz`. In the latter case `npx quartz build` will literally just work without a local npm package at all?. Having some support for components and plugins being in separate npm packages instead of people copying code around is not out of the picture with this too btw. Closes #502 MOVE ME --- ambient.d.ts | 40 +++++++++++++++++++ docs/advanced/architecture.md | 2 +- index.d.ts | 12 ------ index.ts | 1 + package.json | 1 + quartz.config.ts | 3 +- quartz.layout.ts | 3 +- quartz/bootstrap-worker.mjs | 2 +- quartz/build.ts | 9 +++-- quartz/cli/constants.js | 27 ++++++++++--- quartz/cli/handlers.js | 21 +++++++--- quartz/index.ts | 4 ++ quartz/plugins/emitters/404.tsx | 2 +- quartz/plugins/emitters/componentResources.ts | 11 ++++- quartz/plugins/emitters/contentPage.tsx | 2 +- quartz/plugins/emitters/folderPage.tsx | 2 +- quartz/plugins/emitters/static.ts | 10 ++--- quartz/plugins/emitters/tagPage.tsx | 2 +- quartz/processors/parse.ts | 34 +++++++++++----- quartz/util/ctx.ts | 1 + quartz/util/path.ts | 2 - quartz/util/sourcemap.ts | 2 +- quartz/worker.ts | 7 +++- 23 files changed, 142 insertions(+), 58 deletions(-) create mode 100644 ambient.d.ts delete mode 100644 index.d.ts create mode 100644 index.ts create mode 100644 quartz/index.ts diff --git a/ambient.d.ts b/ambient.d.ts new file mode 100644 index 000000000..4d9b9e253 --- /dev/null +++ b/ambient.d.ts @@ -0,0 +1,40 @@ +declare module "*.scss" { + const content: string + export = content +} + +declare module "$config" { + import { QuartzConfig } from "./quartz" + + const config: QuartzConfig + export = config +} + +declare module "$layout" { + import { SharedLayout, PageLayout } from "./quartz/cfg" + + export const sharedPageComponents: SharedLayout + export const defaultContentPageLayout: PageLayout + export const defaultListPageLayout: PageLayout +} + +declare module "$styles" { + const content: string + export = content +} + +declare module "quartz" { + // without this the export below does nothing for some reason + // sometimes TS is funn + import("./quartz") + + export * from "./quartz" +} + +// dom custom event +interface CustomEventMap { + nav: CustomEvent<{ url: FullSlug }> + themechange: CustomEvent<{ theme: "light" | "dark" }> +} + +declare const fetchData: Promise diff --git a/docs/advanced/architecture.md b/docs/advanced/architecture.md index 33da89d90..508f06be8 100644 --- a/docs/advanced/architecture.md +++ b/docs/advanced/architecture.md @@ -17,7 +17,7 @@ This question is best answered by tracing what happens when a user (you!) runs ` 1. A WebSocket server on port 3001 to handle hot-reload signals. This tracks all inbound connections and sends a 'rebuild' message a server-side change is detected (either content or configuration). 2. An HTTP file-server on a user defined port (normally 8080) to serve the actual website files. 4. If the `--serve` flag is set, it also starts a file watcher to detect source-code changes (e.g. anything that is `.ts`, `.tsx`, `.scss`, or packager files). On a change, we rebuild the module (step 2 above) using esbuild's [rebuild API](https://esbuild.github.io/api/#rebuild) which drastically reduces the build times. - 5. After transpiling the main Quartz build module (`quartz/build.ts`), we write it to a cache file `.quartz-cache/transpiled-build.mjs` and then dynamically import this using `await import(cacheFile)`. However, we need to be pretty smart about how to bust Node's [import cache](https://github.com/nodejs/modules/issues/307) so we add a random query string to fake Node into thinking it's a new module. This does, however, cause memory leaks so we just hope that the user doesn't hot-reload their configuration too many times in a single session :)) (it leaks about ~350kB memory on each reload). After importing the module, we then invoke it, passing in the command line arguments we parsed earlier along with a callback function to signal the client to refresh. + 5. After transpiling the main Quartz build module (`quartz/build.ts`), we write it to a cache file `node_modules/.cache/quartz/transpiled-build.mjs` and then dynamically import this using `await import(cacheFile)`. However, we need to be pretty smart about how to bust Node's [import cache](https://github.com/nodejs/modules/issues/307) so we add a random query string to fake Node into thinking it's a new module. This does, however, cause memory leaks so we just hope that the user doesn't hot-reload their configuration too many times in a single session :)) (it leaks about ~350kB memory on each reload). After importing the module, we then invoke it, passing in the command line arguments we parsed earlier along with a callback function to signal the client to refresh. 4. In `build.ts`, we start by installing source map support manually to account for the query string cache busting hack we introduced earlier. Then, we start processing content: 1. Clean the output directory. 2. Recursively glob all files in the `content` folder, respecting the `.gitignore`. diff --git a/index.d.ts b/index.d.ts deleted file mode 100644 index a6c594fff..000000000 --- a/index.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -declare module "*.scss" { - const content: string - export = content -} - -// dom custom event -interface CustomEventMap { - nav: CustomEvent<{ url: FullSlug }> - themechange: CustomEvent<{ theme: "light" | "dark" }> -} - -declare const fetchData: Promise diff --git a/index.ts b/index.ts new file mode 100644 index 000000000..fb616a699 --- /dev/null +++ b/index.ts @@ -0,0 +1 @@ +export * from "./quartz" diff --git a/package.json b/package.json index 726350b3b..781a7a896 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "bin": { "quartz": "./quartz/bootstrap-cli.mjs" }, + "types": "./ambient.d.ts", "dependencies": { "@clack/prompts": "^0.9.1", "@floating-ui/dom": "^1.6.13", diff --git a/quartz.config.ts b/quartz.config.ts index dc339d987..efa60434e 100644 --- a/quartz.config.ts +++ b/quartz.config.ts @@ -1,5 +1,4 @@ -import { QuartzConfig } from "./quartz/cfg" -import * as Plugin from "./quartz/plugins" +import { QuartzConfig, Plugin } from "quartz" /** * Quartz 4.0 Configuration diff --git a/quartz.layout.ts b/quartz.layout.ts index 4a78256aa..1b840097c 100644 --- a/quartz.layout.ts +++ b/quartz.layout.ts @@ -1,5 +1,4 @@ -import { PageLayout, SharedLayout } from "./quartz/cfg" -import * as Component from "./quartz/components" +import { Component, PageLayout, SharedLayout } from "quartz" // components shared across all pages export const sharedPageComponents: SharedLayout = { diff --git a/quartz/bootstrap-worker.mjs b/quartz/bootstrap-worker.mjs index c4c4949b9..7651c7970 100644 --- a/quartz/bootstrap-worker.mjs +++ b/quartz/bootstrap-worker.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node import workerpool from "workerpool" -const cacheFile = "./.quartz-cache/transpiled-worker.mjs" +const cacheFile = process.argv[2] const { parseMarkdown, processHtml } = await import(cacheFile) workerpool.worker({ parseMarkdown, diff --git a/quartz/build.ts b/quartz/build.ts index 64c462b14..91a551e29 100644 --- a/quartz/build.ts +++ b/quartz/build.ts @@ -8,7 +8,7 @@ import chalk from "chalk" import { parseMarkdown } from "./processors/parse" import { filterContent } from "./processors/filter" import { emitContent } from "./processors/emit" -import cfg from "../quartz.config" +import cfg from "$config" import { FilePath, FullSlug, joinSegments, slugifyFilePath } from "./util/path" import chokidar from "chokidar" import { ProcessedContent } from "./plugins/vfile" @@ -42,12 +42,13 @@ function newBuildId() { return Math.random().toString(36).substring(2, 8) } -async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) { +async function buildQuartz(quartzRoot: string, argv: Argv, mut: Mutex, clientRefresh: () => void) { const ctx: BuildCtx = { buildId: newBuildId(), argv, cfg, allSlugs: [], + quartzRoot, } const perf = new PerfTimer() @@ -413,9 +414,9 @@ async function rebuildFromEntrypoint( release() } -export default async (argv: Argv, mut: Mutex, clientRefresh: () => void) => { +export default async (quartzRoot: string, argv: Argv, mut: Mutex, clientRefresh: () => void) => { try { - return await buildQuartz(argv, mut, clientRefresh) + return await buildQuartz(quartzRoot, argv, mut, clientRefresh) } catch (err) { trace("\nExiting Quartz due to a fatal error", err as Error) } diff --git a/quartz/cli/constants.js b/quartz/cli/constants.js index f4a9ce52b..7f2a3a18c 100644 --- a/quartz/cli/constants.js +++ b/quartz/cli/constants.js @@ -1,5 +1,5 @@ import path from "path" -import { readFileSync } from "fs" +import { accessSync, readFileSync } from "fs" /** * All constants relating to helpers or handlers @@ -7,9 +7,26 @@ import { readFileSync } from "fs" export const ORIGIN_NAME = "origin" export const UPSTREAM_NAME = "upstream" export const QUARTZ_SOURCE_BRANCH = "v4" + export const cwd = process.cwd() -export const cacheDir = path.join(cwd, ".quartz-cache") -export const cacheFile = "./quartz/.quartz-cache/transpiled-build.mjs" -export const fp = "./quartz/build.ts" -export const { version } = JSON.parse(readFileSync("./package.json").toString()) + +function selectCacheDir() { + try { + const node_modules = path.join(cwd, "node_modules") + accessSync(node_modules) // check if node_modules exists + return path.join(node_modules, ".cache", "quartz") + } catch { + // standalone quartz bin? + return path.join(cwd, ".quartz-cache") + } +} + +export const cacheDir = selectCacheDir() +export const cacheFile = path.join(cacheDir, "transpiled-build.mjs") export const contentCacheFolder = path.join(cacheDir, "content-cache") + +export const quartzRoot = path.resolve(import.meta.dirname, "..") +export const fp = path.join(quartzRoot, "build.ts") +export const { version } = JSON.parse( + readFileSync(path.resolve(quartzRoot, "..", "package.json")).toString(), +) diff --git a/quartz/cli/handlers.js b/quartz/cli/handlers.js index 6b23d8010..3c083e144 100644 --- a/quartz/cli/handlers.js +++ b/quartz/cli/handlers.js @@ -31,7 +31,9 @@ import { fp, cacheFile, cwd, + quartzRoot, } from "./constants.js" +import { pathToFileURL } from "url" /** * Handles `npx quartz create` @@ -232,6 +234,12 @@ export async function handleBuild(argv) { metafile: true, sourcemap: true, sourcesContent: false, + alias: { + $config: path.join(cwd, "quartz.config.ts"), + $layout: path.join(cwd, "quartz.layout.ts"), + $styles: path.join(cwd, "styles.scss"), + quartz: path.resolve(quartzRoot, ".."), + }, plugins: [ sassPlugin({ type: "css-text", @@ -303,8 +311,9 @@ export async function handleBuild(argv) { release() if (argv.bundleInfo) { - const outputFileName = "quartz/.quartz-cache/transpiled-build.mjs" - const meta = result.metafile.outputs[outputFileName] + // metafile.outputs always uses / + const output = path.relative(cwd, cacheFile).replaceAll("\\", "/") + const meta = result.metafile.outputs[output] console.log( `Successfully transpiled ${Object.keys(meta.inputs).length} files (${prettyBytes( meta.bytes, @@ -313,12 +322,14 @@ export async function handleBuild(argv) { console.log(await esbuild.analyzeMetafile(result.metafile, { color: true })) } + // absolute path on windows has to be a file:// url + const url = pathToFileURL(cacheFile) // bypass module cache // https://github.com/nodejs/modules/issues/307 - const { default: buildQuartz } = await import(`../../${cacheFile}?update=${randomUUID()}`) - // ^ this import is relative, so base "cacheFile" path can't be used + url.searchParams.set("update", randomUUID()) + const { default: buildQuartz } = await import(url) - cleanupBuild = await buildQuartz(argv, buildMutex, clientRefresh) + cleanupBuild = await buildQuartz(quartzRoot, argv, buildMutex, clientRefresh) clientRefresh() } diff --git a/quartz/index.ts b/quartz/index.ts new file mode 100644 index 000000000..169ddbc4b --- /dev/null +++ b/quartz/index.ts @@ -0,0 +1,4 @@ +export * as Component from "./components" +export * as Plugin from "./plugins" +export * from "./i18n" +export * from "./cfg" diff --git a/quartz/plugins/emitters/404.tsx b/quartz/plugins/emitters/404.tsx index 2d518b675..bbccf644c 100644 --- a/quartz/plugins/emitters/404.tsx +++ b/quartz/plugins/emitters/404.tsx @@ -4,7 +4,7 @@ import BodyConstructor from "../../components/Body" import { pageResources, renderPage } from "../../components/renderPage" import { FullPageLayout } from "../../cfg" import { FilePath, FullSlug } from "../../util/path" -import { sharedPageComponents } from "../../../quartz.layout" +import { sharedPageComponents } from "$layout" import { NotFound } from "../../components" import { defaultProcessedContent } from "../vfile" import { write } from "./helpers" diff --git a/quartz/plugins/emitters/componentResources.ts b/quartz/plugins/emitters/componentResources.ts index 08278305e..6e77fab94 100644 --- a/quartz/plugins/emitters/componentResources.ts +++ b/quartz/plugins/emitters/componentResources.ts @@ -14,6 +14,7 @@ import { Features, transform } from "lightningcss" import { transform as transpile } from "esbuild" import { write } from "./helpers" import DepGraph from "../../depgraph" +import path from "path" type ComponentResources = { css: string[] @@ -183,8 +184,13 @@ export const ComponentResources: QuartzEmitterPlugin = () => { getQuartzComponents() { return [] }, - async getDependencyGraph(_ctx, _content, _resources) { - return new DepGraph() + async getDependencyGraph(ctx, _content, _resources) { + const graph = new DepGraph() + graph.addEdge( + path.join(ctx.argv.output, "index.css") as FilePath, + path.join(process.cwd(), "styles.scss") as FilePath, + ) + return graph }, async emit(ctx, _content, _resources): Promise { const promises: Promise[] = [] @@ -245,6 +251,7 @@ export const ComponentResources: QuartzEmitterPlugin = () => { googleFontsStyleSheet, ...componentResources.css, styles, + await import("$styles").then((s) => s.default ?? s).catch(() => ""), ) const [prescript, postscript] = await Promise.all([ joinScripts(componentResources.beforeDOMLoaded), diff --git a/quartz/plugins/emitters/contentPage.tsx b/quartz/plugins/emitters/contentPage.tsx index 8788f331d..439933f80 100644 --- a/quartz/plugins/emitters/contentPage.tsx +++ b/quartz/plugins/emitters/contentPage.tsx @@ -10,7 +10,7 @@ import { pageResources, renderPage } from "../../components/renderPage" import { FullPageLayout } from "../../cfg" import { Argv } from "../../util/ctx" import { FilePath, isRelativeURL, joinSegments, pathToRoot } from "../../util/path" -import { defaultContentPageLayout, sharedPageComponents } from "../../../quartz.layout" +import { defaultContentPageLayout, sharedPageComponents } from "$layout" import { Content } from "../../components" import chalk from "chalk" import { write } from "./helpers" diff --git a/quartz/plugins/emitters/folderPage.tsx b/quartz/plugins/emitters/folderPage.tsx index bafaec916..1ed207260 100644 --- a/quartz/plugins/emitters/folderPage.tsx +++ b/quartz/plugins/emitters/folderPage.tsx @@ -15,7 +15,7 @@ import { pathToRoot, simplifySlug, } from "../../util/path" -import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.layout" +import { defaultListPageLayout, sharedPageComponents } from "$layout" import { FolderContent } from "../../components" import { write } from "./helpers" import { i18n } from "../../i18n" diff --git a/quartz/plugins/emitters/static.ts b/quartz/plugins/emitters/static.ts index c52c62879..a5a791426 100644 --- a/quartz/plugins/emitters/static.ts +++ b/quartz/plugins/emitters/static.ts @@ -1,4 +1,4 @@ -import { FilePath, QUARTZ, joinSegments } from "../../util/path" +import { FilePath, joinSegments } from "../../util/path" import { QuartzEmitterPlugin } from "../types" import fs from "fs" import { glob } from "../../util/glob" @@ -9,10 +9,10 @@ export const Static: QuartzEmitterPlugin = () => ({ getQuartzComponents() { return [] }, - async getDependencyGraph({ argv, cfg }, _content, _resources) { + async getDependencyGraph({ argv, cfg, quartzRoot }, _content, _resources) { const graph = new DepGraph() - const staticPath = joinSegments(QUARTZ, "static") + const staticPath = joinSegments(quartzRoot, "static") const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns) for (const fp of fps) { graph.addEdge( @@ -23,8 +23,8 @@ export const Static: QuartzEmitterPlugin = () => ({ return graph }, - async emit({ argv, cfg }, _content, _resources): Promise { - const staticPath = joinSegments(QUARTZ, "static") + async emit({ argv, cfg, quartzRoot }, _content, _resources): Promise { + const staticPath = joinSegments(quartzRoot, "static") const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns) await fs.promises.cp(staticPath, joinSegments(argv.output, "static"), { recursive: true, diff --git a/quartz/plugins/emitters/tagPage.tsx b/quartz/plugins/emitters/tagPage.tsx index 9913e7d82..1fed141f6 100644 --- a/quartz/plugins/emitters/tagPage.tsx +++ b/quartz/plugins/emitters/tagPage.tsx @@ -12,7 +12,7 @@ import { joinSegments, pathToRoot, } from "../../util/path" -import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.layout" +import { defaultListPageLayout, sharedPageComponents } from "$layout" import { TagContent } from "../../components" import { write } from "./helpers" import { i18n } from "../../i18n" diff --git a/quartz/processors/parse.ts b/quartz/processors/parse.ts index 479313f49..e5dd14de3 100644 --- a/quartz/processors/parse.ts +++ b/quartz/processors/parse.ts @@ -7,7 +7,7 @@ import { Root as HTMLRoot } from "hast" import { MarkdownContent, ProcessedContent } from "../plugins/vfile" import { PerfTimer } from "../util/perf" import { read } from "to-vfile" -import { FilePath, FullSlug, QUARTZ, slugifyFilePath } from "../util/path" +import { FilePath, FullSlug, slugifyFilePath } from "../util/path" import path from "path" import workerpool, { Promise as WorkerPromise } from "workerpool" import { QuartzLogger } from "../util/log" @@ -49,20 +49,28 @@ function* chunks(arr: T[], n: number) { } } -async function transpileWorkerScript() { - // transpile worker script - const cacheFile = "./.quartz-cache/transpiled-worker.mjs" - const fp = "./quartz/worker.ts" - return esbuild.build({ +async function transpileWorkerScript(ctx: BuildCtx): Promise { + // import.meta.dirname is the cache folder, because we're in transpiled-build.mjs atm technically + const cacheFile = path.join(import.meta.dirname, "transpiled-worker.mjs") + const fp = path.join(ctx.quartzRoot, "worker.ts") + await esbuild.build({ entryPoints: [fp], - outfile: path.join(QUARTZ, cacheFile), + outfile: cacheFile, bundle: true, keepNames: true, + minifyWhitespace: true, + minifySyntax: true, platform: "node", format: "esm", packages: "external", sourcemap: true, sourcesContent: false, + alias: { + $config: path.join(process.cwd(), "quartz.config.ts"), + $layout: path.join(process.cwd(), "quartz.layout.ts"), + $styles: path.join(process.cwd(), "styles.scss"), + quartz: path.resolve(ctx.quartzRoot, ".."), + }, plugins: [ { name: "css-and-scripts-as-text", @@ -79,6 +87,7 @@ async function transpileWorkerScript() { }, ], }) + return cacheFile } export function createFileParser(ctx: BuildCtx, fps: FilePath[]) { @@ -164,11 +173,12 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise { console.error(`${err}`.replace(/^error:\s*/i, "")) @@ -177,7 +187,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise[] = [] for (const chunk of chunks(fps, CHUNK_SIZE)) { - mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, argv, chunk])) + mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, ctx.quartzRoot, argv, chunk])) } const mdResults: [MarkdownContent[], FullSlug[]][] = await WorkerPromise.all(mdPromises).catch(errorHandler) @@ -187,7 +197,9 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise = string & { __brand: T } diff --git a/quartz/util/sourcemap.ts b/quartz/util/sourcemap.ts index d3b9cf738..333015df8 100644 --- a/quartz/util/sourcemap.ts +++ b/quartz/util/sourcemap.ts @@ -6,7 +6,7 @@ export const options: sourceMapSupport.Options = { // source map hack to get around query param // import cache busting retrieveSourceMap(source) { - if (source.includes(".quartz-cache")) { + if (source.includes("?update")) { let realSource = fileURLToPath(source.split("?", 2)[0] + ".map") return { map: fs.readFileSync(realSource, "utf8"), diff --git a/quartz/worker.ts b/quartz/worker.ts index c9cd98055..d987ca36c 100644 --- a/quartz/worker.ts +++ b/quartz/worker.ts @@ -1,6 +1,5 @@ import sourceMapSupport from "source-map-support" sourceMapSupport.install(options) -import cfg from "../quartz.config" import { Argv, BuildCtx } from "./util/ctx" import { FilePath, FullSlug } from "./util/path" import { @@ -12,9 +11,12 @@ import { import { options } from "./util/sourcemap" import { MarkdownContent, ProcessedContent } from "./plugins/vfile" +import cfg from "$config" + // only called from worker thread export async function parseMarkdown( buildId: string, + quartzRoot: string, argv: Argv, fps: FilePath[], ): Promise<[MarkdownContent[], FullSlug[]]> { @@ -27,6 +29,7 @@ export async function parseMarkdown( cfg, argv, allSlugs, + quartzRoot, } return [await createFileParser(ctx, fps)(createMdProcessor(ctx)), allSlugs] } @@ -34,6 +37,7 @@ export async function parseMarkdown( // only called from worker thread export function processHtml( buildId: string, + quartzRoot: string, argv: Argv, mds: MarkdownContent[], allSlugs: FullSlug[], @@ -43,6 +47,7 @@ export function processHtml( cfg, argv, allSlugs, + quartzRoot, } return createMarkdownParser(ctx, mds)(createHtmlProcessor(ctx)) }