Merge 8651f887d900407a5dc5c08a9f986f5cc34a7643 into dd6bd498db25344b2cccf56abfb656576a496d38

ambient.d.ts

@@ -0,0 +1,40 @@
+declare module "*.scss" {
+  const content: string
+  export = content
+}
+
+declare module "$config" {
+  import { QuartzConfig } from "./quartz"
+
+  const config: QuartzConfig
+  export = config
+}
+
+declare module "$layout" {
+  import { SharedLayout, PageLayout } from "./quartz/cfg"
+
+  export const sharedPageComponents: SharedLayout
+  export const defaultContentPageLayout: PageLayout
+  export const defaultListPageLayout: PageLayout
+}
+
+declare module "$styles" {
+  const content: string
+  export = content
+}
+
+declare module "quartz" {
+  // without this the export below does nothing for some reason
+  // sometimes TS is funn
+  import("./quartz")
+
+  export * from "./quartz"
+}
+
+// dom custom event
+interface CustomEventMap {
+  nav: CustomEvent<{ url: FullSlug }>
+  themechange: CustomEvent<{ theme: "light" | "dark" }>
+}
+
+declare const fetchData: Promise<ContentIndex>

docs/advanced/architecture.md

@@ -17,7 +17,7 @@ This question is best answered by tracing what happens when a user (you!) runs `
       1. A WebSocket server on port 3001 to handle hot-reload signals. This tracks all inbound connections and sends a 'rebuild' message a server-side change is detected (either content or configuration).
       2. An HTTP file-server on a user defined port (normally 8080) to serve the actual website files.
    4. If the `--serve` flag is set, it also starts a file watcher to detect source-code changes (e.g. anything that is `.ts`, `.tsx`, `.scss`, or packager files). On a change, we rebuild the module (step 2 above) using esbuild's [rebuild API](https://esbuild.github.io/api/#rebuild) which drastically reduces the build times.
-   5. After transpiling the main Quartz build module (`quartz/build.ts`), we write it to a cache file `.quartz-cache/transpiled-build.mjs` and then dynamically import this using `await import(cacheFile)`. However, we need to be pretty smart about how to bust Node's [import cache](https://github.com/nodejs/modules/issues/307) so we add a random query string to fake Node into thinking it's a new module. This does, however, cause memory leaks so we just hope that the user doesn't hot-reload their configuration too many times in a single session :)) (it leaks about ~350kB memory on each reload). After importing the module, we then invoke it, passing in the command line arguments we parsed earlier along with a callback function to signal the client to refresh.
+   5. After transpiling the main Quartz build module (`quartz/build.ts`), we write it to a cache file `node_modules/.cache/quartz/transpiled-build.mjs` and then dynamically import this using `await import(cacheFile)`. However, we need to be pretty smart about how to bust Node's [import cache](https://github.com/nodejs/modules/issues/307) so we add a random query string to fake Node into thinking it's a new module. This does, however, cause memory leaks so we just hope that the user doesn't hot-reload their configuration too many times in a single session :)) (it leaks about ~350kB memory on each reload). After importing the module, we then invoke it, passing in the command line arguments we parsed earlier along with a callback function to signal the client to refresh.
 4. In `build.ts`, we start by installing source map support manually to account for the query string cache busting hack we introduced earlier. Then, we start processing content:
    1. Clean the output directory.
    2. Recursively glob all files in the `content` folder, respecting the `.gitignore`.

index.d.ts

@@ -1,12 +0,0 @@
-declare module "*.scss" {
-  const content: string
-  export = content
-}
-
-// dom custom event
-interface CustomEventMap {
-  nav: CustomEvent<{ url: FullSlug }>
-  themechange: CustomEvent<{ theme: "light" | "dark" }>
-}
-
-declare const fetchData: Promise<ContentIndex>

index.ts

@@ -0,0 +1 @@
+export * from "./quartz"

package.json

@@ -34,6 +34,7 @@
   "bin": {
     "quartz": "./quartz/bootstrap-cli.mjs"
   },
+  "types": "./ambient.d.ts",
   "dependencies": {
     "@clack/prompts": "^0.10.0",
     "@floating-ui/dom": "^1.6.13",

quartz.config.ts

@@ -1,5 +1,4 @@
-import { QuartzConfig } from "./quartz/cfg"
+import { QuartzConfig, Plugin } from "quartz"
-import * as Plugin from "./quartz/plugins"
 
 /**
  * Quartz 4.0 Configuration

quartz.layout.ts

@@ -1,5 +1,4 @@
-import { PageLayout, SharedLayout } from "./quartz/cfg"
+import { Component, PageLayout, SharedLayout } from "quartz"
-import * as Component from "./quartz/components"
 
 // components shared across all pages
 export const sharedPageComponents: SharedLayout = {

quartz/bootstrap-worker.mjs

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import workerpool from "workerpool"
-const cacheFile = "./.quartz-cache/transpiled-worker.mjs"
+const cacheFile = process.argv[2]
 const { parseMarkdown, processHtml } = await import(cacheFile)
 workerpool.worker({
   parseMarkdown,

quartz/build.ts

@@ -8,7 +8,7 @@ import chalk from "chalk"
 import { parseMarkdown } from "./processors/parse"
 import { filterContent } from "./processors/filter"
 import { emitContent } from "./processors/emit"
-import cfg from "../quartz.config"
+import cfg from "$config"
 import { FilePath, FullSlug, joinSegments, slugifyFilePath } from "./util/path"
 import chokidar from "chokidar"
 import { ProcessedContent } from "./plugins/vfile"
@@ -42,12 +42,13 @@ function newBuildId() {
   return Math.random().toString(36).substring(2, 8)
 }
 
-async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) {
+async function buildQuartz(quartzRoot: string, argv: Argv, mut: Mutex, clientRefresh: () => void) {
   const ctx: BuildCtx = {
     buildId: newBuildId(),
     argv,
     cfg,
     allSlugs: [],
+    quartzRoot,
   }
 
   const perf = new PerfTimer()
@@ -413,9 +414,9 @@ async function rebuildFromEntrypoint(
   release()
 }
 
-export default async (argv: Argv, mut: Mutex, clientRefresh: () => void) => {
+export default async (quartzRoot: string, argv: Argv, mut: Mutex, clientRefresh: () => void) => {
   try {
-    return await buildQuartz(argv, mut, clientRefresh)
+    return await buildQuartz(quartzRoot, argv, mut, clientRefresh)
   } catch (err) {
     trace("\nExiting Quartz due to a fatal error", err as Error)
   }

quartz/cli/constants.js

@@ -1,5 +1,5 @@
 import path from "path"
-import { readFileSync } from "fs"
+import { accessSync, readFileSync } from "fs"
 
 /**
  * All constants relating to helpers or handlers
@@ -7,9 +7,26 @@ import { readFileSync } from "fs"
 export const ORIGIN_NAME = "origin"
 export const UPSTREAM_NAME = "upstream"
 export const QUARTZ_SOURCE_BRANCH = "v4"
+
 export const cwd = process.cwd()
-export const cacheDir = path.join(cwd, ".quartz-cache")
+
-export const cacheFile = "./quartz/.quartz-cache/transpiled-build.mjs"
+function selectCacheDir() {
-export const fp = "./quartz/build.ts"
+  try {
-export const { version } = JSON.parse(readFileSync("./package.json").toString())
+    const node_modules = path.join(cwd, "node_modules")
+    accessSync(node_modules) // check if node_modules exists
+    return path.join(node_modules, ".cache", "quartz")
+  } catch {
+    // standalone quartz bin?
+    return path.join(cwd, ".quartz-cache")
+  }
+}
+
+export const cacheDir = selectCacheDir()
+export const cacheFile = path.join(cacheDir, "transpiled-build.mjs")
 export const contentCacheFolder = path.join(cacheDir, "content-cache")
+
+export const quartzRoot = path.resolve(import.meta.dirname, "..")
+export const fp = path.join(quartzRoot, "build.ts")
+export const { version } = JSON.parse(
+  readFileSync(path.resolve(quartzRoot, "..", "package.json")).toString(),
+)

quartz/cli/handlers.js

@@ -31,7 +31,9 @@ import {
   fp,
   cacheFile,
   cwd,
+  quartzRoot,
 } from "./constants.js"
+import { pathToFileURL } from "url"
 
 /**
  * Handles `npx quartz create`
@@ -232,6 +234,12 @@ export async function handleBuild(argv) {
     metafile: true,
     sourcemap: true,
     sourcesContent: false,
+    alias: {
+      $config: path.join(cwd, "quartz.config.ts"),
+      $layout: path.join(cwd, "quartz.layout.ts"),
+      $styles: path.join(cwd, "styles.scss"),
+      quartz: path.resolve(quartzRoot, ".."),
+    },
     plugins: [
       sassPlugin({
         type: "css-text",
@@ -303,8 +311,9 @@ export async function handleBuild(argv) {
     release()
 
     if (argv.bundleInfo) {
-      const outputFileName = "quartz/.quartz-cache/transpiled-build.mjs"
+      // metafile.outputs always uses /
-      const meta = result.metafile.outputs[outputFileName]
+      const output = path.relative(cwd, cacheFile).replaceAll("\\", "/")
+      const meta = result.metafile.outputs[output]
       console.log(
         `Successfully transpiled ${Object.keys(meta.inputs).length} files (${prettyBytes(
           meta.bytes,
@@ -313,12 +322,14 @@ export async function handleBuild(argv) {
       console.log(await esbuild.analyzeMetafile(result.metafile, { color: true }))
     }
 
+    // absolute path on windows has to be a file:// url
+    const url = pathToFileURL(cacheFile)
     // bypass module cache
     // https://github.com/nodejs/modules/issues/307
-    const { default: buildQuartz } = await import(`../../${cacheFile}?update=${randomUUID()}`)
+    url.searchParams.set("update", randomUUID())
-    // ^ this import is relative, so base "cacheFile" path can't be used
+    const { default: buildQuartz } = await import(url)
 
-    cleanupBuild = await buildQuartz(argv, buildMutex, clientRefresh)
+    cleanupBuild = await buildQuartz(quartzRoot, argv, buildMutex, clientRefresh)
     clientRefresh()
   }
 

quartz/index.ts

@@ -0,0 +1,4 @@
+export * as Component from "./components"
+export * as Plugin from "./plugins"
+export * from "./i18n"
+export * from "./cfg"

quartz/plugins/emitters/404.tsx

@@ -4,7 +4,7 @@ import BodyConstructor from "../../components/Body"
 import { pageResources, renderPage } from "../../components/renderPage"
 import { FullPageLayout } from "../../cfg"
 import { FilePath, FullSlug } from "../../util/path"
-import { sharedPageComponents } from "../../../quartz.layout"
+import { sharedPageComponents } from "$layout"
 import { NotFound } from "../../components"
 import { defaultProcessedContent } from "../vfile"
 import { write } from "./helpers"

quartz/plugins/emitters/componentResources.ts

@@ -14,6 +14,7 @@ import { Features, transform } from "lightningcss"
 import { transform as transpile } from "esbuild"
 import { write } from "./helpers"
 import DepGraph from "../../depgraph"
+import path from "path"
 
 type ComponentResources = {
   css: string[]
@@ -183,8 +184,13 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
     getQuartzComponents() {
       return []
     },
-    async getDependencyGraph(_ctx, _content, _resources) {
+    async getDependencyGraph(ctx, _content, _resources) {
-      return new DepGraph<FilePath>()
+      const graph = new DepGraph<FilePath>()
+      graph.addEdge(
+        path.join(ctx.argv.output, "index.css") as FilePath,
+        path.join(process.cwd(), "styles.scss") as FilePath,
+      )
+      return graph
     },
     async emit(ctx, _content, _resources): Promise<FilePath[]> {
       const promises: Promise<FilePath>[] = []
@@ -245,6 +251,7 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
         googleFontsStyleSheet,
         ...componentResources.css,
         styles,
+        await import("$styles").then((s) => s.default ?? s).catch(() => ""),
       )
       const [prescript, postscript] = await Promise.all([
         joinScripts(componentResources.beforeDOMLoaded),

quartz/plugins/emitters/contentPage.tsx

@@ -10,7 +10,7 @@ import { pageResources, renderPage } from "../../components/renderPage"
 import { FullPageLayout } from "../../cfg"
 import { Argv } from "../../util/ctx"
 import { FilePath, isRelativeURL, joinSegments, pathToRoot } from "../../util/path"
-import { defaultContentPageLayout, sharedPageComponents } from "../../../quartz.layout"
+import { defaultContentPageLayout, sharedPageComponents } from "$layout"
 import { Content } from "../../components"
 import chalk from "chalk"
 import { write } from "./helpers"

quartz/plugins/emitters/folderPage.tsx

@@ -15,7 +15,7 @@ import {
   pathToRoot,
   simplifySlug,
 } from "../../util/path"
-import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.layout"
+import { defaultListPageLayout, sharedPageComponents } from "$layout"
 import { FolderContent } from "../../components"
 import { write } from "./helpers"
 import { i18n } from "../../i18n"

quartz/plugins/emitters/static.ts

@@ -1,4 +1,4 @@
-import { FilePath, QUARTZ, joinSegments } from "../../util/path"
+import { FilePath, joinSegments } from "../../util/path"
 import { QuartzEmitterPlugin } from "../types"
 import fs from "fs"
 import { glob } from "../../util/glob"
@@ -9,10 +9,10 @@ export const Static: QuartzEmitterPlugin = () => ({
   getQuartzComponents() {
     return []
   },
-  async getDependencyGraph({ argv, cfg }, _content, _resources) {
+  async getDependencyGraph({ argv, cfg, quartzRoot }, _content, _resources) {
     const graph = new DepGraph<FilePath>()
 
-    const staticPath = joinSegments(QUARTZ, "static")
+    const staticPath = joinSegments(quartzRoot, "static")
     const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
     for (const fp of fps) {
       graph.addEdge(
@@ -23,8 +23,8 @@ export const Static: QuartzEmitterPlugin = () => ({
 
     return graph
   },
-  async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
+  async emit({ argv, cfg, quartzRoot }, _content, _resources): Promise<FilePath[]> {
-    const staticPath = joinSegments(QUARTZ, "static")
+    const staticPath = joinSegments(quartzRoot, "static")
     const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
     await fs.promises.cp(staticPath, joinSegments(argv.output, "static"), {
       recursive: true,

quartz/plugins/emitters/tagPage.tsx

@@ -12,7 +12,7 @@ import {
   joinSegments,
   pathToRoot,
 } from "../../util/path"
-import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.layout"
+import { defaultListPageLayout, sharedPageComponents } from "$layout"
 import { TagContent } from "../../components"
 import { write } from "./helpers"
 import { i18n } from "../../i18n"

quartz/processors/parse.ts

@@ -7,7 +7,7 @@ import { Root as HTMLRoot } from "hast"
 import { MarkdownContent, ProcessedContent } from "../plugins/vfile"
 import { PerfTimer } from "../util/perf"
 import { read } from "to-vfile"
-import { FilePath, FullSlug, QUARTZ, slugifyFilePath } from "../util/path"
+import { FilePath, FullSlug, slugifyFilePath } from "../util/path"
 import path from "path"
 import workerpool, { Promise as WorkerPromise } from "workerpool"
 import { QuartzLogger } from "../util/log"
@@ -49,20 +49,28 @@ function* chunks<T>(arr: T[], n: number) {
   }
 }
 
-async function transpileWorkerScript() {
+async function transpileWorkerScript(ctx: BuildCtx): Promise<string> {
-  // transpile worker script
+  // import.meta.dirname is the cache folder, because we're in transpiled-build.mjs atm technically
-  const cacheFile = "./.quartz-cache/transpiled-worker.mjs"
+  const cacheFile = path.join(import.meta.dirname, "transpiled-worker.mjs")
-  const fp = "./quartz/worker.ts"
+  const fp = path.join(ctx.quartzRoot, "worker.ts")
-  return esbuild.build({
+  await esbuild.build({
     entryPoints: [fp],
-    outfile: path.join(QUARTZ, cacheFile),
+    outfile: cacheFile,
     bundle: true,
     keepNames: true,
+    minifyWhitespace: true,
+    minifySyntax: true,
     platform: "node",
     format: "esm",
     packages: "external",
     sourcemap: true,
     sourcesContent: false,
+    alias: {
+      $config: path.join(process.cwd(), "quartz.config.ts"),
+      $layout: path.join(process.cwd(), "quartz.layout.ts"),
+      $styles: path.join(process.cwd(), "styles.scss"),
+      quartz: path.resolve(ctx.quartzRoot, ".."),
+    },
     plugins: [
       {
         name: "css-and-scripts-as-text",
@@ -79,6 +87,7 @@ async function transpileWorkerScript() {
       },
     ],
   })
+  return cacheFile
 }
 
 export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
@@ -164,11 +173,12 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
       throw error
     }
   } else {
-    await transpileWorkerScript()
+    const transpiledWorker = await transpileWorkerScript(ctx)
-    const pool = workerpool.pool("./quartz/bootstrap-worker.mjs", {
+    const pool = workerpool.pool(path.join(ctx.quartzRoot, "bootstrap-worker.mjs"), {
       minWorkers: "max",
       maxWorkers: concurrency,
       workerType: "thread",
+      workerThreadOpts: { argv: [transpiledWorker] },
     })
     const errorHandler = (err: any) => {
       console.error(`${err}`.replace(/^error:\s*/i, ""))
@@ -177,7 +187,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
 
     const mdPromises: WorkerPromise<[MarkdownContent[], FullSlug[]]>[] = []
     for (const chunk of chunks(fps, CHUNK_SIZE)) {
-      mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, argv, chunk]))
+      mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, ctx.quartzRoot, argv, chunk]))
     }
     const mdResults: [MarkdownContent[], FullSlug[]][] =
       await WorkerPromise.all(mdPromises).catch(errorHandler)
@@ -187,7 +197,9 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
       ctx.allSlugs.push(...extraSlugs)
     }
     for (const [mdChunk, _] of mdResults) {
-      childPromises.push(pool.exec("processHtml", [ctx.buildId, argv, mdChunk, ctx.allSlugs]))
+      childPromises.push(
+        pool.exec("processHtml", [ctx.buildId, ctx.quartzRoot, argv, mdChunk, ctx.allSlugs]),
+      )
     }
     const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch(errorHandler)
 

quartz/util/ctx.ts

@@ -18,4 +18,5 @@ export interface BuildCtx {
   argv: Argv
   cfg: QuartzConfig
   allSlugs: FullSlug[]
+  quartzRoot: string
 }

quartz/util/path.ts

@@ -6,8 +6,6 @@ export const clone = rfdc()
 
 // this file must be isomorphic so it can't use node libs (e.g. path)
 
-export const QUARTZ = "quartz"
-
 /// Utility type to simulate nominal types in TypeScript
 type SlugLike<T> = string & { __brand: T }
 

quartz/util/sourcemap.ts

@@ -6,7 +6,7 @@ export const options: sourceMapSupport.Options = {
   // source map hack to get around query param
   // import cache busting
   retrieveSourceMap(source) {
-    if (source.includes(".quartz-cache")) {
+    if (source.includes("?update")) {
       let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
       return {
         map: fs.readFileSync(realSource, "utf8"),

quartz/worker.ts

@@ -1,6 +1,5 @@
 import sourceMapSupport from "source-map-support"
 sourceMapSupport.install(options)
-import cfg from "../quartz.config"
 import { Argv, BuildCtx } from "./util/ctx"
 import { FilePath, FullSlug } from "./util/path"
 import {
@@ -12,9 +11,12 @@ import {
 import { options } from "./util/sourcemap"
 import { MarkdownContent, ProcessedContent } from "./plugins/vfile"
 
+import cfg from "$config"
+
 // only called from worker thread
 export async function parseMarkdown(
   buildId: string,
+  quartzRoot: string,
   argv: Argv,
   fps: FilePath[],
 ): Promise<[MarkdownContent[], FullSlug[]]> {
@@ -27,6 +29,7 @@ export async function parseMarkdown(
     cfg,
     argv,
     allSlugs,
+    quartzRoot,
   }
   return [await createFileParser(ctx, fps)(createMdProcessor(ctx)), allSlugs]
 }
@@ -34,6 +37,7 @@ export async function parseMarkdown(
 // only called from worker thread
 export function processHtml(
   buildId: string,
+  quartzRoot: string,
   argv: Argv,
   mds: MarkdownContent[],
   allSlugs: FullSlug[],
@@ -43,6 +47,7 @@ export function processHtml(
     cfg,
     argv,
     allSlugs,
+    quartzRoot,
   }
   return createMarkdownParser(ctx, mds)(createHtmlProcessor(ctx))
 }