feat: untangle quartz from local configs in least amount of changes

For the current setup where people have to fork or at least clone quartz
this changes nothing - but it allows you to install quartz as a
devDependency via npm and have it actually work.

One real change is switch from `.quartz-cache` to
`node_modules/.cache/quartz` for transpilation results, this is an
artifact from my previous attempts, I guess with this one I can change
it back - but `node_modules/.cache` feels more better imo idk.

edit: OTOH if you want to have quartz be a _completely_ separate binary
(which this also enables I think), having it create a node_modules
folder is weird, so I made a quick hack for that for now.

Example:
```bash
$ mkdir my-repo && cd my-repo
$ npm i quartz@necauqua/quartz#untangled # quartz@ prefix is important
$ cp node_modules/quartz/quartz.*.ts .   # copy the default configs
$ mkdir content && echo "# Hello World!" > content/index.md
$ npx quartz build --serve # this just works!
$ echo 'body { background: red !important; }' > styles.scss
```
Notice how I used my branch in the `npm i` line, ideally it'd be
`npm i quartz@jackyzho0/quartz`, or maybe we can somehow get the quartz
package on npm and it'll just be `npm i quartz`.
In the latter case `npx quartz build` will literally just work without
a local npm package at all?.

Having some support for components and plugins being in separate npm
packages instead of people copying code around is not out of the picture
with this too btw.

Closes #502

MOVE ME

ambient.d.ts

@@ -0,0 +1,40 @@
+declare module "*.scss" {
+  const content: string
+  export = content
+}
+
+declare module "$config" {
+  import { QuartzConfig } from "./quartz"
+
+  const config: QuartzConfig
+  export = config
+}
+
+declare module "$layout" {
+  import { SharedLayout, PageLayout } from "./quartz/cfg"
+
+  export const sharedPageComponents: SharedLayout
+  export const defaultContentPageLayout: PageLayout
+  export const defaultListPageLayout: PageLayout
+}
+
+declare module "$styles" {
+  const content: string
+  export = content
+}
+
+declare module "quartz" {
+  // without this the export below does nothing for some reason
+  // sometimes TS is funn
+  import("./quartz")
+
+  export * from "./quartz"
+}
+
+// dom custom event
+interface CustomEventMap {
+  nav: CustomEvent<{ url: FullSlug }>
+  themechange: CustomEvent<{ theme: "light" | "dark" }>
+}
+
+declare const fetchData: Promise<ContentIndex>

docs/advanced/architecture.md

@@ -17,7 +17,7 @@ This question is best answered by tracing what happens when a user (you!) runs `
       1. A WebSocket server on port 3001 to handle hot-reload signals. This tracks all inbound connections and sends a 'rebuild' message a server-side change is detected (either content or configuration).
       2. An HTTP file-server on a user defined port (normally 8080) to serve the actual website files.
    4. If the `--serve` flag is set, it also starts a file watcher to detect source-code changes (e.g. anything that is `.ts`, `.tsx`, `.scss`, or packager files). On a change, we rebuild the module (step 2 above) using esbuild's [rebuild API](https://esbuild.github.io/api/#rebuild) which drastically reduces the build times.
-   5. After transpiling the main Quartz build module (`quartz/build.ts`), we write it to a cache file `.quartz-cache/transpiled-build.mjs` and then dynamically import this using `await import(cacheFile)`. However, we need to be pretty smart about how to bust Node's [import cache](https://github.com/nodejs/modules/issues/307) so we add a random query string to fake Node into thinking it's a new module. This does, however, cause memory leaks so we just hope that the user doesn't hot-reload their configuration too many times in a single session :)) (it leaks about ~350kB memory on each reload). After importing the module, we then invoke it, passing in the command line arguments we parsed earlier along with a callback function to signal the client to refresh.
+   5. After transpiling the main Quartz build module (`quartz/build.ts`), we write it to a cache file `node_modules/.cache/quartz/transpiled-build.mjs` and then dynamically import this using `await import(cacheFile)`. However, we need to be pretty smart about how to bust Node's [import cache](https://github.com/nodejs/modules/issues/307) so we add a random query string to fake Node into thinking it's a new module. This does, however, cause memory leaks so we just hope that the user doesn't hot-reload their configuration too many times in a single session :)) (it leaks about ~350kB memory on each reload). After importing the module, we then invoke it, passing in the command line arguments we parsed earlier along with a callback function to signal the client to refresh.
 4. In `build.ts`, we start by installing source map support manually to account for the query string cache busting hack we introduced earlier. Then, we start processing content:
    1. Clean the output directory.
    2. Recursively glob all files in the `content` folder, respecting the `.gitignore`.

index.d.ts

@@ -1,12 +0,0 @@
-declare module "*.scss" {
-  const content: string
-  export = content
-}
-
-// dom custom event
-interface CustomEventMap {
-  nav: CustomEvent<{ url: FullSlug }>
-  themechange: CustomEvent<{ theme: "light" | "dark" }>
-}
-
-declare const fetchData: Promise<ContentIndex>

index.ts

@@ -0,0 +1 @@
+export * from "./quartz"

package.json

@@ -34,6 +34,7 @@
   "bin": {
     "quartz": "./quartz/bootstrap-cli.mjs"
   },
+  "types": "./ambient.d.ts",
   "dependencies": {
     "@clack/prompts": "^0.10.0",
     "@floating-ui/dom": "^1.6.13",

quartz.config.ts

@@ -1,5 +1,4 @@
-import { QuartzConfig } from "./quartz/cfg"
-import * as Plugin from "./quartz/plugins"
+import { QuartzConfig, Plugin } from "quartz"
 
 /**
  * Quartz 4.0 Configuration

quartz.layout.ts

@@ -1,5 +1,4 @@
-import { PageLayout, SharedLayout } from "./quartz/cfg"
-import * as Component from "./quartz/components"
+import { Component, PageLayout, SharedLayout } from "quartz"
 
 // components shared across all pages
 export const sharedPageComponents: SharedLayout = {

quartz/bootstrap-worker.mjs

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import workerpool from "workerpool"
-const cacheFile = "./.quartz-cache/transpiled-worker.mjs"
+const cacheFile = process.argv[2]
 const { parseMarkdown, processHtml } = await import(cacheFile)
 workerpool.worker({
   parseMarkdown,

quartz/build.ts

@@ -8,7 +8,7 @@ import chalk from "chalk"
 import { parseMarkdown } from "./processors/parse"
 import { filterContent } from "./processors/filter"
 import { emitContent } from "./processors/emit"
-import cfg from "../quartz.config"
+import cfg from "$config"
 import { FilePath, FullSlug, joinSegments, slugifyFilePath } from "./util/path"
 import chokidar from "chokidar"
 import { ProcessedContent } from "./plugins/vfile"
@@ -42,12 +42,13 @@ function newBuildId() {
   return Math.random().toString(36).substring(2, 8)
 }
 
-async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) {
+async function buildQuartz(quartzRoot: string, argv: Argv, mut: Mutex, clientRefresh: () => void) {
   const ctx: BuildCtx = {
     buildId: newBuildId(),
     argv,
     cfg,
     allSlugs: [],
+    quartzRoot,
   }
 
   const perf = new PerfTimer()
@@ -413,9 +414,9 @@ async function rebuildFromEntrypoint(
   release()
 }
 
-export default async (argv: Argv, mut: Mutex, clientRefresh: () => void) => {
+export default async (quartzRoot: string, argv: Argv, mut: Mutex, clientRefresh: () => void) => {
   try {
-    return await buildQuartz(argv, mut, clientRefresh)
+    return await buildQuartz(quartzRoot, argv, mut, clientRefresh)
   } catch (err) {
     trace("\nExiting Quartz due to a fatal error", err as Error)
   }

quartz/cli/constants.js

@@ -1,5 +1,5 @@
 import path from "path"
-import { readFileSync } from "fs"
+import { accessSync, readFileSync } from "fs"
 
 /**
  * All constants relating to helpers or handlers
@@ -7,9 +7,26 @@ import { readFileSync } from "fs"
 export const ORIGIN_NAME = "origin"
 export const UPSTREAM_NAME = "upstream"
 export const QUARTZ_SOURCE_BRANCH = "v4"
+
 export const cwd = process.cwd()
-export const cacheDir = path.join(cwd, ".quartz-cache")
-export const cacheFile = "./quartz/.quartz-cache/transpiled-build.mjs"
-export const fp = "./quartz/build.ts"
-export const { version } = JSON.parse(readFileSync("./package.json").toString())
+
+function selectCacheDir() {
+  try {
+    const node_modules = path.join(cwd, "node_modules")
+    accessSync(node_modules) // check if node_modules exists
+    return path.join(node_modules, ".cache", "quartz")
+  } catch {
+    // standalone quartz bin?
+    return path.join(cwd, ".quartz-cache")
+  }
+}
+
+export const cacheDir = selectCacheDir()
+export const cacheFile = path.join(cacheDir, "transpiled-build.mjs")
 export const contentCacheFolder = path.join(cacheDir, "content-cache")
+
+export const quartzRoot = path.resolve(import.meta.dirname, "..")
+export const fp = path.join(quartzRoot, "build.ts")
+export const { version } = JSON.parse(
+  readFileSync(path.resolve(quartzRoot, "..", "package.json")).toString(),
+)

quartz/cli/handlers.js

@@ -31,7 +31,9 @@ import {
   fp,
   cacheFile,
   cwd,
+  quartzRoot,
 } from "./constants.js"
+import { pathToFileURL } from "url"
 
 /**
  * Handles `npx quartz create`
@@ -232,6 +234,12 @@ export async function handleBuild(argv) {
     metafile: true,
     sourcemap: true,
     sourcesContent: false,
+    alias: {
+      $config: path.join(cwd, "quartz.config.ts"),
+      $layout: path.join(cwd, "quartz.layout.ts"),
+      $styles: path.join(cwd, "styles.scss"),
+      quartz: path.resolve(quartzRoot, ".."),
+    },
     plugins: [
       sassPlugin({
         type: "css-text",
@@ -303,8 +311,9 @@ export async function handleBuild(argv) {
     release()
 
     if (argv.bundleInfo) {
-      const outputFileName = "quartz/.quartz-cache/transpiled-build.mjs"
-      const meta = result.metafile.outputs[outputFileName]
+      // metafile.outputs always uses /
+      const output = path.relative(cwd, cacheFile).replaceAll("\\", "/")
+      const meta = result.metafile.outputs[output]
       console.log(
         `Successfully transpiled ${Object.keys(meta.inputs).length} files (${prettyBytes(
           meta.bytes,
@@ -313,12 +322,14 @@ export async function handleBuild(argv) {
       console.log(await esbuild.analyzeMetafile(result.metafile, { color: true }))
     }
 
+    // absolute path on windows has to be a file:// url
+    const url = pathToFileURL(cacheFile)
     // bypass module cache
     // https://github.com/nodejs/modules/issues/307
-    const { default: buildQuartz } = await import(`../../${cacheFile}?update=${randomUUID()}`)
-    // ^ this import is relative, so base "cacheFile" path can't be used
+    url.searchParams.set("update", randomUUID())
+    const { default: buildQuartz } = await import(url)
 
-    cleanupBuild = await buildQuartz(argv, buildMutex, clientRefresh)
+    cleanupBuild = await buildQuartz(quartzRoot, argv, buildMutex, clientRefresh)
     clientRefresh()
   }
 

quartz/index.ts

@@ -0,0 +1,4 @@
+export * as Component from "./components"
+export * as Plugin from "./plugins"
+export * from "./i18n"
+export * from "./cfg"

quartz/plugins/emitters/404.tsx

@@ -4,7 +4,7 @@ import BodyConstructor from "../../components/Body"
 import { pageResources, renderPage } from "../../components/renderPage"
 import { FullPageLayout } from "../../cfg"
 import { FilePath, FullSlug } from "../../util/path"
-import { sharedPageComponents } from "../../../quartz.layout"
+import { sharedPageComponents } from "$layout"
 import { NotFound } from "../../components"
 import { defaultProcessedContent } from "../vfile"
 import { write } from "./helpers"

quartz/plugins/emitters/componentResources.ts

@@ -14,6 +14,7 @@ import { Features, transform } from "lightningcss"
 import { transform as transpile } from "esbuild"
 import { write } from "./helpers"
 import DepGraph from "../../depgraph"
+import path from "path"
 
 type ComponentResources = {
   css: string[]
@@ -183,8 +184,13 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
     getQuartzComponents() {
       return []
     },
-    async getDependencyGraph(_ctx, _content, _resources) {
-      return new DepGraph<FilePath>()
+    async getDependencyGraph(ctx, _content, _resources) {
+      const graph = new DepGraph<FilePath>()
+      graph.addEdge(
+        path.join(ctx.argv.output, "index.css") as FilePath,
+        path.join(process.cwd(), "styles.scss") as FilePath,
+      )
+      return graph
     },
     async emit(ctx, _content, _resources): Promise<FilePath[]> {
       const promises: Promise<FilePath>[] = []
@@ -245,6 +251,7 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
         googleFontsStyleSheet,
         ...componentResources.css,
         styles,
+        await import("$styles").then((s) => s.default ?? s).catch(() => ""),
       )
       const [prescript, postscript] = await Promise.all([
         joinScripts(componentResources.beforeDOMLoaded),

quartz/plugins/emitters/contentPage.tsx

@@ -10,7 +10,7 @@ import { pageResources, renderPage } from "../../components/renderPage"
 import { FullPageLayout } from "../../cfg"
 import { Argv } from "../../util/ctx"
 import { FilePath, isRelativeURL, joinSegments, pathToRoot } from "../../util/path"
-import { defaultContentPageLayout, sharedPageComponents } from "../../../quartz.layout"
+import { defaultContentPageLayout, sharedPageComponents } from "$layout"
 import { Content } from "../../components"
 import chalk from "chalk"
 import { write } from "./helpers"

quartz/plugins/emitters/folderPage.tsx

@@ -15,7 +15,7 @@ import {
   pathToRoot,
   simplifySlug,
 } from "../../util/path"
-import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.layout"
+import { defaultListPageLayout, sharedPageComponents } from "$layout"
 import { FolderContent } from "../../components"
 import { write } from "./helpers"
 import { i18n } from "../../i18n"

quartz/plugins/emitters/static.ts

@@ -1,4 +1,4 @@
-import { FilePath, QUARTZ, joinSegments } from "../../util/path"
+import { FilePath, joinSegments } from "../../util/path"
 import { QuartzEmitterPlugin } from "../types"
 import fs from "fs"
 import { glob } from "../../util/glob"
@@ -9,10 +9,10 @@ export const Static: QuartzEmitterPlugin = () => ({
   getQuartzComponents() {
     return []
   },
-  async getDependencyGraph({ argv, cfg }, _content, _resources) {
+  async getDependencyGraph({ argv, cfg, quartzRoot }, _content, _resources) {
     const graph = new DepGraph<FilePath>()
 
-    const staticPath = joinSegments(QUARTZ, "static")
+    const staticPath = joinSegments(quartzRoot, "static")
     const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
     for (const fp of fps) {
       graph.addEdge(
@@ -23,8 +23,8 @@ export const Static: QuartzEmitterPlugin = () => ({
 
     return graph
   },
-  async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> {
-    const staticPath = joinSegments(QUARTZ, "static")
+  async emit({ argv, cfg, quartzRoot }, _content, _resources): Promise<FilePath[]> {
+    const staticPath = joinSegments(quartzRoot, "static")
     const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
     await fs.promises.cp(staticPath, joinSegments(argv.output, "static"), {
       recursive: true,

quartz/plugins/emitters/tagPage.tsx

@@ -12,7 +12,7 @@ import {
   joinSegments,
   pathToRoot,
 } from "../../util/path"
-import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.layout"
+import { defaultListPageLayout, sharedPageComponents } from "$layout"
 import { TagContent } from "../../components"
 import { write } from "./helpers"
 import { i18n } from "../../i18n"

quartz/processors/parse.ts

@@ -7,7 +7,7 @@ import { Root as HTMLRoot } from "hast"
 import { MarkdownContent, ProcessedContent } from "../plugins/vfile"
 import { PerfTimer } from "../util/perf"
 import { read } from "to-vfile"
-import { FilePath, FullSlug, QUARTZ, slugifyFilePath } from "../util/path"
+import { FilePath, FullSlug, slugifyFilePath } from "../util/path"
 import path from "path"
 import workerpool, { Promise as WorkerPromise } from "workerpool"
 import { QuartzLogger } from "../util/log"
@@ -49,20 +49,28 @@ function* chunks<T>(arr: T[], n: number) {
   }
 }
 
-async function transpileWorkerScript() {
-  // transpile worker script
-  const cacheFile = "./.quartz-cache/transpiled-worker.mjs"
-  const fp = "./quartz/worker.ts"
-  return esbuild.build({
+async function transpileWorkerScript(ctx: BuildCtx): Promise<string> {
+  // import.meta.dirname is the cache folder, because we're in transpiled-build.mjs atm technically
+  const cacheFile = path.join(import.meta.dirname, "transpiled-worker.mjs")
+  const fp = path.join(ctx.quartzRoot, "worker.ts")
+  await esbuild.build({
     entryPoints: [fp],
-    outfile: path.join(QUARTZ, cacheFile),
+    outfile: cacheFile,
     bundle: true,
     keepNames: true,
+    minifyWhitespace: true,
+    minifySyntax: true,
     platform: "node",
     format: "esm",
     packages: "external",
     sourcemap: true,
     sourcesContent: false,
+    alias: {
+      $config: path.join(process.cwd(), "quartz.config.ts"),
+      $layout: path.join(process.cwd(), "quartz.layout.ts"),
+      $styles: path.join(process.cwd(), "styles.scss"),
+      quartz: path.resolve(ctx.quartzRoot, ".."),
+    },
     plugins: [
       {
         name: "css-and-scripts-as-text",
@@ -79,6 +87,7 @@ async function transpileWorkerScript() {
       },
     ],
   })
+  return cacheFile
 }
 
 export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
@@ -164,11 +173,12 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
       throw error
     }
   } else {
-    await transpileWorkerScript()
-    const pool = workerpool.pool("./quartz/bootstrap-worker.mjs", {
+    const transpiledWorker = await transpileWorkerScript(ctx)
+    const pool = workerpool.pool(path.join(ctx.quartzRoot, "bootstrap-worker.mjs"), {
       minWorkers: "max",
       maxWorkers: concurrency,
       workerType: "thread",
+      workerThreadOpts: { argv: [transpiledWorker] },
     })
     const errorHandler = (err: any) => {
       console.error(`${err}`.replace(/^error:\s*/i, ""))
@@ -177,7 +187,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
 
     const mdPromises: WorkerPromise<[MarkdownContent[], FullSlug[]]>[] = []
     for (const chunk of chunks(fps, CHUNK_SIZE)) {
-      mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, argv, chunk]))
+      mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, ctx.quartzRoot, argv, chunk]))
     }
     const mdResults: [MarkdownContent[], FullSlug[]][] =
       await WorkerPromise.all(mdPromises).catch(errorHandler)
@@ -187,7 +197,9 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
       ctx.allSlugs.push(...extraSlugs)
     }
     for (const [mdChunk, _] of mdResults) {
-      childPromises.push(pool.exec("processHtml", [ctx.buildId, argv, mdChunk, ctx.allSlugs]))
+      childPromises.push(
+        pool.exec("processHtml", [ctx.buildId, ctx.quartzRoot, argv, mdChunk, ctx.allSlugs]),
+      )
     }
     const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch(errorHandler)
 

quartz/util/ctx.ts

@@ -18,4 +18,5 @@ export interface BuildCtx {
   argv: Argv
   cfg: QuartzConfig
   allSlugs: FullSlug[]
+  quartzRoot: string
 }

quartz/util/path.ts

@@ -6,8 +6,6 @@ export const clone = rfdc()
 
 // this file must be isomorphic so it can't use node libs (e.g. path)
 
-export const QUARTZ = "quartz"
-
 /// Utility type to simulate nominal types in TypeScript
 type SlugLike<T> = string & { __brand: T }
 

quartz/util/sourcemap.ts

@@ -6,7 +6,7 @@ export const options: sourceMapSupport.Options = {
   // source map hack to get around query param
   // import cache busting
   retrieveSourceMap(source) {
-    if (source.includes(".quartz-cache")) {
+    if (source.includes("?update")) {
       let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
       return {
         map: fs.readFileSync(realSource, "utf8"),

quartz/worker.ts

@@ -1,6 +1,5 @@
 import sourceMapSupport from "source-map-support"
 sourceMapSupport.install(options)
-import cfg from "../quartz.config"
 import { Argv, BuildCtx } from "./util/ctx"
 import { FilePath, FullSlug } from "./util/path"
 import {
@@ -12,9 +11,12 @@ import {
 import { options } from "./util/sourcemap"
 import { MarkdownContent, ProcessedContent } from "./plugins/vfile"
 
+import cfg from "$config"
+
 // only called from worker thread
 export async function parseMarkdown(
   buildId: string,
+  quartzRoot: string,
   argv: Argv,
   fps: FilePath[],
 ): Promise<[MarkdownContent[], FullSlug[]]> {
@@ -27,6 +29,7 @@ export async function parseMarkdown(
     cfg,
     argv,
     allSlugs,
+    quartzRoot,
   }
   return [await createFileParser(ctx, fps)(createMdProcessor(ctx)), allSlugs]
 }
@@ -34,6 +37,7 @@ export async function parseMarkdown(
 // only called from worker thread
 export function processHtml(
   buildId: string,
+  quartzRoot: string,
   argv: Argv,
   mds: MarkdownContent[],
   allSlugs: FullSlug[],
@@ -43,6 +47,7 @@ export function processHtml(
     cfg,
     argv,
     allSlugs,
+    quartzRoot,
   }
   return createMarkdownParser(ctx, mds)(createHtmlProcessor(ctx))
 }