Compare commits

...

3 Commits

Author SHA1 Message Date
Anton Bulakh
95d57ad29e
Merge 2e299c67ccbd48552273b9bae1d4e9afecbd0715 into 7be47742a6dc86f22d148ca9d304f7a9eea318cf 2025-01-31 06:46:48 -05:00
dependabot[bot]
7be47742a6
chore(deps): bump the production-dependencies group across 1 directory with 3 updates (#1744)
Some checks failed
Build and Test / build-and-test (macos-latest) (push) Has been cancelled
Build and Test / build-and-test (ubuntu-latest) (push) Has been cancelled
Build and Test / build-and-test (windows-latest) (push) Has been cancelled
Build and Test / publish-tag (push) Has been cancelled
Docker build & push image / build (push) Has been cancelled
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-01-31 06:46:45 -05:00
Anton Bulakh
2e299c67cc
feat: untangle quartz from local configs in least amount of changes
For the current setup where people have to fork or at least clone quartz
this changes nothing - but it allows you to install quartz as a
devDependency via npm and have it actually work.

One real change is switch from `.quartz-cache` to
`node_modules/.cache/quartz` for transpilation results, this is an
artifact from my previous attempts, I guess with this one I can change
it back - but `node_modules/.cache` feels more better imo idk.

edit: OTOH if you want to have quartz be a _completely_ separate binary
(which this also enables I think), having it create a node_modules
folder is weird, so I made a quick hack for that for now.

Example:
```bash
$ mkdir my-repo && cd my-repo
$ npm i quartz@necauqua/quartz#untangled # quartz@ prefix is important
$ cp node_modules/quartz/quartz.*.ts .   # copy the default configs
$ mkdir content && echo "# Hello World!" > content/index.md
$ npx quartz build --serve # this just works!
$ echo 'body { background: red !important; }' > styles.scss
```
Notice how I used my branch in the `npm i` line, ideally it'd be
`npm i quartz@jackyzho0/quartz`, or maybe we can somehow get the quartz
package on npm and it'll just be `npm i quartz`.
In the latter case `npx quartz build` will literally just work without
a local npm package at all?.

Having some support for components and plugins being in separate npm
packages instead of people copying code around is not out of the picture
with this too btw.

Closes #502

MOVE ME
2025-01-23 20:53:58 +02:00
24 changed files with 160 additions and 73 deletions

40
ambient.d.ts vendored Normal file
View File

@ -0,0 +1,40 @@
declare module "*.scss" {
const content: string
export = content
}
declare module "$config" {
import { QuartzConfig } from "./quartz"
const config: QuartzConfig
export = config
}
declare module "$layout" {
import { SharedLayout, PageLayout } from "./quartz/cfg"
export const sharedPageComponents: SharedLayout
export const defaultContentPageLayout: PageLayout
export const defaultListPageLayout: PageLayout
}
declare module "$styles" {
const content: string
export = content
}
declare module "quartz" {
// without this the export below does nothing for some reason
// sometimes TS is funn
import("./quartz")
export * from "./quartz"
}
// dom custom event
interface CustomEventMap {
nav: CustomEvent<{ url: FullSlug }>
themechange: CustomEvent<{ theme: "light" | "dark" }>
}
declare const fetchData: Promise<ContentIndex>

View File

@ -17,7 +17,7 @@ This question is best answered by tracing what happens when a user (you!) runs `
1. A WebSocket server on port 3001 to handle hot-reload signals. This tracks all inbound connections and sends a 'rebuild' message a server-side change is detected (either content or configuration). 1. A WebSocket server on port 3001 to handle hot-reload signals. This tracks all inbound connections and sends a 'rebuild' message a server-side change is detected (either content or configuration).
2. An HTTP file-server on a user defined port (normally 8080) to serve the actual website files. 2. An HTTP file-server on a user defined port (normally 8080) to serve the actual website files.
4. If the `--serve` flag is set, it also starts a file watcher to detect source-code changes (e.g. anything that is `.ts`, `.tsx`, `.scss`, or packager files). On a change, we rebuild the module (step 2 above) using esbuild's [rebuild API](https://esbuild.github.io/api/#rebuild) which drastically reduces the build times. 4. If the `--serve` flag is set, it also starts a file watcher to detect source-code changes (e.g. anything that is `.ts`, `.tsx`, `.scss`, or packager files). On a change, we rebuild the module (step 2 above) using esbuild's [rebuild API](https://esbuild.github.io/api/#rebuild) which drastically reduces the build times.
5. After transpiling the main Quartz build module (`quartz/build.ts`), we write it to a cache file `.quartz-cache/transpiled-build.mjs` and then dynamically import this using `await import(cacheFile)`. However, we need to be pretty smart about how to bust Node's [import cache](https://github.com/nodejs/modules/issues/307) so we add a random query string to fake Node into thinking it's a new module. This does, however, cause memory leaks so we just hope that the user doesn't hot-reload their configuration too many times in a single session :)) (it leaks about ~350kB memory on each reload). After importing the module, we then invoke it, passing in the command line arguments we parsed earlier along with a callback function to signal the client to refresh. 5. After transpiling the main Quartz build module (`quartz/build.ts`), we write it to a cache file `node_modules/.cache/quartz/transpiled-build.mjs` and then dynamically import this using `await import(cacheFile)`. However, we need to be pretty smart about how to bust Node's [import cache](https://github.com/nodejs/modules/issues/307) so we add a random query string to fake Node into thinking it's a new module. This does, however, cause memory leaks so we just hope that the user doesn't hot-reload their configuration too many times in a single session :)) (it leaks about ~350kB memory on each reload). After importing the module, we then invoke it, passing in the command line arguments we parsed earlier along with a callback function to signal the client to refresh.
4. In `build.ts`, we start by installing source map support manually to account for the query string cache busting hack we introduced earlier. Then, we start processing content: 4. In `build.ts`, we start by installing source map support manually to account for the query string cache busting hack we introduced earlier. Then, we start processing content:
1. Clean the output directory. 1. Clean the output directory.
2. Recursively glob all files in the `content` folder, respecting the `.gitignore`. 2. Recursively glob all files in the `content` folder, respecting the `.gitignore`.

12
index.d.ts vendored
View File

@ -1,12 +0,0 @@
declare module "*.scss" {
const content: string
export = content
}
// dom custom event
interface CustomEventMap {
nav: CustomEvent<{ url: FullSlug }>
themechange: CustomEvent<{ theme: "light" | "dark" }>
}
declare const fetchData: Promise<ContentIndex>

1
index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./quartz"

27
package-lock.json generated
View File

@ -34,7 +34,7 @@
"mdast-util-to-hast": "^13.2.0", "mdast-util-to-hast": "^13.2.0",
"mdast-util-to-string": "^4.0.0", "mdast-util-to-string": "^4.0.0",
"micromorph": "^0.4.5", "micromorph": "^0.4.5",
"pixi.js": "^8.6.6", "pixi.js": "^8.7.3",
"preact": "^10.25.4", "preact": "^10.25.4",
"preact-render-to-string": "^6.5.13", "preact-render-to-string": "^6.5.13",
"pretty-bytes": "^6.1.1", "pretty-bytes": "^6.1.1",
@ -79,10 +79,10 @@
"@types/d3": "^7.4.3", "@types/d3": "^7.4.3",
"@types/hast": "^3.0.4", "@types/hast": "^3.0.4",
"@types/js-yaml": "^4.0.9", "@types/js-yaml": "^4.0.9",
"@types/node": "^22.10.6", "@types/node": "^22.12.0",
"@types/pretty-time": "^1.1.5", "@types/pretty-time": "^1.1.5",
"@types/source-map-support": "^0.5.10", "@types/source-map-support": "^0.5.10",
"@types/ws": "^8.5.13", "@types/ws": "^8.5.14",
"@types/yargs": "^17.0.33", "@types/yargs": "^17.0.33",
"esbuild": "^0.24.2", "esbuild": "^0.24.2",
"prettier": "^3.4.2", "prettier": "^3.4.2",
@ -1914,10 +1914,11 @@
} }
}, },
"node_modules/@types/node": { "node_modules/@types/node": {
"version": "22.10.6", "version": "22.12.0",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.6.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.12.0.tgz",
"integrity": "sha512-qNiuwC4ZDAUNcY47xgaSuS92cjf8JbSUoaKS77bmLG1rU7MlATVSiw/IlrjtIyyskXBZ8KkNfjK/P5na7rgXbQ==", "integrity": "sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==",
"dev": true, "dev": true,
"license": "MIT",
"dependencies": { "dependencies": {
"undici-types": "~6.20.0" "undici-types": "~6.20.0"
} }
@ -1943,10 +1944,11 @@
"integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==" "integrity": "sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ=="
}, },
"node_modules/@types/ws": { "node_modules/@types/ws": {
"version": "8.5.13", "version": "8.5.14",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.13.tgz", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.14.tgz",
"integrity": "sha512-osM/gWBTPKgHV8XkTunnegTRIsvF6owmf5w+JtAfOw472dptdm0dlGv4xCt6GwQRcC2XVOvvRE/0bAoQcL2QkA==", "integrity": "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw==",
"dev": true, "dev": true,
"license": "MIT",
"dependencies": { "dependencies": {
"@types/node": "*" "@types/node": "*"
} }
@ -5583,9 +5585,10 @@
} }
}, },
"node_modules/pixi.js": { "node_modules/pixi.js": {
"version": "8.6.6", "version": "8.7.3",
"resolved": "https://registry.npmjs.org/pixi.js/-/pixi.js-8.6.6.tgz", "resolved": "https://registry.npmjs.org/pixi.js/-/pixi.js-8.7.3.tgz",
"integrity": "sha512-o5pw7G2yuIrnBx0G4npBlmFp+XGNcapI/Ufs62rRj/4XKxc1Zo74YJr/BtEXcXTraTKd+pQvYOLvnfxRjxBMvQ==", "integrity": "sha512-wfWlhJYnGx1s4f2yoouevQjaeacbJ12LTkJGa+n9AIYNIjOnmJylBtZ2mARX7iFk3mr2xv0wuo//XPe2hk5OBw==",
"license": "MIT",
"dependencies": { "dependencies": {
"@pixi/colord": "^2.9.6", "@pixi/colord": "^2.9.6",
"@types/css-font-loading-module": "^0.0.12", "@types/css-font-loading-module": "^0.0.12",

View File

@ -34,6 +34,7 @@
"bin": { "bin": {
"quartz": "./quartz/bootstrap-cli.mjs" "quartz": "./quartz/bootstrap-cli.mjs"
}, },
"types": "./ambient.d.ts",
"dependencies": { "dependencies": {
"@clack/prompts": "^0.9.1", "@clack/prompts": "^0.9.1",
"@floating-ui/dom": "^1.6.13", "@floating-ui/dom": "^1.6.13",
@ -60,7 +61,7 @@
"mdast-util-to-hast": "^13.2.0", "mdast-util-to-hast": "^13.2.0",
"mdast-util-to-string": "^4.0.0", "mdast-util-to-string": "^4.0.0",
"micromorph": "^0.4.5", "micromorph": "^0.4.5",
"pixi.js": "^8.6.6", "pixi.js": "^8.7.3",
"preact": "^10.25.4", "preact": "^10.25.4",
"preact-render-to-string": "^6.5.13", "preact-render-to-string": "^6.5.13",
"pretty-bytes": "^6.1.1", "pretty-bytes": "^6.1.1",
@ -102,10 +103,10 @@
"@types/d3": "^7.4.3", "@types/d3": "^7.4.3",
"@types/hast": "^3.0.4", "@types/hast": "^3.0.4",
"@types/js-yaml": "^4.0.9", "@types/js-yaml": "^4.0.9",
"@types/node": "^22.10.6", "@types/node": "^22.12.0",
"@types/pretty-time": "^1.1.5", "@types/pretty-time": "^1.1.5",
"@types/source-map-support": "^0.5.10", "@types/source-map-support": "^0.5.10",
"@types/ws": "^8.5.13", "@types/ws": "^8.5.14",
"@types/yargs": "^17.0.33", "@types/yargs": "^17.0.33",
"esbuild": "^0.24.2", "esbuild": "^0.24.2",
"prettier": "^3.4.2", "prettier": "^3.4.2",

View File

@ -1,5 +1,4 @@
import { QuartzConfig } from "./quartz/cfg" import { QuartzConfig, Plugin } from "quartz"
import * as Plugin from "./quartz/plugins"
/** /**
* Quartz 4.0 Configuration * Quartz 4.0 Configuration

View File

@ -1,5 +1,4 @@
import { PageLayout, SharedLayout } from "./quartz/cfg" import { Component, PageLayout, SharedLayout } from "quartz"
import * as Component from "./quartz/components"
// components shared across all pages // components shared across all pages
export const sharedPageComponents: SharedLayout = { export const sharedPageComponents: SharedLayout = {

View File

@ -1,6 +1,6 @@
#!/usr/bin/env node #!/usr/bin/env node
import workerpool from "workerpool" import workerpool from "workerpool"
const cacheFile = "./.quartz-cache/transpiled-worker.mjs" const cacheFile = process.argv[2]
const { parseMarkdown, processHtml } = await import(cacheFile) const { parseMarkdown, processHtml } = await import(cacheFile)
workerpool.worker({ workerpool.worker({
parseMarkdown, parseMarkdown,

View File

@ -8,7 +8,7 @@ import chalk from "chalk"
import { parseMarkdown } from "./processors/parse" import { parseMarkdown } from "./processors/parse"
import { filterContent } from "./processors/filter" import { filterContent } from "./processors/filter"
import { emitContent } from "./processors/emit" import { emitContent } from "./processors/emit"
import cfg from "../quartz.config" import cfg from "$config"
import { FilePath, FullSlug, joinSegments, slugifyFilePath } from "./util/path" import { FilePath, FullSlug, joinSegments, slugifyFilePath } from "./util/path"
import chokidar from "chokidar" import chokidar from "chokidar"
import { ProcessedContent } from "./plugins/vfile" import { ProcessedContent } from "./plugins/vfile"
@ -42,12 +42,13 @@ function newBuildId() {
return Math.random().toString(36).substring(2, 8) return Math.random().toString(36).substring(2, 8)
} }
async function buildQuartz(argv: Argv, mut: Mutex, clientRefresh: () => void) { async function buildQuartz(quartzRoot: string, argv: Argv, mut: Mutex, clientRefresh: () => void) {
const ctx: BuildCtx = { const ctx: BuildCtx = {
buildId: newBuildId(), buildId: newBuildId(),
argv, argv,
cfg, cfg,
allSlugs: [], allSlugs: [],
quartzRoot,
} }
const perf = new PerfTimer() const perf = new PerfTimer()
@ -413,9 +414,9 @@ async function rebuildFromEntrypoint(
release() release()
} }
export default async (argv: Argv, mut: Mutex, clientRefresh: () => void) => { export default async (quartzRoot: string, argv: Argv, mut: Mutex, clientRefresh: () => void) => {
try { try {
return await buildQuartz(argv, mut, clientRefresh) return await buildQuartz(quartzRoot, argv, mut, clientRefresh)
} catch (err) { } catch (err) {
trace("\nExiting Quartz due to a fatal error", err as Error) trace("\nExiting Quartz due to a fatal error", err as Error)
} }

View File

@ -1,5 +1,5 @@
import path from "path" import path from "path"
import { readFileSync } from "fs" import { accessSync, readFileSync } from "fs"
/** /**
* All constants relating to helpers or handlers * All constants relating to helpers or handlers
@ -7,9 +7,26 @@ import { readFileSync } from "fs"
export const ORIGIN_NAME = "origin" export const ORIGIN_NAME = "origin"
export const UPSTREAM_NAME = "upstream" export const UPSTREAM_NAME = "upstream"
export const QUARTZ_SOURCE_BRANCH = "v4" export const QUARTZ_SOURCE_BRANCH = "v4"
export const cwd = process.cwd() export const cwd = process.cwd()
export const cacheDir = path.join(cwd, ".quartz-cache")
export const cacheFile = "./quartz/.quartz-cache/transpiled-build.mjs" function selectCacheDir() {
export const fp = "./quartz/build.ts" try {
export const { version } = JSON.parse(readFileSync("./package.json").toString()) const node_modules = path.join(cwd, "node_modules")
accessSync(node_modules) // check if node_modules exists
return path.join(node_modules, ".cache", "quartz")
} catch {
// standalone quartz bin?
return path.join(cwd, ".quartz-cache")
}
}
export const cacheDir = selectCacheDir()
export const cacheFile = path.join(cacheDir, "transpiled-build.mjs")
export const contentCacheFolder = path.join(cacheDir, "content-cache") export const contentCacheFolder = path.join(cacheDir, "content-cache")
export const quartzRoot = path.resolve(import.meta.dirname, "..")
export const fp = path.join(quartzRoot, "build.ts")
export const { version } = JSON.parse(
readFileSync(path.resolve(quartzRoot, "..", "package.json")).toString(),
)

View File

@ -31,7 +31,9 @@ import {
fp, fp,
cacheFile, cacheFile,
cwd, cwd,
quartzRoot,
} from "./constants.js" } from "./constants.js"
import { pathToFileURL } from "url"
/** /**
* Handles `npx quartz create` * Handles `npx quartz create`
@ -232,6 +234,12 @@ export async function handleBuild(argv) {
metafile: true, metafile: true,
sourcemap: true, sourcemap: true,
sourcesContent: false, sourcesContent: false,
alias: {
$config: path.join(cwd, "quartz.config.ts"),
$layout: path.join(cwd, "quartz.layout.ts"),
$styles: path.join(cwd, "styles.scss"),
quartz: path.resolve(quartzRoot, ".."),
},
plugins: [ plugins: [
sassPlugin({ sassPlugin({
type: "css-text", type: "css-text",
@ -303,8 +311,9 @@ export async function handleBuild(argv) {
release() release()
if (argv.bundleInfo) { if (argv.bundleInfo) {
const outputFileName = "quartz/.quartz-cache/transpiled-build.mjs" // metafile.outputs always uses /
const meta = result.metafile.outputs[outputFileName] const output = path.relative(cwd, cacheFile).replaceAll("\\", "/")
const meta = result.metafile.outputs[output]
console.log( console.log(
`Successfully transpiled ${Object.keys(meta.inputs).length} files (${prettyBytes( `Successfully transpiled ${Object.keys(meta.inputs).length} files (${prettyBytes(
meta.bytes, meta.bytes,
@ -313,12 +322,14 @@ export async function handleBuild(argv) {
console.log(await esbuild.analyzeMetafile(result.metafile, { color: true })) console.log(await esbuild.analyzeMetafile(result.metafile, { color: true }))
} }
// absolute path on windows has to be a file:// url
const url = pathToFileURL(cacheFile)
// bypass module cache // bypass module cache
// https://github.com/nodejs/modules/issues/307 // https://github.com/nodejs/modules/issues/307
const { default: buildQuartz } = await import(`../../${cacheFile}?update=${randomUUID()}`) url.searchParams.set("update", randomUUID())
// ^ this import is relative, so base "cacheFile" path can't be used const { default: buildQuartz } = await import(url)
cleanupBuild = await buildQuartz(argv, buildMutex, clientRefresh) cleanupBuild = await buildQuartz(quartzRoot, argv, buildMutex, clientRefresh)
clientRefresh() clientRefresh()
} }

4
quartz/index.ts Normal file
View File

@ -0,0 +1,4 @@
export * as Component from "./components"
export * as Plugin from "./plugins"
export * from "./i18n"
export * from "./cfg"

View File

@ -4,7 +4,7 @@ import BodyConstructor from "../../components/Body"
import { pageResources, renderPage } from "../../components/renderPage" import { pageResources, renderPage } from "../../components/renderPage"
import { FullPageLayout } from "../../cfg" import { FullPageLayout } from "../../cfg"
import { FilePath, FullSlug } from "../../util/path" import { FilePath, FullSlug } from "../../util/path"
import { sharedPageComponents } from "../../../quartz.layout" import { sharedPageComponents } from "$layout"
import { NotFound } from "../../components" import { NotFound } from "../../components"
import { defaultProcessedContent } from "../vfile" import { defaultProcessedContent } from "../vfile"
import { write } from "./helpers" import { write } from "./helpers"

View File

@ -14,6 +14,7 @@ import { Features, transform } from "lightningcss"
import { transform as transpile } from "esbuild" import { transform as transpile } from "esbuild"
import { write } from "./helpers" import { write } from "./helpers"
import DepGraph from "../../depgraph" import DepGraph from "../../depgraph"
import path from "path"
type ComponentResources = { type ComponentResources = {
css: string[] css: string[]
@ -183,8 +184,13 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
getQuartzComponents() { getQuartzComponents() {
return [] return []
}, },
async getDependencyGraph(_ctx, _content, _resources) { async getDependencyGraph(ctx, _content, _resources) {
return new DepGraph<FilePath>() const graph = new DepGraph<FilePath>()
graph.addEdge(
path.join(ctx.argv.output, "index.css") as FilePath,
path.join(process.cwd(), "styles.scss") as FilePath,
)
return graph
}, },
async emit(ctx, _content, _resources): Promise<FilePath[]> { async emit(ctx, _content, _resources): Promise<FilePath[]> {
const promises: Promise<FilePath>[] = [] const promises: Promise<FilePath>[] = []
@ -245,6 +251,7 @@ export const ComponentResources: QuartzEmitterPlugin = () => {
googleFontsStyleSheet, googleFontsStyleSheet,
...componentResources.css, ...componentResources.css,
styles, styles,
await import("$styles").then((s) => s.default ?? s).catch(() => ""),
) )
const [prescript, postscript] = await Promise.all([ const [prescript, postscript] = await Promise.all([
joinScripts(componentResources.beforeDOMLoaded), joinScripts(componentResources.beforeDOMLoaded),

View File

@ -10,7 +10,7 @@ import { pageResources, renderPage } from "../../components/renderPage"
import { FullPageLayout } from "../../cfg" import { FullPageLayout } from "../../cfg"
import { Argv } from "../../util/ctx" import { Argv } from "../../util/ctx"
import { FilePath, isRelativeURL, joinSegments, pathToRoot } from "../../util/path" import { FilePath, isRelativeURL, joinSegments, pathToRoot } from "../../util/path"
import { defaultContentPageLayout, sharedPageComponents } from "../../../quartz.layout" import { defaultContentPageLayout, sharedPageComponents } from "$layout"
import { Content } from "../../components" import { Content } from "../../components"
import chalk from "chalk" import chalk from "chalk"
import { write } from "./helpers" import { write } from "./helpers"

View File

@ -15,7 +15,7 @@ import {
pathToRoot, pathToRoot,
simplifySlug, simplifySlug,
} from "../../util/path" } from "../../util/path"
import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.layout" import { defaultListPageLayout, sharedPageComponents } from "$layout"
import { FolderContent } from "../../components" import { FolderContent } from "../../components"
import { write } from "./helpers" import { write } from "./helpers"
import { i18n } from "../../i18n" import { i18n } from "../../i18n"

View File

@ -1,4 +1,4 @@
import { FilePath, QUARTZ, joinSegments } from "../../util/path" import { FilePath, joinSegments } from "../../util/path"
import { QuartzEmitterPlugin } from "../types" import { QuartzEmitterPlugin } from "../types"
import fs from "fs" import fs from "fs"
import { glob } from "../../util/glob" import { glob } from "../../util/glob"
@ -9,10 +9,10 @@ export const Static: QuartzEmitterPlugin = () => ({
getQuartzComponents() { getQuartzComponents() {
return [] return []
}, },
async getDependencyGraph({ argv, cfg }, _content, _resources) { async getDependencyGraph({ argv, cfg, quartzRoot }, _content, _resources) {
const graph = new DepGraph<FilePath>() const graph = new DepGraph<FilePath>()
const staticPath = joinSegments(QUARTZ, "static") const staticPath = joinSegments(quartzRoot, "static")
const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns) const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
for (const fp of fps) { for (const fp of fps) {
graph.addEdge( graph.addEdge(
@ -23,8 +23,8 @@ export const Static: QuartzEmitterPlugin = () => ({
return graph return graph
}, },
async emit({ argv, cfg }, _content, _resources): Promise<FilePath[]> { async emit({ argv, cfg, quartzRoot }, _content, _resources): Promise<FilePath[]> {
const staticPath = joinSegments(QUARTZ, "static") const staticPath = joinSegments(quartzRoot, "static")
const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns) const fps = await glob("**", staticPath, cfg.configuration.ignorePatterns)
await fs.promises.cp(staticPath, joinSegments(argv.output, "static"), { await fs.promises.cp(staticPath, joinSegments(argv.output, "static"), {
recursive: true, recursive: true,

View File

@ -12,7 +12,7 @@ import {
joinSegments, joinSegments,
pathToRoot, pathToRoot,
} from "../../util/path" } from "../../util/path"
import { defaultListPageLayout, sharedPageComponents } from "../../../quartz.layout" import { defaultListPageLayout, sharedPageComponents } from "$layout"
import { TagContent } from "../../components" import { TagContent } from "../../components"
import { write } from "./helpers" import { write } from "./helpers"
import { i18n } from "../../i18n" import { i18n } from "../../i18n"

View File

@ -7,7 +7,7 @@ import { Root as HTMLRoot } from "hast"
import { MarkdownContent, ProcessedContent } from "../plugins/vfile" import { MarkdownContent, ProcessedContent } from "../plugins/vfile"
import { PerfTimer } from "../util/perf" import { PerfTimer } from "../util/perf"
import { read } from "to-vfile" import { read } from "to-vfile"
import { FilePath, FullSlug, QUARTZ, slugifyFilePath } from "../util/path" import { FilePath, FullSlug, slugifyFilePath } from "../util/path"
import path from "path" import path from "path"
import workerpool, { Promise as WorkerPromise } from "workerpool" import workerpool, { Promise as WorkerPromise } from "workerpool"
import { QuartzLogger } from "../util/log" import { QuartzLogger } from "../util/log"
@ -49,20 +49,28 @@ function* chunks<T>(arr: T[], n: number) {
} }
} }
async function transpileWorkerScript() { async function transpileWorkerScript(ctx: BuildCtx): Promise<string> {
// transpile worker script // import.meta.dirname is the cache folder, because we're in transpiled-build.mjs atm technically
const cacheFile = "./.quartz-cache/transpiled-worker.mjs" const cacheFile = path.join(import.meta.dirname, "transpiled-worker.mjs")
const fp = "./quartz/worker.ts" const fp = path.join(ctx.quartzRoot, "worker.ts")
return esbuild.build({ await esbuild.build({
entryPoints: [fp], entryPoints: [fp],
outfile: path.join(QUARTZ, cacheFile), outfile: cacheFile,
bundle: true, bundle: true,
keepNames: true, keepNames: true,
minifyWhitespace: true,
minifySyntax: true,
platform: "node", platform: "node",
format: "esm", format: "esm",
packages: "external", packages: "external",
sourcemap: true, sourcemap: true,
sourcesContent: false, sourcesContent: false,
alias: {
$config: path.join(process.cwd(), "quartz.config.ts"),
$layout: path.join(process.cwd(), "quartz.layout.ts"),
$styles: path.join(process.cwd(), "styles.scss"),
quartz: path.resolve(ctx.quartzRoot, ".."),
},
plugins: [ plugins: [
{ {
name: "css-and-scripts-as-text", name: "css-and-scripts-as-text",
@ -79,6 +87,7 @@ async function transpileWorkerScript() {
}, },
], ],
}) })
return cacheFile
} }
export function createFileParser(ctx: BuildCtx, fps: FilePath[]) { export function createFileParser(ctx: BuildCtx, fps: FilePath[]) {
@ -164,11 +173,12 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
throw error throw error
} }
} else { } else {
await transpileWorkerScript() const transpiledWorker = await transpileWorkerScript(ctx)
const pool = workerpool.pool("./quartz/bootstrap-worker.mjs", { const pool = workerpool.pool(path.join(ctx.quartzRoot, "bootstrap-worker.mjs"), {
minWorkers: "max", minWorkers: "max",
maxWorkers: concurrency, maxWorkers: concurrency,
workerType: "thread", workerType: "thread",
workerThreadOpts: { argv: [transpiledWorker] },
}) })
const errorHandler = (err: any) => { const errorHandler = (err: any) => {
console.error(`${err}`.replace(/^error:\s*/i, "")) console.error(`${err}`.replace(/^error:\s*/i, ""))
@ -177,7 +187,7 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
const mdPromises: WorkerPromise<[MarkdownContent[], FullSlug[]]>[] = [] const mdPromises: WorkerPromise<[MarkdownContent[], FullSlug[]]>[] = []
for (const chunk of chunks(fps, CHUNK_SIZE)) { for (const chunk of chunks(fps, CHUNK_SIZE)) {
mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, argv, chunk])) mdPromises.push(pool.exec("parseMarkdown", [ctx.buildId, ctx.quartzRoot, argv, chunk]))
} }
const mdResults: [MarkdownContent[], FullSlug[]][] = const mdResults: [MarkdownContent[], FullSlug[]][] =
await WorkerPromise.all(mdPromises).catch(errorHandler) await WorkerPromise.all(mdPromises).catch(errorHandler)
@ -187,7 +197,9 @@ export async function parseMarkdown(ctx: BuildCtx, fps: FilePath[]): Promise<Pro
ctx.allSlugs.push(...extraSlugs) ctx.allSlugs.push(...extraSlugs)
} }
for (const [mdChunk, _] of mdResults) { for (const [mdChunk, _] of mdResults) {
childPromises.push(pool.exec("processHtml", [ctx.buildId, argv, mdChunk, ctx.allSlugs])) childPromises.push(
pool.exec("processHtml", [ctx.buildId, ctx.quartzRoot, argv, mdChunk, ctx.allSlugs]),
)
} }
const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch(errorHandler) const results: ProcessedContent[][] = await WorkerPromise.all(childPromises).catch(errorHandler)

View File

@ -18,4 +18,5 @@ export interface BuildCtx {
argv: Argv argv: Argv
cfg: QuartzConfig cfg: QuartzConfig
allSlugs: FullSlug[] allSlugs: FullSlug[]
quartzRoot: string
} }

View File

@ -6,8 +6,6 @@ export const clone = rfdc()
// this file must be isomorphic so it can't use node libs (e.g. path) // this file must be isomorphic so it can't use node libs (e.g. path)
export const QUARTZ = "quartz"
/// Utility type to simulate nominal types in TypeScript /// Utility type to simulate nominal types in TypeScript
type SlugLike<T> = string & { __brand: T } type SlugLike<T> = string & { __brand: T }

View File

@ -6,7 +6,7 @@ export const options: sourceMapSupport.Options = {
// source map hack to get around query param // source map hack to get around query param
// import cache busting // import cache busting
retrieveSourceMap(source) { retrieveSourceMap(source) {
if (source.includes(".quartz-cache")) { if (source.includes("?update")) {
let realSource = fileURLToPath(source.split("?", 2)[0] + ".map") let realSource = fileURLToPath(source.split("?", 2)[0] + ".map")
return { return {
map: fs.readFileSync(realSource, "utf8"), map: fs.readFileSync(realSource, "utf8"),

View File

@ -1,6 +1,5 @@
import sourceMapSupport from "source-map-support" import sourceMapSupport from "source-map-support"
sourceMapSupport.install(options) sourceMapSupport.install(options)
import cfg from "../quartz.config"
import { Argv, BuildCtx } from "./util/ctx" import { Argv, BuildCtx } from "./util/ctx"
import { FilePath, FullSlug } from "./util/path" import { FilePath, FullSlug } from "./util/path"
import { import {
@ -12,9 +11,12 @@ import {
import { options } from "./util/sourcemap" import { options } from "./util/sourcemap"
import { MarkdownContent, ProcessedContent } from "./plugins/vfile" import { MarkdownContent, ProcessedContent } from "./plugins/vfile"
import cfg from "$config"
// only called from worker thread // only called from worker thread
export async function parseMarkdown( export async function parseMarkdown(
buildId: string, buildId: string,
quartzRoot: string,
argv: Argv, argv: Argv,
fps: FilePath[], fps: FilePath[],
): Promise<[MarkdownContent[], FullSlug[]]> { ): Promise<[MarkdownContent[], FullSlug[]]> {
@ -27,6 +29,7 @@ export async function parseMarkdown(
cfg, cfg,
argv, argv,
allSlugs, allSlugs,
quartzRoot,
} }
return [await createFileParser(ctx, fps)(createMdProcessor(ctx)), allSlugs] return [await createFileParser(ctx, fps)(createMdProcessor(ctx)), allSlugs]
} }
@ -34,6 +37,7 @@ export async function parseMarkdown(
// only called from worker thread // only called from worker thread
export function processHtml( export function processHtml(
buildId: string, buildId: string,
quartzRoot: string,
argv: Argv, argv: Argv,
mds: MarkdownContent[], mds: MarkdownContent[],
allSlugs: FullSlug[], allSlugs: FullSlug[],
@ -43,6 +47,7 @@ export function processHtml(
cfg, cfg,
argv, argv,
allSlugs, allSlugs,
quartzRoot,
} }
return createMarkdownParser(ctx, mds)(createHtmlProcessor(ctx)) return createMarkdownParser(ctx, mds)(createHtmlProcessor(ctx))
} }