feat: Update FlexSearch and Add Support for All Languages (#2108)

* chore(deps): update flexsearch to version 0.8.205 and adjust search encoder.

* refactor(search): enhance search encoder and update search results type

- Improved the encoder function to filter out empty tokens.
- Updated the search results type from a specific FlexSearch type to a more generic 'any' type for flexibility.
- Removed redundant rtl property from the index configuration.

* refactor(search): remove rtl property from search index configuration

* refactor(search): improve encoder function formatting

- Updated the encoder function to use consistent arrow function syntax for better readability.

* refactor(search): update search results type to DefaultDocumentSearchResults

- Imported DefaultDocumentSearchResults from FlexSearch for improved type safety.
- Changed the type of searchResults from 'any' to DefaultDocumentSearchResults<Item> for better clarity and maintainability.

package-lock.json

@@ -20,7 +20,7 @@
         "cli-spinner": "^0.2.10",
         "d3": "^7.9.0",
         "esbuild-sass-plugin": "^3.3.1",
-        "flexsearch": "0.7.43",
+        "flexsearch": "^0.8.205",
         "github-slugger": "^2.0.0",
         "globby": "^14.1.0",
         "gray-matter": "^4.0.3",
@@ -3189,9 +3189,36 @@
       }
     },
     "node_modules/flexsearch": {
-      "version": "0.7.43",
-      "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.7.43.tgz",
-      "integrity": "sha512-c5o/+Um8aqCSOXGcZoqZOm+NqtVwNsvVpWv6lfmSclU954O3wvQKxxK8zj74fPaSJbXpSLTs4PRhh+wnoCXnKg=="
+      "version": "0.8.205",
+      "resolved": "https://registry.npmjs.org/flexsearch/-/flexsearch-0.8.205.tgz",
+      "integrity": "sha512-REFjMqy86DKkCTJ4gIE42c9MVm9t1vUWfEub/8taixYuhvyu4jd4XmFALk5VuKW4GH4VLav8A4BJboTsslHF1w==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/ts-thomas"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/flexsearch"
+        },
+        {
+          "type": "patreon",
+          "url": "https://patreon.com/user?u=96245532"
+        },
+        {
+          "type": "liberapay",
+          "url": "https://liberapay.com/ts-thomas"
+        },
+        {
+          "type": "paypal",
+          "url": "https://www.paypal.com/donate/?hosted_button_id=GEVR88FC9BWRW"
+        },
+        {
+          "type": "bountysource",
+          "url": "https://salt.bountysource.com/teams/ts-thomas"
+        }
+      ],
+      "license": "Apache-2.0"
     },
     "node_modules/format": {
       "version": "0.2.2",

package.json

@@ -46,7 +46,7 @@
     "cli-spinner": "^0.2.10",
     "d3": "^7.9.0",
     "esbuild-sass-plugin": "^3.3.1",
-    "flexsearch": "0.7.43",
+    "flexsearch": "^0.8.205",
     "github-slugger": "^2.0.0",
     "globby": "^14.1.0",
     "gray-matter": "^4.0.3",

quartz/components/scripts/search.inline.ts

@@ -1,4 +1,4 @@
-import FlexSearch from "flexsearch"
+import FlexSearch, { DefaultDocumentSearchResults } from "flexsearch"
 import { ContentDetails } from "../../plugins/emitters/contentIndex"
 import { registerEscapeHandler, removeAllChildren } from "./util"
 import { FullSlug, normalizeRelativeURLs, resolveRelative } from "../../util/path"
@@ -9,15 +9,21 @@ interface Item {
   title: string
   content: string
   tags: string[]
+  [key: string]: any
 }
 
 // Can be expanded with things like "term" in the future
 type SearchType = "basic" | "tags"
 let searchType: SearchType = "basic"
 let currentSearchTerm: string = ""
-const encoder = (str: string) => str.toLowerCase().split(/([^a-z]|[^\x00-\x7F])/)
+const encoder = (str: string) => {
+  return str
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((token) => token.length > 0)
+}
+
 let index = new FlexSearch.Document<Item>({
-  charset: "latin:extra",
   encode: encoder,
   document: {
     id: "id",
@@ -397,7 +403,7 @@ async function setupSearch(searchElement: Element, currentSlug: FullSlug, data:
     searchLayout.classList.toggle("display-results", currentSearchTerm !== "")
     searchType = currentSearchTerm.startsWith("#") ? "tags" : "basic"
 
-    let searchResults: FlexSearch.SimpleDocumentSearchResultSetUnit[]
+    let searchResults: DefaultDocumentSearchResults<Item>
     if (searchType === "tags") {
       currentSearchTerm = currentSearchTerm.substring(1).trim()
       const separatorIndex = currentSearchTerm.indexOf(" ")
@@ -410,7 +416,7 @@ async function setupSearch(searchElement: Element, currentSlug: FullSlug, data:
           // return at least 10000 documents, so it is enough to filter them by tag (implemented in flexsearch)
           limit: Math.max(numSearchResults, 10000),
           index: ["title", "content"],
-          tag: tag,
+          tag: { tags: tag },
         })
         for (let searchResult of searchResults) {
           searchResult.result = searchResult.result.slice(0, numSearchResults)