diff --git a/docs/plugins/CrawlLinks.md b/docs/plugins/CrawlLinks.md index 47b7bdd77..aa6d3815a 100644 --- a/docs/plugins/CrawlLinks.md +++ b/docs/plugins/CrawlLinks.md @@ -19,6 +19,11 @@ This plugin accepts the following configuration options: - `openLinksInNewTab`: If `true`, configures external links to open in a new tab. Defaults to `false`. - `lazyLoad`: If `true`, adds lazy loading to resource elements (`img`, `video`, etc.) to improve page load performance. Defaults to `false`. - `externalLinkIcon`: Adds an icon next to external links when `true` (default) to visually distinguishing them from internal links. +- `showLinkFavicon`: If `true`, displays the favicon of external websites before each external link, making it easier to visually identify the source of the link. Defaults to `false`. + - **Note:** This feature is **disabled by default**. Favicons are fetched from Google's favicon service. + - **Caching:** Favicons are cached in memory during the build process. Each unique domain is fetched only once, even if there are multiple links to that domain. This prevents excessive API calls and rate limiting issues. +- `cacheLinkFavicons`: If `true`, preemptively fetches and caches external link favicons during the build process. This ensures favicons are available immediately on page load without additional runtime requests. Defaults to `false`. + - **Use case:** Enable this option if you want to avoid any runtime favicon requests and ensure all favicons are cached during build time. > [!warning] > Removing this plugin is _not_ recommended and will likely break the page. diff --git a/package-lock.json b/package-lock.json index cebf44726..8fca2b03f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -96,13 +96,13 @@ "node_modules/@bufbuild/protobuf": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.2.0.tgz", - "integrity": "sha512-+imAQkHf7U/Rwvu0wk1XWgsP3WnpCWmK7B48f0XqSNzgk64+grljTKC7pnO/xBiEMUziF7vKRfbBnOQhg126qQ==" + "integrity": "sha512-+imAQkHf7U/Rwvu0wk1XWgsP3WnpCWmK7B48f0XqSNzgk64+grljTKC7pnO/xBiEMUziF7vKRfbBnOQhg126qQ==", + "peer": true }, "node_modules/@citation-js/core": { "version": "0.7.14", "resolved": "https://registry.npmjs.org/@citation-js/core/-/core-0.7.14.tgz", "integrity": "sha512-dgeGqYDSQmn2MtnWZkwPGpJQPh43yr1lAAr9jl1NJ9pIY1RXUQxtlAUZVur0V9PHdbfQC+kkvB1KC3VpgVV3MA==", - "peer": true, "dependencies": { "@citation-js/date": "^0.5.0", "@citation-js/name": "^0.4.2", @@ -2300,7 +2300,8 @@ "node_modules/buffer-builder": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/buffer-builder/-/buffer-builder-0.2.0.tgz", - "integrity": "sha512-7VPMEPuYznPSoR21NE1zvd2Xna6c/CloiZCfcMXR1Jny6PjX0N4Nsa38zcBFo/FMK+BlA+FLKbJCQ0i2yxp+Xg==" + "integrity": "sha512-7VPMEPuYznPSoR21NE1zvd2Xna6c/CloiZCfcMXR1Jny6PjX0N4Nsa38zcBFo/FMK+BlA+FLKbJCQ0i2yxp+Xg==", + "peer": true }, "node_modules/buffer-from": { "version": "1.1.2", @@ -2419,7 +2420,8 @@ "node_modules/colorjs.io": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/colorjs.io/-/colorjs.io-0.5.2.tgz", - "integrity": "sha512-twmVoizEW7ylZSN32OgKdXRmo1qg+wT5/6C3xu5b9QsWzSFAhHLn2xd8ro0diCsKfCj1RdaTP/nrcW+vAoQPIw==" + "integrity": "sha512-twmVoizEW7ylZSN32OgKdXRmo1qg+wT5/6C3xu5b9QsWzSFAhHLn2xd8ro0diCsKfCj1RdaTP/nrcW+vAoQPIw==", + "peer": true }, "node_modules/comma-separated-tokens": { "version": "2.0.3", @@ -2792,7 +2794,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", - "peer": true, "engines": { "node": ">=12" } @@ -2971,7 +2972,6 @@ "integrity": "sha512-yY35KZckJJuVVPXpvjgxiCuVEJT67F6zDeVTv4rizyPrfGBUpZQsvmxnN+C371c2esD/hNMjj4tpBhuueLN7aA==", "hasInstallScript": true, "license": "MIT", - "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -3382,6 +3382,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "peer": true, "engines": { "node": ">=8" } @@ -5525,7 +5526,6 @@ "resolved": "https://registry.npmjs.org/preact/-/preact-10.28.0.tgz", "integrity": "sha512-rytDAoiXr3+t6OIP3WGlDd0ouCUG1iCWzkcY3++Nreuoi17y6T5i/zRhe6uYfoVcxq6YU+sBtJouuRDsq8vvqA==", "license": "MIT", - "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/preact" @@ -6082,6 +6082,7 @@ "version": "7.8.1", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "peer": true, "dependencies": { "tslib": "^2.1.0" } @@ -6166,6 +6167,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6181,6 +6183,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6196,6 +6199,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6211,6 +6215,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6226,6 +6231,7 @@ "os": [ "android" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6241,6 +6247,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6256,6 +6263,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6271,6 +6279,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6286,6 +6295,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6301,6 +6311,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6316,6 +6327,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6331,6 +6343,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6346,6 +6359,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6361,6 +6375,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6376,6 +6391,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6391,6 +6407,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6406,6 +6423,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6421,6 +6439,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6436,6 +6455,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6451,6 +6471,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=14.0.0" } @@ -6620,7 +6641,6 @@ "version": "1.26.2", "resolved": "https://registry.npmjs.org/shiki/-/shiki-1.26.2.tgz", "integrity": "sha512-iP7u2NA9A6JwRRCkIUREEX2cMhlYV5EBmbbSlfSRvPThwca8HBRbVkWuNWW+kw9+i6BSUZqqG6YeUs5dC2SjZw==", - "peer": true, "dependencies": { "@shikijs/core": "1.26.2", "@shikijs/engine-javascript": "1.26.2", @@ -6790,6 +6810,7 @@ "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "peer": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -7125,7 +7146,8 @@ "node_modules/varint": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/varint/-/varint-6.0.0.tgz", - "integrity": "sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==" + "integrity": "sha512-cXEIW6cfr15lFv563k4GuVuW/fiwjknytD37jIOLSdSWuOI6WnO/oKwmP2FQTU2l01LP8/M5TSAJpzUaGe3uWg==", + "peer": true }, "node_modules/vfile": { "version": "6.0.3", diff --git a/quartz/plugins/emitters/faviconCache.ts b/quartz/plugins/emitters/faviconCache.ts new file mode 100644 index 000000000..9753be9ac --- /dev/null +++ b/quartz/plugins/emitters/faviconCache.ts @@ -0,0 +1,166 @@ +import { QuartzEmitterPlugin } from "../types" +import { FilePath, joinSegments } from "../../util/path" +import fs from "fs" +import path from "path" + +interface Options { + /** + * Enable favicon caching during build time. + * When enabled, all external link favicons are pre-fetched and cached to /static/externalFavicons/ + */ + enabled: boolean + /** + * Number of concurrent favicon fetch requests. + * Keep this low (1-3) to avoid hitting API rate limits. + */ + concurrency: number + /** + * Timeout for individual favicon fetch requests in milliseconds. + */ + timeout: number + /** + * Whether to continue build if favicon fetch fails. + * If false, a single failed favicon will fail the entire build. + */ + continueOnError: boolean +} + +const defaultOptions: Options = { + enabled: false, + concurrency: 2, + timeout: 5000, + continueOnError: true, +} + +/** + * Pre-fetches and caches external link favicons during the build process. + * When enabled, this plugin collects all unique external domains found in links, + * fetches their favicons, and stores them locally in /static/externalFavicons/. + * + * This reduces runtime requests and improves page load performance. + * Used in conjunction with CrawlLinks plugin configured with cacheLinkFavicons: true. + */ +export const FaviconCache: QuartzEmitterPlugin> = (userOpts) => { + const opts = { ...defaultOptions, ...userOpts } + + return { + name: "FaviconCache", + async *emit(ctx, content) { + if (!opts.enabled) { + return + } + + // Collect all unique domains from external links + const domains = new Set() + + for (const [_tree, vfile] of content) { + // CrawlLinks plugin stores external link information in file.data.externalDomains + const externalDomains = (vfile.data.externalDomains ?? []) as string[] + for (const domain of externalDomains) { + if (domain && domain.trim()) { + domains.add(domain.toLowerCase()) + } + } + } + + if (domains.size === 0) { + return + } + + // Create favicon cache directory + const faviconCachePath = joinSegments( + ctx.argv.output, + "static/externalFavicons", + ) as FilePath + + await fs.promises.mkdir(faviconCachePath, { recursive: true }) + + // Fetch favicons with concurrency control + const domainArray = Array.from(domains) + let successCount = 0 + let failureCount = 0 + + console.log(`Caching favicons for ${domainArray.length} unique domains...`) + + for (let i = 0; i < domainArray.length; i += opts.concurrency) { + const batch = domainArray.slice(i, i + opts.concurrency) + + const results = await Promise.allSettled( + batch.map((domain) => fetchAndCacheFavicon(domain, faviconCachePath, opts)), + ) + + for (let j = 0; j < results.length; j++) { + const result = results[j] + const domain = batch[j] + + if (result.status === "fulfilled") { + successCount++ + yield joinSegments(faviconCachePath, `${domain}.ico`) as FilePath + } else { + failureCount++ + if (!opts.continueOnError) { + throw result.reason + } + } + } + } + + console.log( + `Favicon cache complete: ${successCount} succeeded, ${failureCount} failed`, + ) + }, + + async *partialEmit(_ctx, _content, _resources, _changeEvents) { + // For incremental builds, we could optimize by only fetching favicons + // for newly added/modified files. For now, we skip partial emission + // since favicons are relatively small and the cost is minimal. + }, + } +} + +async function fetchAndCacheFavicon( + domain: string, + cachePath: FilePath, + opts: Options, +): Promise { + const faviconUrl = `https://s2.googleusercontent.com/s2/favicons?domain_url=${domain}` + const fileName = `${domain}.ico` + const destPath = path.join(cachePath, fileName) as FilePath + + try { + // Create abort controller for timeout + const controller = new AbortController() + const timeoutId = setTimeout(() => controller.abort(), opts.timeout) + + const response = await fetch(faviconUrl, { + signal: controller.signal, + // Add User-Agent to avoid potential blocks + headers: { + "User-Agent": + "Mozilla/5.0 (compatible; Quartz/4.0 +https://quartz.jzhao.xyz/)", + }, + }) + + clearTimeout(timeoutId) + + if (!response.ok) { + throw new Error(`HTTP ${response.status} when fetching favicon for ${domain}`) + } + + const buffer = await response.arrayBuffer() + await fs.promises.writeFile(destPath, Buffer.from(buffer)) + } catch (err) { + if (!opts.continueOnError) { + throw err + } + // Log warning but continue + const errorMsg = err instanceof Error ? err.message : String(err) + console.warn(`⚠️ Failed to cache favicon for domain '${domain}': ${errorMsg}`) + } +} + +declare module "vfile" { + interface DataMap { + externalDomains: string[] + } +} diff --git a/quartz/plugins/emitters/index.ts b/quartz/plugins/emitters/index.ts index d2de2ed1e..4f8bcc951 100644 --- a/quartz/plugins/emitters/index.ts +++ b/quartz/plugins/emitters/index.ts @@ -6,6 +6,7 @@ export { AliasRedirects } from "./aliases" export { Assets } from "./assets" export { Static } from "./static" export { Favicon } from "./favicon" +export { FaviconCache } from "./faviconCache" export { ComponentResources } from "./componentResources" export { NotFoundPage } from "./404" export { CNAME } from "./cname" diff --git a/quartz/plugins/transformers/links.ts b/quartz/plugins/transformers/links.ts index f4451d927..59cde2b40 100644 --- a/quartz/plugins/transformers/links.ts +++ b/quartz/plugins/transformers/links.ts @@ -22,6 +22,10 @@ interface Options { openLinksInNewTab: boolean lazyLoad: boolean externalLinkIcon: boolean + /** Show favicon for external links (fetched from local cache or Google during build). Defaults to false. */ + showLinkFavicon: boolean + /** Fetch and cache external link favicons during build time. Defaults to false. */ + cacheLinkFavicons: boolean } const defaultOptions: Options = { @@ -30,6 +34,21 @@ const defaultOptions: Options = { openLinksInNewTab: false, lazyLoad: false, externalLinkIcon: true, + showLinkFavicon: false, + cacheLinkFavicons: false, +} + +const _faviconCache: Map = new Map() + +function getFaviconUrl(domain: string): string | null { + if (_faviconCache.has(domain)) { + const cached = _faviconCache.get(domain) + return cached === "ERROR" ? null : cached! + } + + const faviconUrl = `https://s2.googleusercontent.com/s2/favicons?domain_url=${domain}` + _faviconCache.set(domain, faviconUrl) + return faviconUrl } export const CrawlLinks: QuartzTransformerPlugin> = (userOpts) => { @@ -39,9 +58,10 @@ export const CrawlLinks: QuartzTransformerPlugin> = (userOpts) htmlPlugins(ctx) { return [ () => { - return (tree: Root, file) => { + return (tree: Root, file: any) => { const curSlug = simplifySlug(file.data.slug!) const outgoing: Set = new Set() + const externalDomains: Set = new Set() const transformOptions: TransformOptions = { strategy: opts.markdownLinkResolution, @@ -83,6 +103,34 @@ export const CrawlLinks: QuartzTransformerPlugin> = (userOpts) }) } + if (isExternal && opts.showLinkFavicon) { + const domain = new URL(node.properties.href).hostname + if (domain) { + // Track this domain for favicon caching if enabled + externalDomains.add(domain) + + // Determine favicon URL based on caching option + const faviconUrl = opts.cacheLinkFavicons + ? `/static/externalFavicons/${domain}.ico` // Use locally cached favicon + : getFaviconUrl(domain) // Use Google Favicon API + + if (faviconUrl) { + node.children.unshift({ + type: "element", + tagName: "img", + properties: { + src: faviconUrl, + alt: "", + loading: "lazy", + style: + "width: 1em; height: auto; margin-left: 4px; margin-right: 4px; vertical-align: middle;", + }, + children: [], + }) + } + } + } + // Check if the link has alias text if ( node.children.length === 1 && @@ -160,6 +208,7 @@ export const CrawlLinks: QuartzTransformerPlugin> = (userOpts) }) file.data.links = [...outgoing] + file.data.externalDomains = [...externalDomains] } }, ] @@ -170,5 +219,6 @@ export const CrawlLinks: QuartzTransformerPlugin> = (userOpts) declare module "vfile" { interface DataMap { links: SimpleSlug[] + externalDomains: string[] } }