better content system

Dockerfile

@@ -50,6 +50,6 @@ COPY --from=build /app/package.json /app/package.json
 COPY --from=build /app/server.js /app/server.js
 COPY --from=build /app/server-utils.js /app/server-utils.js
 COPY --from=build /app/server-monitoring.js /app/server-monitoring.js
-COPY ./mdx ./mdx
+COPY ./content ./content
 
 CMD [ "pnpm","start" ]

app/routes/_site+/blog.tsx

@@ -5,7 +5,8 @@ import {
 	Link,
 	useLoaderData,
 } from 'react-router'
-import { type PostInfo, getPostInfos } from '#app/utils/mdx/blog.server.ts'
+import { posts as config } from '#app/utils/content/config.ts'
+import { retrieveAll } from '#app/utils/content/retrieve.ts'
 import { pipeHeaders } from '#app/utils/remix.server.ts'
 import { ServerTiming } from '#app/utils/timings.server.ts'
 
@@ -26,7 +27,7 @@ export const loader = async () => {
 	const timing = new ServerTiming()
 
 	timing.time('get posts', 'Get posts')
-	const posts = await getPostInfos()
+	const posts = await retrieveAll(config, timing)
 	timing.timeEnd('get posts')
 
 	return data(
@@ -46,35 +47,39 @@ export default function Blog() {
 		<div className="container max-w-[80ch]">
 			<ul className="flex flex-col gap-y-2">
 				{data.posts.map((post) => (
-					<PostItem key={post.slug} post={post} />
+					<PostItem key={post.meta.name} post={post} />
 				))}
 			</ul>
 		</div>
 	)
 }
 
-function PostItem({ post }: { post: PostInfo }) {
+function PostItem({
+	post,
+}: {
+	post: ReturnType<typeof useLoaderData<typeof loader>>['posts'][number]
+}) {
 	return (
 		<li>
 			<Link
-				to={post.slug}
+				to={post.meta.slug}
 				className="inline-block w-full rounded-md p-4 hover:bg-accent hover:text-accent-foreground"
 			>
 				<div className="flex items-baseline justify-between gap-x-2">
 					<div>
-						<h3 className="mr-2 inline text-lg">{post.meta.matter.title}</h3>
+						<h3 className="mr-2 inline text-lg">{post.matter.title}</h3>
 
 						<span className="whitespace-pre text-sm text-muted-foreground">
-							{post.meta.readingTime.text}
+							{post.readingTime}
 						</span>
 					</div>
 
-					<time dateTime={post.meta.matter.publishedDate} className="shrink-0">
-						{post.meta.matter.publishedDate}
+					<time dateTime={post.matter.publishedDate} className="shrink-0">
+						{post.matter.publishedDate}
 					</time>
 				</div>
 
-				<p className="text-muted-foreground">{post.meta.matter.description}</p>
+				<p className="text-muted-foreground">{post.matter.description}</p>
 			</Link>
 		</li>
 	)

app/routes/_site+/blog_.$slug.tsx

@@ -11,17 +11,18 @@ import {
 	GeneralErrorBoundary,
 	generalNotFoundHandler,
 } from '#app/components/error-boundary.tsx'
-import { getPostInfos, getPostMeta } from '#app/utils/mdx/blog.server'
-import { bundleMDX } from '#app/utils/mdx/compile-mdx.server'
-import { useMdxComponent } from '#app/utils/mdx/mdx'
-import { getMdxBundleSource, getMdxEntry } from '#app/utils/mdx/mdx.server'
+import { posts as config } from '#app/utils/content/config.ts'
+import { bundleMDX } from '#app/utils/content/mdx/bundler.server.ts'
+import { getMdxSource } from '#app/utils/content/mdx/mdx.server.ts'
+import { useMdxComponent } from '#app/utils/content/mdx/mdx.tsx'
+import { retrieve, retrieveAll } from '#app/utils/content/retrieve.ts'
 import { pipeHeaders } from '#app/utils/remix.server'
 import { ServerTiming } from '#app/utils/timings.server'
 
 export const handle: SEOHandle = {
 	getSitemapEntries: serverOnly$(async () => {
-		const posts = await getPostInfos()
-		return posts.map((post) => ({ route: `/blog/${post.slug}` }))
+		const posts = await retrieveAll(config)
+		return posts.map((post) => ({ route: `/blog/${post.meta.name}` }))
 	}),
 }
 
@@ -35,8 +36,8 @@ export const meta: MetaFunction<typeof loader> = ({ data }) => {
 		]
 	}
 	return [
-		{ title: `${data.meta.matter.title} | nichtsam` },
-		{ name: 'description', content: data.meta.matter.description },
+		{ title: `${data.matter.title} | nichtsam` },
+		{ name: 'description', content: data.matter.description },
 	]
 }
 
@@ -48,35 +49,30 @@ export const loader = async ({ params }: LoaderFunctionArgs) => {
 	const slug = params.slug
 	const timing = new ServerTiming()
 
-	timing.time('get post mdx entry', 'Get post mdx entry')
-	const mdxEntry = getMdxEntry('blog', slug)
-	if (!mdxEntry) {
-		timing.timeEnd('get post mdx entry')
+	timing.time('get post', 'Get post')
+	const post = await retrieve(config, slug, timing)
+	timing.timeEnd('get post')
+
+	if (!post) {
 		throw new Response('Not found', {
 			status: 404,
 			headers: {
 				'Server-Timing': timing.toString(),
 			},
 		})
 	}
-	timing.timeEnd('get post mdx entry')
-
-	timing.time('get post mdx bundle', 'Get post mdx bundle')
-	const mdxBundle = await getMdxBundleSource(mdxEntry)
-	timing.timeEnd('get post mdx bundle')
-
-	timing.time('bundle post mdx', 'Bundle post mdx')
-	const { code } = await bundleMDX({ slug, bundle: mdxBundle, timing })
-	timing.timeEnd('bundle post mdx')
 
-	timing.time('get post meta', 'Get post meta')
-	const meta = getPostMeta(mdxBundle.source)
-	timing.timeEnd('get post meta')
+	const bundleSource = await getMdxSource(post.meta)
+	const { code } = await bundleMDX({
+		slug,
+		bundleSource,
+		timing,
+	})
 
 	return data(
 		{
 			code,
-			meta,
+			matter: post.matter,
 		},
 		{
 			headers: {
@@ -89,6 +85,7 @@ export const loader = async ({ params }: LoaderFunctionArgs) => {
 
 export default function BlogPost() {
 	const data = useLoaderData<typeof loader>()
+	console.log({ name: data.matter.title, code: data.code })
 	const Component = useMdxComponent(data.code)
 
 	return (

app/utils/cache.server.ts

@@ -12,7 +12,7 @@ import { remember } from '@epic-web/remember'
 import { eq } from 'drizzle-orm'
 import { drizzle } from 'drizzle-orm/libsql'
 import { LRUCache } from 'lru-cache'
-import { z } from 'zod'
+import { optional, z } from 'zod'
 import * as cacheDbSchema from '#drizzle/cache.ts'
 import { env } from './env.server'
 import { type ServerTiming } from './timings.server'

app/utils/content/config.ts

@@ -0,0 +1,25 @@
+import dayjs from 'dayjs'
+import readingTime from 'reading-time'
+import { z } from 'zod'
+import { defineCollection } from './model'
+
+export const posts = defineCollection({
+	name: 'posts',
+	directory: 'content/blog',
+	includes: '**/*.{md,mdx}',
+	excludes: '_*',
+	schema: z.object({
+		title: z.string(),
+		description: z.string(),
+		publishedDate: z.date().transform((date) => dayjs(date).format('MMM YYYY')),
+		keywords: z.array(z.string()),
+	}),
+	transform: (doc) => ({
+		...doc,
+		readingTime: readingTime(doc.meta.content).text,
+	}),
+})
+
+export default {
+	collections: [posts],
+}

app/utils/mdx/compile-mdx.server.ts → app/utils/content/mdx/bundler.server.ts

@@ -1,11 +1,16 @@
 import { remember } from '@epic-web/remember'
 import { bundleMDX as _bundleMDX } from 'mdx-bundler'
+import { type BundleMDX } from 'mdx-bundler/dist/types'
 import PQueue from 'p-queue'
-import { cachified, longLivedCache } from '../cache.server.ts'
-import { type ServerTiming } from '../timings.server.ts'
-import { type MdxBundleSource } from './mdx.server.ts'
+import { cachified, longLivedCache } from '#app/utils/cache.server.ts'
+import { type ServerTiming } from '#app/utils/timings.server.ts'
 
-async function bundleMDX({ source, files }: MdxBundleSource) {
+export type MdxSource = {
+	source: Required<BundleMDX<any>>['source']
+	files?: BundleMDX<any>['files']
+}
+
+async function bundleMDX({ source, files }: MdxSource) {
 	const mdxBundle = await _bundleMDX({
 		source,
 		files,
@@ -36,11 +41,11 @@ const queuedBundleMDX = async (...args: Parameters<typeof bundleMDX>) =>
 
 function cachedBundleMDX({
 	slug,
-	bundle,
+	bundleSource,
 	timing,
 }: {
 	slug: string
-	bundle: MdxBundleSource
+	bundleSource: MdxSource
 	timing?: ServerTiming
 }) {
 	const key = `mdx:${slug}:compile`
@@ -50,7 +55,7 @@ function cachedBundleMDX({
 		ttl: 1000 * 60 * 60 * 24 * 14,
 		swr: Infinity,
 		timing,
-		getFreshValue: () => queuedBundleMDX(bundle),
+		getFreshValue: () => queuedBundleMDX(bundleSource),
 	})
 
 	return compileMdx

app/utils/content/mdx/mdx.server.ts

@@ -0,0 +1,61 @@
+import { existsSync, statSync } from 'node:fs'
+import { readFile, readdir } from 'node:fs/promises'
+import path from 'node:path'
+import { type Meta } from '../model.ts'
+import { type MdxSource } from './bundler.server.ts'
+
+export async function getMdxSource(meta: Meta): Promise<MdxSource> {
+	const filepath = path.resolve(meta.contentDir, meta.dir, meta.base)
+	const source = await readFile(filepath, 'utf-8')
+
+	const hasFiles = meta.dir !== '.'
+
+	if (!hasFiles) {
+		return {
+			source,
+		}
+	}
+
+	const dirPath = path.resolve(meta.contentDir, meta.dir)
+	const files = await getFilesInDirectory(dirPath)
+
+	return {
+		source,
+		files,
+	}
+}
+
+type Files = Record<string, string>
+async function getFilesInDirectory(
+	dirPath: string,
+	rootPath: string = dirPath,
+): Promise<Record<string, string>> {
+	if (!existsSync(dirPath)) {
+		return {}
+	}
+
+	if (statSync(dirPath).isFile()) {
+		const relativePath = path.relative(rootPath, dirPath)
+
+		return {
+			[relativePath]: await readFile(dirPath, 'utf-8'),
+		}
+	} else {
+		const dir = await readdir(dirPath, { withFileTypes: true })
+
+		const fileSets = await Promise.all(
+			dir.map((dirent) =>
+				getFilesInDirectory(
+					path.resolve(dirent.parentPath, dirent.name),
+					rootPath,
+				),
+			),
+		)
+
+		const files = fileSets.reduce<Files>((acc, files) => {
+			return Object.assign(acc, files)
+		}, {})
+
+		return files
+	}
+}

app/utils/content/model.ts

@@ -0,0 +1,39 @@
+import { type ZodRawShape, type ZodObject, type z } from 'zod'
+
+export function defineCollection<
+	TName extends string,
+	TSchema extends ZodObject<ZodRawShape>,
+	TResult,
+>(collection: Collection<TName, TSchema, TResult>) {
+	return collection
+}
+
+export type Meta = {
+	/** Directory of the content type */
+	contentDir: string
+	/** Directory of the file */
+	dir: string
+	/** File name with extension */
+	base: string
+	/** File extension */
+	ext: string
+	/** File name without extension */
+	name: string
+	/** Content of the file */
+	content: string
+	/** Slug identifier of the content */
+	slug: string
+}
+
+export type Collection<
+	TName extends string,
+	TSchema extends ZodObject<ZodRawShape>,
+	TResult,
+> = {
+	name: TName
+	directory: string
+	includes: string | string[]
+	excludes?: string | string[]
+	schema: TSchema
+	transform: (data: { matter: z.infer<TSchema>; meta: Meta }) => TResult
+}