diff --git a/examples/astro/package-lock.json b/examples/astro/package-lock.json index 29dcbb73..36528769 100644 --- a/examples/astro/package-lock.json +++ b/examples/astro/package-lock.json @@ -36,6 +36,7 @@ } }, "../../packages/adapters/astro": { + "name": "@galactiks/astro-integration", "version": "0.3.5", "license": "MIT", "dependencies": { @@ -48,7 +49,7 @@ "@astrojs/partytown": "2.0.4", "@astrojs/react": "3.0.10", "@astrojs/rss": "4.0.5", - "@astrojs/sitemap": "3.1.1", + "@galactiks/sitemap": "workspace:^", "@types/debug": "4.1.12", "@types/react": "18.2.58", "@types/react-dom": "18.2.19", @@ -68,12 +69,13 @@ "@astrojs/partytown": "~2.0", "@astrojs/react": "~3.0", "@astrojs/rss": "~4.0", - "@astrojs/sitemap": "~3.1", + "@galactiks/sitemap": "^0.1.0", "astro": "~4.4", "astro-robots-txt": "~1.0" } }, "../../packages/config": { + "name": "@galactiks/config", "version": "0.3.1", "license": "MIT", "dependencies": { @@ -89,6 +91,7 @@ } }, "../../packages/explorer": { + "name": "@galactiks/explorer", "version": "0.3.4", "license": "MIT", "dependencies": { diff --git a/examples/astro/src/assets/.gitkeep b/examples/astro/src/assets/.gitkeep new file mode 120000 index 00000000..4352783b --- /dev/null +++ b/examples/astro/src/assets/.gitkeep @@ -0,0 +1 @@ +/var/data/workspace/thegalactiks/explorer/examples/contentlayer/assets/.gitkeep \ No newline at end of file diff --git a/examples/contentlayer/content/websites/example.com.mdx b/examples/contentlayer/content/websites/galactiks.com.mdx similarity index 59% rename from examples/contentlayer/content/websites/example.com.mdx rename to examples/contentlayer/content/websites/galactiks.com.mdx index 0fa97ea7..fffe2f1c 100644 --- a/examples/contentlayer/content/websites/example.com.mdx +++ b/examples/contentlayer/content/websites/galactiks.com.mdx @@ -1,7 +1,7 @@ --- name: Website name description: Website description. -identifier: example.com -url: https://example.com/ +identifier: galactiks.com +url: https://www.galactiks.com/ dateCreated: 1970-01-01 --- diff --git a/packages/adapters/astro/package.json b/packages/adapters/astro/package.json index 0541f912..e106a7ee 100644 --- a/packages/adapters/astro/package.json +++ b/packages/adapters/astro/package.json @@ -51,7 +51,7 @@ "@astrojs/partytown": "~2.0", "@astrojs/react": "~3.0", "@astrojs/rss": "~4.0", - "@astrojs/sitemap": "~3.1", + "@galactiks/sitemap": "^0.1.0", "astro": "~4.4", "astro-robots-txt": "~1.0" }, @@ -59,7 +59,7 @@ "@astrojs/partytown": "2.0.4", "@astrojs/react": "3.0.10", "@astrojs/rss": "4.0.5", - "@astrojs/sitemap": "3.1.1", + "@galactiks/sitemap": "workspace:^", "@types/debug": "4.1.12", "@types/react": "18.2.58", "@types/react-dom": "18.2.19", diff --git a/packages/adapters/astro/src/index.ts b/packages/adapters/astro/src/index.ts index f9d15a51..5ffa5182 100644 --- a/packages/adapters/astro/src/index.ts +++ b/packages/adapters/astro/src/index.ts @@ -53,6 +53,13 @@ export default function createPlugin( updateConfig, addWatchFile, }) => { + let trailingSlash = config.trailingSlash; + if (trailingSlash === 'ignore') { + trailingSlash = config.build.format === 'directory' ? 'always' : 'never'; + } + + setConfig('trailingSlash', config.trailingSlash); + assetsPath = join(fileURLToPath(config.srcDir), 'assets'); galactiksConfig = setConfig('content.assets', assetsPath); @@ -61,6 +68,7 @@ export default function createPlugin( updateConfig({ site: galactiksConfig.webManifest.start_url, + trailingSlash, prefetch: true, vite: { resolve: { diff --git a/packages/adapters/astro/src/pages.ts b/packages/adapters/astro/src/pages.ts index 60abd597..5b29a298 100644 --- a/packages/adapters/astro/src/pages.ts +++ b/packages/adapters/astro/src/pages.ts @@ -6,11 +6,11 @@ import { export async function getStaticPaths() { return (await getPages({ inLanguages: getLanguages() })) - .filter((_p) => _p.path && _p.path !== '/') .map((page) => ({ - params: { path: page.path.slice(1) }, + params: { path: page.path.endsWith('/') ? page.path.slice(0, -1) : page.path }, props: { page }, - })); + })) + .filter(page => page.params.path && page.params.path !== '/'); } export function getIndexPage() { diff --git a/packages/adapters/astro/src/plugins/sitemap.ts b/packages/adapters/astro/src/plugins/sitemap.ts new file mode 100644 index 00000000..f527b11b --- /dev/null +++ b/packages/adapters/astro/src/plugins/sitemap.ts @@ -0,0 +1,112 @@ +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { AstroConfig, AstroIntegration } from 'astro'; +import { getDefaultLanguage } from '@galactiks/config'; +import { type Content, getPageByURL } from '@galactiks/explorer'; +import { generateSitemaps } from '@galactiks/sitemap'; + +const sitemapIndexOutput = 'sitemap-index.xml'; + +const createPlugin = (): AstroIntegration => { + let config: AstroConfig; + + return { + name: '@galactiks/astro-integration/sitemap', + + hooks: { + 'astro:config:done': async ({ config: cfg }) => { + config = cfg; + }, + + 'astro:build:done': async ({ dir, routes, pages, logger }) => { + if (!config.site) { + logger.warn( + 'The Sitemap integration requires the `site` astro.config option. Skipping.' + ); + return; + } + + let finalSiteUrl: URL; + if (config.site) { + finalSiteUrl = new URL(config.base, config.site); + } else { + console.warn( + 'The Sitemap integration requires the `site` astro.config option. Skipping.' + ); + return; + } + + let pageUrls = pages + .map((p) => { + if (p.pathname !== '' && !finalSiteUrl.pathname.endsWith('/')) + finalSiteUrl.pathname += '/'; + if (p.pathname.startsWith('/')) p.pathname = p.pathname.slice(1); + const fullPath = finalSiteUrl.pathname + p.pathname; + return new URL(fullPath, finalSiteUrl).href; + }); + + const routeUrls = routes.reduce((urls, r) => { + if (r.type !== 'page') return urls; + + /** + * Dynamic URLs have entries with `undefined` pathnames + */ + if (r.pathname) { + // `finalSiteUrl` may end with a trailing slash + // or not because of base paths. + let fullPath = finalSiteUrl.pathname; + if (fullPath.endsWith('/')) fullPath += r.generate(r.pathname).substring(1); + else fullPath += r.generate(r.pathname); + + const newUrl = new URL(fullPath, finalSiteUrl).href; + + if (config.trailingSlash === 'never') { + urls.push(newUrl); + } else if (config.build.format === 'directory' && !newUrl.endsWith('/')) { + urls.push(newUrl + '/'); + } else { + urls.push(newUrl); + } + } + + return urls; + }, []); + + pageUrls = Array.from(new Set([...pageUrls, ...routeUrls])); + logger.info(`Generating sitemap for ${pageUrls.length} pages`); + logger.info(pageUrls.join('\n')); + + const contentPages = ( + await Promise.all( + pageUrls.map(getPageByURL) + ) + ).filter((page) => page !== undefined) as Content[]; + if (contentPages.length === 0) { + logger.warn(`No pages found!\n\`${sitemapIndexOutput}\` not created.`); + return; + } + + const destDir = fileURLToPath(dir); + await generateSitemaps({ + destinationDir: destDir, + hostname: finalSiteUrl.href, + pages: contentPages, + defaultLanguage: getDefaultLanguage(), + publication: { + name: 'Galactiks', + }, + }); + // await simpleSitemapAndIndex({ + // hostname: finalSiteUrl.href, + // destinationDir: destDir, + // sourceData: urlData, + // limit: entryLimit, + // gzip: false, + // }); + logger.info(`\`${sitemapIndexOutput}\` created at \`${path.relative(process.cwd(), destDir)}\``); + }, + }, + }; +}; + +export default createPlugin; diff --git a/packages/adapters/astro/src/preset.ts b/packages/adapters/astro/src/preset.ts index 1babdd23..3d8fe5c7 100644 --- a/packages/adapters/astro/src/preset.ts +++ b/packages/adapters/astro/src/preset.ts @@ -1,29 +1,15 @@ import type { AstroIntegration } from 'astro'; import partytown from '@astrojs/partytown'; import react from '@astrojs/react'; -import sitemap from '@astrojs/sitemap'; -import { getDefaultLanguage, getLanguages } from '@galactiks/config'; import robotsTxt from 'astro-robots-txt'; -import { sitemapSerialize } from './sitemap.js'; +import sitemap from './plugins/sitemap.js'; export const integrationsPreset = (): AstroIntegration[] => { - const defaultLanguage = getDefaultLanguage(); - return [ react(), partytown(), - sitemap({ - i18n: defaultLanguage - ? { - defaultLocale: defaultLanguage, - locales: Object.fromEntries( - getLanguages().map((lang) => [lang, lang]) - ), - } - : undefined, - serialize: sitemapSerialize(), - }), + sitemap(), robotsTxt(), ]; }; diff --git a/packages/adapters/astro/src/sitemap.ts b/packages/adapters/astro/src/sitemap.ts deleted file mode 100644 index 55e9825f..00000000 --- a/packages/adapters/astro/src/sitemap.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { SitemapItem } from '@astrojs/sitemap'; -import { getPageByURL } from '@galactiks/explorer'; -import { isValid } from 'date-fns'; -import Debug from 'debug'; - -const debug = Debug('@galactiks/astro-integration:sitemap'); - -export const sitemapSerialize = - () => - async (item: SitemapItem): Promise => { - debug('serializing item', item); - - const page = await getPageByURL(item.url); - if (!page) { - debug('page not found for the item', item); - - return undefined; - } - - return { - url: page.url, - lastmod: isValid(page.dateModified) - ? page.dateModified.toISOString() - : undefined, - changefreq: undefined, - priority: undefined, - links: [], // TODO: add alternates links - // news: page.type === 'Article' ? { - // publication: { - // name: getConfig().webManifest.name, - // language: (page.inLanguage || defaultLanguage) - // ?.substring(0, 2) - // .toLowerCase(), - // }, - - // publication_date: isValid(page.datePublished) - // ? page.datePublished.toISOString() - // : undefined, - // title: page.name, - // keywords: page.keywords?.join(', '), - // } : undefined, - }; - }; diff --git a/packages/config/src/config.ts b/packages/config/src/config.ts index 702c7346..fcf105ad 100644 --- a/packages/config/src/config.ts +++ b/packages/config/src/config.ts @@ -29,7 +29,7 @@ const galactiksConfigFileSchema = z.object({ locales: localesSchema.optional(), template: z.string(), analytics: analyticsConfigSchema.optional(), - trailingSlash: z.enum(['always', 'never']).optional(), + trailingSlash: z.enum(['ignore', 'always', 'never']).optional(), pages: z .object({ articles: pagesObjectItemSchema, diff --git a/packages/explorer/src/index.ts b/packages/explorer/src/index.ts index c2934fab..49c6fe94 100644 --- a/packages/explorer/src/index.ts +++ b/packages/explorer/src/index.ts @@ -1 +1,3 @@ export * from './content/index.js'; + +export { Content } from './types/index.js'; diff --git a/packages/sitemap/README.md b/packages/sitemap/README.md new file mode 100644 index 00000000..81d62503 --- /dev/null +++ b/packages/sitemap/README.md @@ -0,0 +1,28 @@ +# @galactiks/config + +This package allows reading [Galactiks](https://www.galactiks.com) configurations files and get config during website generation. + +## Installation + +Install the `@galactiks/config` package using your preferred package manager: + +```sh +# npm +npm i @galactiks/config + +# yarn +yarn add @galactiks/config + +# pnpm +pnpm i @galactiks/config +``` + +If you run into any issues, [feel free to report them to us on GitHub](https://github.com/thegalactiks/explorer/issues). + +## Getting started + +Coming soon + +## License + +MIT © [Galactiks](https://www.galactiks.com) diff --git a/packages/sitemap/package.json b/packages/sitemap/package.json new file mode 100644 index 00000000..6a808082 --- /dev/null +++ b/packages/sitemap/package.json @@ -0,0 +1,43 @@ +{ + "name": "@galactiks/sitemap", + "version": "0.0.1", + "description": "A simple sitemap generator for Galactiks", + "author": "thegalactiks", + "type": "module", + "types": "./dist/index.d.ts", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + }, + "bugs": "https://github.com/thegalactiks/explorer/issues", + "homepage": "https://www.galactiks.com", + "repository": { + "type": "git", + "url": "https://github.com/thegalactiks/explorer.git", + "directory": "packages/sitemap" + }, + "main": "dist/index.js", + "scripts": { + "build": "tsc -p ./tsconfig.json", + "build:ci": "tsc -p ./tsconfig.json" + }, + "exports": { + "./package.json": "./package.json", + ".": "./dist/index.js" + }, + "files": [ + "dist" + ], + "keywords": [ + "galactiks", + "sitemap", + "seo" + ], + "dependencies": { + "date-fns": "3.3.1", + "sitemap": "7.1.1" + }, + "devDependencies": { + "@galactiks/explorer": "workspace:^" + } +} diff --git a/packages/sitemap/src/index.ts b/packages/sitemap/src/index.ts new file mode 100644 index 00000000..056d598b --- /dev/null +++ b/packages/sitemap/src/index.ts @@ -0,0 +1,66 @@ +import type { Content } from '@galactiks/explorer'; +import { type SitemapItem, SitemapStream, streamToPromise } from 'sitemap'; +import { createWriteStream } from 'fs'; +import { join } from 'path'; + +import { type SerializeOptions, sitemapSerialize } from './serialize.js'; + +function createSitemap(hostname: string, type: string, items: SitemapItem[]) { + const stream = new SitemapStream({ + hostname, + xmlns: { + news: type === 'Article' ? true : false, + xhtml: true, + image: false, + video: false, + }, + }); + items.forEach(item => stream.write(item)); + stream.end(); + + return stream; +} + +type GenerateSitemapsOptions = SerializeOptions & { + destinationDir: string; + pages: Content[]; + hostname: string; +}; + +export async function generateSitemaps({ destinationDir, hostname, pages, defaultLanguage, publication }: GenerateSitemapsOptions) { + const serialize = sitemapSerialize({ defaultLanguage, publication }); + + const sitemapsItems = pages.reduce((acc, page) => { + const { type } = page; + if (!acc[type]) { + acc[type] = []; + } + + acc[type].push(serialize(page)); + return acc; + }, {} as Record); + + const sitemaps = Object.entries(sitemapsItems).reduce((acc, [type, items]) => { + acc[type] = createSitemap(hostname, type, items); + return acc; + }, {} as Record); + + // Generate index sitemap + const indexSitemap = new SitemapStream({ + hostname, + xmlns: { + news: true, + xhtml: true, + image: false, + video: false, + }, + }); + Object.entries(sitemaps).forEach(([type, stream]) => { + const sitemapPath = `/sitemap-${type.toLowerCase()}.xml`; + + indexSitemap.write({ url: sitemapPath }); + stream.pipe(createWriteStream(join(destinationDir, sitemapPath))); + }); + indexSitemap.pipe(createWriteStream(join(destinationDir, `sitemap-index.xml`))); + return streamToPromise(indexSitemap).then(data => data.toString()); +} diff --git a/packages/sitemap/src/serialize.ts b/packages/sitemap/src/serialize.ts new file mode 100644 index 00000000..75d44b24 --- /dev/null +++ b/packages/sitemap/src/serialize.ts @@ -0,0 +1,41 @@ +import type { Content } from '@galactiks/explorer'; +import { isValid } from 'date-fns'; +import type { SitemapItem } from 'sitemap'; + +export type SerializeOptions = { + defaultLanguage?: string; + publication: { + name: string; + }; +}; + +export function sitemapSerialize(config: SerializeOptions) { + return (page: Content): SitemapItem => { + const isArticle = page.type === 'Article'; + const news: SitemapItem['news'] = isArticle ? { + publication: { + name: config.publication.name, + language: (page.inLanguage || config.defaultLanguage || '') + ?.substring(0, 2) + .toLowerCase(), + }, + + publication_date: isValid(page.datePublished) + ? page.datePublished.toISOString() + : new Date().toISOString(), + title: page.name, + keywords: page.keywords?.join(', '), + } : undefined; + + return { + url: page.url, + img: [], + video: [], + lastmod: isValid(page.dateModified) + ? page.dateModified.toISOString() + : undefined, + links: [], // TODO: add alternates links + news, + }; + }; +}; diff --git a/packages/sitemap/tsconfig.json b/packages/sitemap/tsconfig.json new file mode 100644 index 00000000..1421f3cd --- /dev/null +++ b/packages/sitemap/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*"], + "compilerOptions": { + "baseUrl": "./src", + "allowJs": true, + "target": "ES2021", + "module": "ES2022", + "declarationDir": "./dist", + "outDir": "./dist" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b962cacb..ed8582bd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -74,9 +74,9 @@ importers: '@astrojs/rss': specifier: 4.0.5 version: 4.0.5 - '@astrojs/sitemap': - specifier: 3.1.1 - version: 3.1.1 + '@galactiks/sitemap': + specifier: workspace:^ + version: link:../../sitemap '@types/debug': specifier: 4.1.12 version: 4.1.12 @@ -200,6 +200,19 @@ importers: specifier: 29.1.2 version: 29.1.2(@babel/core@7.23.9)(babel-jest@29.7.0)(esbuild@0.20.1)(jest@29.7.0)(typescript@5.3.3) + packages/sitemap: + dependencies: + date-fns: + specifier: 3.3.1 + version: 3.3.1 + sitemap: + specifier: 7.1.1 + version: 7.1.1 + devDependencies: + '@galactiks/explorer': + specifier: workspace:^ + version: link:../explorer + packages: /@aashutoshrathi/word-wrap@1.2.6: @@ -289,13 +302,6 @@ packages: kleur: 4.1.5 dev: true - /@astrojs/sitemap@3.1.1: - resolution: {integrity: sha512-qPgdBIcDUaea98mTtLfi5z9oXZpzSjEn/kes70/Ex8FOZZ+DIHVKRYOLOtvy8p+FTXr/9oc7BjmIbTYmYLLJVg==} - dependencies: - sitemap: 7.1.1 - zod: 3.22.4 - dev: true - /@astrojs/telemetry@3.0.4: resolution: {integrity: sha512-A+0c7k/Xy293xx6odsYZuXiaHO0PL+bnDoXOc47sGDF5ffIKdKQGRPFl2NMlCF4L0NqN4Ynbgnaip+pPF0s7pQ==} engines: {node: '>=18.14.1'} @@ -3549,13 +3555,12 @@ packages: /@types/node@17.0.45: resolution: {integrity: sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==} - dev: true + dev: false /@types/node@20.11.20: resolution: {integrity: sha512-7/rR21OS+fq8IyHTgtLkDK949uzsa6n8BkziAKtPVpugIkO6D+/ooXMvzXxDnZrmtXVfjb1bKQafYpb8s89LOg==} dependencies: undici-types: 5.26.5 - dev: true /@types/normalize-package-data@2.4.1: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} @@ -3590,7 +3595,7 @@ packages: resolution: {integrity: sha512-pSAff4IAxJjfAXUG6tFkO7dsSbTmf8CtUpfhhZ5VhkRpC4628tJhh3+V6H1E+/Gs9piSzYKT5yzHO5M4GG9jkw==} dependencies: '@types/node': 20.11.20 - dev: true + dev: false /@types/scheduler@0.16.3: resolution: {integrity: sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==} @@ -3869,7 +3874,7 @@ packages: /arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} - dev: true + dev: false /argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -9437,7 +9442,7 @@ packages: /sax@1.2.4: resolution: {integrity: sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==} - dev: true + dev: false /scheduler@0.23.0: resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==} @@ -9616,7 +9621,7 @@ packages: '@types/sax': 1.2.4 arg: 5.0.2 sax: 1.2.4 - dev: true + dev: false /slash@3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} @@ -10344,7 +10349,6 @@ packages: /undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} - dev: true /unherit@3.0.1: resolution: {integrity: sha512-akOOQ/Yln8a2sgcLj4U0Jmx0R5jpIg2IUyRrWOzmEbjBtGzBdHtSeFKgoEcoH4KYIG/Pb8GQ/BwtYm0GCq1Sqg==}