feat: shiki highlighting, content cleanup, component org

Author: aulneau

.gitignore

@@ -39,3 +39,7 @@ yarn-error.log*
 
 .now
 .mdx-data
+.cache
+.yalc
+yalc.lock
+.cache

.vercelignore

@@ -6,3 +6,6 @@
 node_modules
 build
 README.md
+.cache
+.yalc
+yalc.lock

lib/remark-include.js

@@ -0,0 +1,51 @@
+const path = require('path');
+const remark = require('remark');
+const flatMap = require('unist-util-flatmap');
+const { readSync } = require('to-vfile');
+
+module.exports = function includeMarkdownPlugin({ resolveFrom } = {}) {
+  return function transformer(tree, file) {
+    return flatMap(tree, node => {
+      if (node.type !== 'paragraph') return [node];
+
+      // detect an `@include` statement
+      const includeMatch =
+        node.children[0].value && node.children[0].value.match(/^@include\s['"](.*)['"]$/);
+      if (!includeMatch) return [node];
+
+      // read the file contents
+      const includePath = path.join(resolveFrom || file.dirname, includeMatch[1]);
+      let includeContents;
+      try {
+        includeContents = readSync(includePath, 'utf8');
+      } catch (err) {
+        console.log(err);
+        throw new Error(
+          `The @include file path at ${includePath} was not found.\n\nInclude Location: ${file.path}:${node.position.start.line}:${node.position.start.column}`
+        );
+      }
+
+      // if we are including a ".md" or ".mdx" file, we add the contents as processed markdown
+      // if any other file type, they are embedded into a code block
+      if (includePath.match(/\.md(?:x)?$/)) {
+        // return the file contents in place of the @include
+        // this takes a couple steps because we allow recursive includes
+        const processor = remark().use(includeMarkdownPlugin, { resolveFrom });
+        const ast = processor.parse(includeContents);
+        return processor.runSync(ast, includeContents).children;
+      } else {
+        // trim trailing newline
+        includeContents.contents = includeContents.contents.trim();
+
+        // return contents wrapped inside a "code" node
+        return [
+          {
+            type: 'code',
+            lang: includePath.match(/\.(\w+)$/)[1],
+            value: includeContents,
+          },
+        ];
+      }
+    });
+  };
+};

lib/remark-paragraph-alerts.js

@@ -12,10 +12,10 @@ module.exports = function paragraphCustomAlertsPlugin() {
   return function transformer(tree) {
     visit(tree, 'paragraph', (pNode, _, parent) => {
       visit(pNode, 'text', textNode => {
-        Object.keys(sigils).forEach(symbol => {
-          if (textNode.value.startsWith(`${symbol} `)) {
+        Object.keys(sigils).forEach(sigil => {
+          if (textNode.value.startsWith(`${sigil} `)) {
             // Remove the literal sigil symbol from string contents
-            textNode.value = textNode.value.replace(`${symbol} `, '');
+            textNode.value = textNode.value.replace(`${sigil} `, '');
 
             // Wrap matched nodes with <div> (containing proper attributes)
             parent.children = parent.children.map(node => {
@@ -26,7 +26,7 @@ module.exports = function paragraphCustomAlertsPlugin() {
                     data: {
                       hName: 'blockquote',
                       hProperties: {
-                        className: ['alert', `alert-${sigils[symbol]}`],
+                        className: ['alert', `alert-${sigils[sigil]}`],
                         role: 'alert',
                       },
                     },

lib/remark-plugins.js

@@ -0,0 +1,16 @@
+const memoize = require('micro-memoize');
+const path = require('path');
+
+const remarkPlugins = [
+  [require('./remark-include'), { resolveFrom: path.join(__dirname, '../src/includes') }],
+  require('remark-vscode'),
+  memoize(require('./remark-paragraph-alerts')),
+  memoize(require('remark-external-links')),
+  memoize(require('remark-emoji')),
+  memoize(require('remark-images')),
+  memoize(require('remark-unwrap-images')),
+  memoize(require('remark-normalize-headings')),
+  memoize(require('remark-slug')),
+];
+
+module.exports = { remarkPlugins };

lib/remark-shiki.js

@@ -1,26 +1,8 @@
 const withBundleAnalyzer = require('@next/bundle-analyzer')({
   enabled: process.env.ANALYZE === 'true',
 });
-
-const webpack = require('webpack');
-
 const path = require('path');
-
-const remarkPlugins = [
-  [
-    require('@hashicorp/remark-plugins').includeMarkdown,
-    { resolveFrom: path.join(__dirname, 'src/_includes') },
-  ],
-  require('remark-squeeze-paragraphs'),
-  require('./lib/remark-paragraph-alerts'),
-  require('remark-external-links'),
-  require('remark-emoji'),
-  require('remark-images'),
-  require('remark-unwrap-images'),
-  require('remark-normalize-headings'),
-  require('remark-slug'),
-  require('remark-footnotes'),
-];
+const { remarkPlugins } = require('./lib/remark-plugins');
 
 module.exports = withBundleAnalyzer({
   experimental: {
@@ -70,12 +52,6 @@ module.exports = withBundleAnalyzer({
       const aliases = config.resolve.alias || (config.resolve.alias = {});
       aliases.react = aliases['react-dom'] = 'preact/compat';
       aliases['react-ssr-prepass'] = 'preact-ssr-prepass';
-
-      // to fix a dupe dependency
-      config.externals.push('prismjs');
-
-      // https://github.com/FormidableLabs/react-live#what-bundle-size-can-i-expect
-      aliases['buble'] = '@philpl/buble';
     }
 
     return config;

next.config.js

@@ -2,9 +2,10 @@
   "name": "@blockstack/docs",
   "version": "1.0.0",
   "dependencies": {
-    "@blockstack/ui": "^2.10.7",
+    "@blockstack/ui": "^2.12.2-beta.0",
     "@docsearch/react": "^1.0.0-alpha.14",
     "@hashicorp/remark-plugins": "^3.0.0",
+    "@mapbox/rehype-prism": "^0.5.0",
     "@mdx-js/loader": "1.6.13",
     "@mdx-js/mdx": "^1.6.13",
     "@mdx-js/react": "^1.6.13",
@@ -21,26 +22,29 @@
     "algoliasearch": "^4.3.1",
     "babel-plugin-macros": "^2.8.0",
     "babel-plugin-prismjs": "^2.0.1",
+    "cache-manager": "^3.3.0",
+    "cache-manager-fs-hash": "^0.0.9",
     "csvtojson": "^2.0.10",
     "docsearch.js": "^2.6.3",
     "eslint": "^7.4.0",
     "fathom-client": "^3.0.0",
     "front-matter": "^4.0.2",
     "fs-extra": "^9.0.1",
-    "gatsby-remark-shiki": "^0.1.2",
     "github-slugger": "^1.3.0",
     "gray-matter": "^4.0.2",
     "html-react-parser": "^0.13.0",
     "jsx-to-string": "^1.4.0",
     "lodash.debounce": "^4.0.8",
     "mdi-react": "7.3.0",
-    "next": "^9.4.5-canary.42",
+    "micro-memoize": "^4.0.9",
+    "next": "^9.4.5-canary.45",
     "next-google-fonts": "^1.1.0",
     "next-mdx-enhanced": "^3.0.0",
     "next-mdx-remote": "^0.6.0",
     "next-optimized-images": "^2.6.1",
     "nookies": "^2.3.2",
     "nprogress": "^0.2.0",
+    "p-all": "^3.0.0",
     "preact": "^10.4.4",
     "preact-render-to-string": "^5.1.4",
     "preact-ssr-prepass": "^1.1.0",
@@ -55,7 +59,6 @@
     "react-live": "^2.2.2",
     "react-simple-code-editor": "^0.11.0",
     "react-virtualized-auto-sizer": "^1.0.2",
-    "react-waypoint": "^9.0.3",
     "react-window": "^1.8.5",
     "remark": "^12.0.1",
     "remark-emoji": "2.1.0",
@@ -64,21 +67,26 @@
     "remark-frontmatter": "^2.0.0",
     "remark-images": "2.0.0",
     "remark-normalize-headings": "^2.0.0",
+    "remark-parse": "^8.0.3",
     "remark-slug": "6.0.0",
     "remark-squeeze-paragraphs": "^4.0.0",
     "remark-unwrap-images": "2.0.0",
+    "remark-vscode": "^1.0.0-beta.1",
     "store": "^2.0.12",
     "strip-markdown": "^3.1.2",
     "swr": "^0.2.3",
     "touchable-hook": "^1.3.0",
     "turndown": "^6.0.0",
     "typescript": "^3.9.7",
+    "unist-builder": "^2.0.3",
     "unist-util-is": "^4.0.2",
+    "unist-util-select": "^3.0.1",
     "unist-util-visit": "^2.0.3",
     "use-events": "^1.4.2",
     "webpack": "^4.43.0"
   },
   "devDependencies": {
+    "@babel/preset-react": "^7.10.4",
     "@blockstack/eslint-config": "^1.0.5",
     "@blockstack/prettier-config": "^0.0.6",
     "@next/bundle-analyzer": "^9.4.4",
@@ -92,10 +100,10 @@
   },
   "private": true,
   "scripts": {
-    "build": "yarn clean:build-files && next telemetry disable && NODE_ENV=production next build && next export -o build",
+    "build": "yarn clean:build-files && next telemetry disable && NODE_ENV=production next build",
     "build:analyze": "yarn clean:build-files && next telemetry disable && NODE_ENV=production ANALYZE=true next build",
     "start": "next telemetry disable && NODE_ENV=production next start",
-    "clean:build-files": "rimraf .next",
+    "clean:build-files": "rimraf .next && rimraf .cache",
     "dev": "yarn clean:build-files && next dev",
     "export": "next export",
     "lint": "yarn lint:eslint && yarn lint:prettier",

package.json

@@ -1,22 +1,86 @@
-import { convertRemoteDataToMDX } from '@common/data/mdx';
+import { renderMdx } from '@common/data/mdx';
 import CLARITY_REFERENCE from '../../_data/clarityRef.json';
 
-export const convertClarityRefUsageToMdx = async () => {
-  const [_functions, _keywords] = await Promise.all([
-    convertRemoteDataToMDX(CLARITY_REFERENCE.functions, 'description'),
-    convertRemoteDataToMDX(CLARITY_REFERENCE.keywords, 'description'),
-  ]);
+const wrapInClarityTicks = (string: string) => {
+  let newString = '';
+  newString += '```clarity\n';
+  newString += string.trim() + '\n';
+  newString += '```';
+  return newString;
+};
 
-  const functions = CLARITY_REFERENCE.functions.map((fn, index) => ({
-    ...fn,
-    description: _functions[index],
-  }));
+const inlineCode = (string: string) => {
+  let newString = '';
+  newString += '`';
+  newString += string.trim();
+  newString += '`';
+  return newString;
+};
+
+const generateMarkdown = () => {
+  let keywords = '';
+  let functions = '';
+
+  CLARITY_REFERENCE.functions.forEach(entry => {
+    functions += `### ${entry.name}
+
+**Signature:** ${inlineCode(entry.signature)}\n
+
+
+**Input:** ${inlineCode(entry.input_type)}\n
+
+
+**Output:** ${inlineCode(entry.output_type)}\n
+
+${entry.description}
+
+#### Example
+
+${wrapInClarityTicks(entry.example)}
+`;
+  });
 
-  const keywords = CLARITY_REFERENCE.keywords.map((fn, index) => ({
-    ...fn,
-    description: _keywords[index],
+  CLARITY_REFERENCE.keywords.forEach(entry => {
+    keywords += `### ${entry.name}
+
+**Output:** ${inlineCode(entry.output_type)}
+
+${entry.description}
+
+#### Example
+
+${wrapInClarityTicks(entry.example)}
+`;
+  });
+
+  return {
+    keywords,
+    functions,
+  };
+};
+
+const getHeadings = arr =>
+  arr.map(entry => ({
+    content: entry.name,
+    level: 1,
   }));
 
+export const convertClarityRefToMdx = async () => {
+  const markdown = generateMarkdown();
+  const [_functions, _keywords] = await Promise.all([
+    renderMdx(markdown.functions),
+    renderMdx(markdown.keywords),
+  ]);
+
+  const functions = {
+    content: _functions,
+    headings: getHeadings(CLARITY_REFERENCE.functions),
+  };
+  const keywords = {
+    content: _keywords,
+    headings: getHeadings(CLARITY_REFERENCE.keywords),
+  };
+
   return {
     props: {
       mdx: {

src/common/data/clarity-ref.ts

@@ -2,7 +2,13 @@ import { convertRemoteDataToMDX } from '@common/data/mdx';
 import { cliReferenceData } from '../../_data/cliRef';
 
 export const convertBlockstackCLIUsageToMdx = async () => {
-  const results = await convertRemoteDataToMDX(cliReferenceData, 'usage');
+  const transformed = cliReferenceData.map(entry => {
+    return {
+      ...entry,
+    };
+  });
+  const results = await convertRemoteDataToMDX(transformed, 'usage');
+
   return {
     props: {
       mdx: results,