cleanup

.gitea/workflows/build-and-pubilsh.yaml

@@ -0,0 +1,36 @@
+
+name: Build and publish
+
+on:
+- workflow_dispatch
+  # push:
+  #   branches:
+  #   - master
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    env:
+      NPM_PUBLISH_TOKEN: ${{ secrets.NPM_TOKEN }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Use Node.js 20
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Login to package registry
+        run: |
+          npm config set @doc-utils:registry https://gitea.jbrumond.me/api/packages/doc-utils/npm/
+          npm config set -- '//gitea.jbrumond.me/api/packages/doc-utils/npm/:_authToken' "$NPM_PUBLISH_TOKEN"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Compile TypeScript
+        run: npm run tsc
+
+      - name: Publish package
+        run: npm publish

.gitea/workflows/build-and-test.yaml

@@ -0,0 +1,40 @@
+
+name: Build and test
+
+on:
+  push:
+    branches:
+    - master
+  pull_request:
+    branches:
+    - master
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [18.x, 20.x]
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Login to package registry
+        run: |
+          npm config set @doc-utils:registry https://gitea.jbrumond.me/api/packages/doc-utils/npm/
+          npm config set -- '//gitea.jbrumond.me/api/packages/doc-utils/npm/:_authToken' "$NPM_PUBLISH_TOKEN"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Compile TypeScript
+        run: npm run tsc
+
+      # todo: tests
+      - name: Run tests
+        run: exit 0

.npmignore

@@ -0,0 +1,2 @@
+src
+node_modules

bin/markdown2html

@@ -1,2 +1,2 @@
 #!/bin/sh
-node $(dirname $0)/markdown2html.js $*
+node $(dirname "$(readlink -f "$0")")/markdown2html.js $*

bin/strip-frontmatter

@@ -1,2 +1,2 @@
 #!/bin/sh
-node $(dirname $0)/strip-frontmatter.js $*
+node $(dirname "$(readlink -f "$0")")/strip-frontmatter.js $*

package-lock.json

@@ -1,16 +1,18 @@
 {
-  "name": "markdown-to-html",
+  "name": "@doc-utils/markdown2html",
+  "version": "0.3.6",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
-      "name": "markdown-to-html",
+      "name": "@doc-utils/markdown2html",
+      "version": "0.3.6",
       "dependencies": {
         "bytefield-svg": "^1.6.1",
         "dompurify": "^2.3.6",
         "jsdom": "^20.0.1",
         "katex": "^0.16.7",
-        "marked": "^4.1.1",
+        "marked": "^5.0.2",
         "nomnoml": "^1.5.2",
         "pikchr": "^0.0.5",
         "prismjs": "^1.29.0",
@@ -18,18 +20,20 @@
         "vega": "^5.22.1",
         "yaml": "^2.2.2"
       },
+      "bin": {
+        "markdown2html": "bin/markdown2html"
+      },
       "devDependencies": {
         "@types/dompurify": "^2.3.3",
         "@types/jsdom": "^20.0.0",
         "@types/katex": "^0.16.0",
         "@types/luxon": "^3.1.0",
-        "@types/marked": "^4.0.3",
+        "@types/marked": "^5.0.0",
         "@types/node": "^18.11.18",
         "@types/prismjs": "^1.26.0",
         "@types/qrcode": "^1.5.0",
         "typescript": "^5.0.4"
-      },
+      }
-      "version": "0.1.1"
     },
     "node_modules/@tootallnate/once": {
       "version": "2.0.0",
@@ -82,15 +86,15 @@
       "dev": true
     },
     "node_modules/@types/marked": {
-      "version": "4.0.8",
+      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/@types/marked/-/marked-4.0.8.tgz",
+      "resolved": "https://registry.npmjs.org/@types/marked/-/marked-5.0.0.tgz",
-      "integrity": "sha512-HVNzMT5QlWCOdeuBsgXP8EZzKUf0+AXzN+sLmjvaB3ZlLqO+e4u0uXrdw9ub69wBKFs+c6/pA4r9sy6cCDvImw==",
+      "integrity": "sha512-YcZe50jhltsCq7rc9MNZC/4QB/OnA2Pd6hrOSTOFajtabN+38slqgDDCeE/0F83SjkKBQcsZUj7VLWR0H5cKRA==",
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "18.16.3",
+      "version": "18.16.6",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.6.tgz",
-      "integrity": "sha512-OPs5WnnT1xkCBiuQrZA4+YAV4HEJejmHneyraIaxsbev5yCEr6KMwINNFP9wQeFIw8FWcoTqF3vQsa5CDaI+8Q==",
+      "integrity": "sha512-N7KINmeB8IN3vRR8dhgHEp+YpWvGFcpDoh5XZ8jB5a00AdFKCKEyyGTOPTddUf4JqU1ZKTVxkOxakDvchNVI2Q==",
       "dev": true
     },
     "node_modules/@types/prismjs": {
@@ -950,14 +954,14 @@
       "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
     },
     "node_modules/marked": {
-      "version": "4.3.0",
+      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-5.0.2.tgz",
-      "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==",
+      "integrity": "sha512-TXksm9GwqXCRNbFUZmMtqNLvy3K2cQHuWmyBDLOrY1e6i9UvZpOTJXoz7fBjYkJkaUFzV9hBFxMuZSyQt8R6KQ==",
       "bin": {
         "marked": "bin/marked.js"
       },
       "engines": {
-        "node": ">= 12"
+        "node": ">= 18"
       }
     },
     "node_modules/mime-db": {
@@ -985,9 +989,9 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "node_modules/node-fetch": {
-      "version": "2.6.9",
+      "version": "2.6.11",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.11.tgz",
-      "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==",
+      "integrity": "sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w==",
       "dependencies": {
         "whatwg-url": "^5.0.0"
       },
@@ -2011,15 +2015,15 @@
       "dev": true
     },
     "@types/marked": {
-      "version": "4.0.8",
+      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/@types/marked/-/marked-4.0.8.tgz",
+      "resolved": "https://registry.npmjs.org/@types/marked/-/marked-5.0.0.tgz",
-      "integrity": "sha512-HVNzMT5QlWCOdeuBsgXP8EZzKUf0+AXzN+sLmjvaB3ZlLqO+e4u0uXrdw9ub69wBKFs+c6/pA4r9sy6cCDvImw==",
+      "integrity": "sha512-YcZe50jhltsCq7rc9MNZC/4QB/OnA2Pd6hrOSTOFajtabN+38slqgDDCeE/0F83SjkKBQcsZUj7VLWR0H5cKRA==",
       "dev": true
     },
     "@types/node": {
-      "version": "18.16.3",
+      "version": "18.16.6",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.3.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.6.tgz",
-      "integrity": "sha512-OPs5WnnT1xkCBiuQrZA4+YAV4HEJejmHneyraIaxsbev5yCEr6KMwINNFP9wQeFIw8FWcoTqF3vQsa5CDaI+8Q==",
+      "integrity": "sha512-N7KINmeB8IN3vRR8dhgHEp+YpWvGFcpDoh5XZ8jB5a00AdFKCKEyyGTOPTddUf4JqU1ZKTVxkOxakDvchNVI2Q==",
       "dev": true
     },
     "@types/prismjs": {
@@ -2647,9 +2651,9 @@
       "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
     },
     "marked": {
-      "version": "4.3.0",
+      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-5.0.2.tgz",
-      "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A=="
+      "integrity": "sha512-TXksm9GwqXCRNbFUZmMtqNLvy3K2cQHuWmyBDLOrY1e6i9UvZpOTJXoz7fBjYkJkaUFzV9hBFxMuZSyQt8R6KQ=="
     },
     "mime-db": {
       "version": "1.52.0",
@@ -2670,9 +2674,9 @@
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
     "node-fetch": {
-      "version": "2.6.9",
+      "version": "2.6.11",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.11.tgz",
-      "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==",
+      "integrity": "sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w==",
       "requires": {
         "whatwg-url": "^5.0.0"
       },
@@ -3482,6 +3486,5 @@
         "decamelize": "^1.2.0"
       }
     }
-  },
+  }
-  "version": "0.1.1"
 }

package.json

@@ -1,22 +1,23 @@
 {
   "name": "@doc-utils/markdown2html",
-  "version": "0.1.1",
+  "version": "0.3.6",
   "publishConfig": {
     "registry": "https://gitea.home.jbrumond.me/api/packages/doc-utils/npm/"
   },
   "scripts": {
-    "tsc": "tsc"
+    "tsc": "tsc",
+    "clean": "rm -rf ./build"
   },
   "bin": {
     "markdown2html": "./bin/markdown2html"
   },
-  "exports": "./build/index.js",
+  "main": "./build/index.js",
   "devDependencies": {
     "@types/dompurify": "^2.3.3",
     "@types/jsdom": "^20.0.0",
     "@types/katex": "^0.16.0",
     "@types/luxon": "^3.1.0",
-    "@types/marked": "^4.0.3",
+    "@types/marked": "^5.0.0",
     "@types/node": "^18.11.18",
     "@types/prismjs": "^1.26.0",
     "@types/qrcode": "^1.5.0",
@@ -27,7 +28,7 @@
     "dompurify": "^2.3.6",
     "jsdom": "^20.0.1",
     "katex": "^0.16.7",
-    "marked": "^4.1.1",
+    "marked": "^5.0.2",
     "nomnoml": "^1.5.2",
     "pikchr": "^0.0.5",
     "prismjs": "^1.29.0",

readme.md

@@ -22,8 +22,11 @@ These are the primary projects used to render various types of content:
 # Update project npm config to refer to correct registry for the @doc-utils scope
 echo '@doc-utils:registry=https://gitea.jbrumond.me/api/packages/doc-utils/npm/' >> ./.npmrc
 
-# Install package
+# Install package for programatic use
 npm install --save @doc-utils/markdown2html
+
+# Install globally for CLI usage
+npm install --global @doc-utils/markdown2html
 ```
 
 
@@ -50,13 +53,13 @@ async function main() {
 }
 ```
 
-### Handling Frontmatter
+### Handling front matter
 
 ```ts
 import { process_frontmatter } from '@doc-utils/markdown2html';
 
 const raw_content = `---
-title: Example Markdown with Frontmatter
+title: Example Markdown with front matter
 foo:
 - bar
 - baz
@@ -65,11 +68,11 @@ foo:
 # This is some markdown
 `;
 
-const { frontmatter, markdown } = process_frontmatter(raw_content);
+const { frontmatter, document } = process_frontmatter(raw_content);
 
-console.log(frontmatter.title);  // "Example Markdown with Frontmatter"
+console.log(frontmatter.title);  // "Example Markdown with front matter"
 console.log(frontmatter.foo);    // [ "bar", "baz" ]
-console.log(markdown);           // "\n# This is some markdown\n"
+console.log(document);           // "\n# This is some markdown\n"
 ```
 
 
@@ -80,11 +83,11 @@ console.log(markdown);           // "\n# This is some markdown\n"
 echo '# This is some markdown' | ./bin/markdown2html --base-url 'https://example.com' > output.html
 ```
 
-### Handling Frontmatter
+### Handling front matter
 
 ```bash
 filecontents="---
-title: Example Markdown with Frontmatter
+title: Example Markdown with front matter
 foo:
 - bar
 - baz

src/attrs.ts

@@ -88,7 +88,5 @@ export function parse_attributes(text: string, fallback_id?: Lazy<string>) {
 		}
 	}
 
-	console.log(attrs);
-
 	return attrs;
 }

src/base-url.ts

@@ -0,0 +1,21 @@
+
+import { marked } from 'marked';
+import { MarkdownOptions } from './render';
+
+export const placeholder_base_url = 'https://markdown2html.base-url.placeholder.invalid';
+
+export function base_url_walk_tokens(token: marked.Token, options: MarkdownOptions) {
+	if (options.base_url) {
+		const base_url = options.base_url.startsWith('http://') || options.base_url.startsWith('https://')
+			? options.base_url
+			: placeholder_base_url + options.base_url;
+	
+		if (token.type === 'link' || token.type === 'image') {
+			token.href = (new URL(token.href, base_url)).toString();
+	
+			if (token.href.startsWith(placeholder_base_url)) {
+				token.href = token.href.slice(placeholder_base_url.length);
+			}
+		}
+	}
+}

src/description-list.ts

@@ -3,14 +3,19 @@ import { marked } from 'marked';
 import { MarkdownOptions } from './render';
 
 export interface DescriptionListToken extends marked.Tokens.Generic {
+	type: 'description_list';
 	items: (DescriptionTermToken | DescriptionDetailToken)[];
 }
 
+export type DescriptionElemToken = DescriptionTermToken | DescriptionDetailToken;
+
 export interface DescriptionTermToken extends marked.Tokens.Generic {
+	type: 'description_term';
 	text: string;
 }
 
 export interface DescriptionDetailToken extends marked.Tokens.Generic {
+	type: 'description_detail';
 	text: string;
 }
 
@@ -18,72 +23,106 @@ export function description_list_ext(renderer: marked.Renderer, opts: MarkdownOp
 	return {
 		name: 'description_list',
 		level: 'block',
-		start: (src) => src.match(/^:[:#-]/)?.index,
+		start: (src) => src.match(/^: /)?.index,
 		tokenizer(src, tokens) {
-			const rule = /^(?::[:#-](?:\s[^\n]*)?(?:\n|$))+/;
+			const start = src.match(/^: /)?.index;
-			const match = rule.exec(src);
+			const lines = src.slice(start).split(/\n/g);
+			const token: DescriptionListToken = {
+				type: 'description_list',
+				raw: '',
+				items: [ ]
+			};
 
-			if (match) {
+			let current: DescriptionElemToken;
-				const token: DescriptionListToken = {
+			const render_current = () => {
-					type: 'description_list',
+				if (current) {
-					raw: match[0],
+					this.lexer.blockTokens(current.text, current.tokens);
-					items: [ ]
+					current = null;
-				};
+				}
+			};
 
-				const items = token.raw.trim().split('\n');
+			for (const line of lines) {
-				const raw_buffer: string[] = [ ];
+				// Skip empty lines
-				const text_buffer: string[] = [ ];
+				if (! line.trim()) {
+					token.raw += line + '\n';
 
-				const flush_buffer = () => {
+					if (current) {
-					if (! raw_buffer.length) {
+						current.raw += line + '\n';
-						return;
+						current.text += '\n';
 					}
 
-					// Grab the second character from the first line to determine the
+					continue;
-					// token type (should be "#" or "-")
-					const type = raw_buffer[0][1] === '#' ? 'description_term' : 'description_detail';
-					
-					const sub_token: (DescriptionTermToken | DescriptionDetailToken) = {
-						type,
-						raw: raw_buffer.join('\n'),
-						text: text_buffer.join('\n'),
-						tokens: [ ],
-					};
-
-					raw_buffer.length = 0;
-					text_buffer.length = 0;
-
-					this.lexer.blockTokens(sub_token.text, sub_token.tokens);
-					token.items.push(sub_token);
-				};
-
-				for (const line of items) {
-					const rule = /^:([:#-])(?:\s([^\n]*))?(?:\n|$)/;
-					const match = rule.exec(line);
-		
-					if (match) {
-						if (match[1] !== ':') {
-							flush_buffer();
-						}
-
-						raw_buffer.push(match[0]);
-						text_buffer.push(match[2]);
-					}
 				}
 
-				flush_buffer();
+				// If the line starts immediately with a colon, it is a <dt>
-				
+				if (line.startsWith(': ')) {
+					render_current();
+					token.raw += line + '\n';
+					token.items.push(
+						current = {
+							type: 'description_term',
+							raw: line,
+							text: line.slice(2),
+							tokens: [ ],
+						}
+					);
+					continue;
+				}
+
+				// If the line starts with a colon after an indent, it is a <dd>
+				if (line.startsWith('  : ')) {
+					render_current();
+					token.raw += line + '\n';
+					token.items.push(
+						current = {
+							type: 'description_detail',
+							raw: line,
+							text: line.slice(4),
+							tokens: [ ],
+						}
+					);
+					continue;
+				}
+
+				// If the line starts with (at least) two indents, it is a child
+				// of the current element
+				if (line.startsWith('    ')) {
+					token.raw += line + '\n';
+					current.raw += '\n' + line;
+					current.text += '\n' + line.slice(current.type === 'description_term' ? 2 : 4);
+					continue;
+				}
+
+				// If the line starts with one indent, it is a child of the current
+				// <dt> (but is not allowed after a <dd>)
+				if (line.startsWith('  ')) {
+					if (current.type !== 'description_term') {
+						render_current();
+						break;
+					}
+
+					token.raw += line + '\n';
+					current.raw += '\n' + line;
+					current.text += '\n' + line.slice(2);
+					continue;
+				}
+
+				// If the line starts any other way, it is the start of new content
+				// and we are done parsing
+				render_current();
+				break;
+			}
+			
+			render_current();
+
+			if (token.items.length) {
 				return token;
 			}
 		},
 		renderer(token: DescriptionListToken) {
 			const items = token.items.map((item) => {
 				const tag = item.type === 'description_term' ? 'dt' : 'dd';
-				return `
+				return `<${tag}>${this.parser.parse(item.tokens)}</${tag}>`;
-					<${tag}>
-						${this.parser.parse(item.tokens)}
-					</${tag}>
-				`;
 			});
 
 			return `<dl>${items.join('')}</dl>`;

src/embed.ts

@@ -0,0 +1,33 @@
+
+import type { marked } from 'marked';
+import { MarkdownOptions } from './render';
+
+export interface EmbedToken extends marked.Tokens.Generic {
+	media_type: string;
+	href: string;
+}
+
+export function embed_ext(renderer: marked.Renderer, opts: MarkdownOptions) : marked.TokenizerExtension & marked.RendererExtension {
+	return {
+		name: 'embed',
+		level: 'inline',
+		start: (src) => src.match(/\{&/)?.index,
+		tokenizer(src, tokens) {
+			const rule = /^\{&\s+([^\s]+):\s*([^\s]+)\s+&\}/;
+			const match = rule.exec(src);
+
+			if (match) {
+				return {
+					type: 'embed',
+					raw: match[0],
+					media_type: match[1],
+					href: match[2],
+					tokens: [ ]
+				};
+			}
+		},
+		renderer(token: EmbedToken) {
+			return `<embed type="${token.media_type}" src="${token.href}"></embed>`;
+		}
+	};
+}

src/footnotes.ts

@@ -23,7 +23,7 @@ export function footnote_ref_ext(renderer: marked.Renderer, opts: MarkdownOption
 		level: 'inline',
 		start: (src) => src.match(/\[\^/)?.index,
 		tokenizer(src, tokens) {
-			const rule = /^\[\^([a-zA-Z0-9-._§]+)]/;
+			const rule = /^\[\^([a-zA-Z0-9-\._, §]+)]/;
 			const match = rule.exec(src);
 
 			if (match) {
@@ -68,7 +68,7 @@ export function footnote_list_ext(renderer: marked.Renderer, opts: MarkdownOptio
 			};
 
 			let remaining = src;
-			const prefix_rule = /^\[\^([a-zA-Z0-9\._§-]+)]:/;
+			const prefix_rule = /^\[\^([a-zA-Z0-9\., _§-]+)]:/;
 			const whitespace_rule = /^\s*(?:\n|$)/;
 
 			if (! prefix_rule.test(src)) {

src/frontmatter.ts

@@ -1,18 +1,26 @@
 
 import * as yaml from 'yaml';
 
-export function process_frontmatter(document: string, parse = true) {
+export function process_frontmatter(raw_document: string, parse = true) {
-	if (! document.startsWith('---\n')) {
+	if (! raw_document.startsWith('---\n')) {
 		return {
 			frontmatter: null,
-			markdown: document,
+			document: raw_document,
 		};
 	}
 
-	const endIndex = document.slice(3).indexOf('\n---\n') + 3;
+	const end_index = raw_document.slice(3).indexOf('\n---\n') + 3;
-	const frontmatterYaml = document.slice(3, endIndex);
+	
-	const markdown = document.slice(endIndex + 4);
+	if (end_index === 2) {
-	const frontmatter = parse ? frontmatterYaml : yaml.parse(frontmatterYaml);
+		return {
+			frontmatter: null,
+			document: raw_document,
+		};
+	}
 
-	return { frontmatter, markdown };
+	const formatter_yaml = raw_document.slice(3, end_index);
+	const document = raw_document.slice(end_index + 4);
+	const frontmatter = parse ? yaml.parse(formatter_yaml) : formatter_yaml;
+
+	return { frontmatter, document };
 }

src/html-sanitize.ts

@@ -2,13 +2,13 @@
 import { JSDOM } from 'jsdom';
 import createDOMPurify = require('dompurify');
 
-export function sanitize_html(html: string) : string {
+export type CustomElementHandling = createDOMPurify.Config['CUSTOM_ELEMENT_HANDLING'];
+
+export function sanitize_html(html: string, custom_elements?: CustomElementHandling) : string {
 	const { window } = new JSDOM('');
 	const dom_purify = createDOMPurify(window as any as Window);
 	return dom_purify.sanitize(html, {
-		CUSTOM_ELEMENT_HANDLING: {
+		CUSTOM_ELEMENT_HANDLING: custom_elements,
-			tagNameCheck: (tag_name) => tag_name === 'svg-icon',
+		ADD_TAGS: [ 'embed', 'object', 'video', 'audio', 'iframe' ],
-			attributeNameCheck: (attr_name) => attr_name === 'icon',
-		}
 	});
 }

src/index.ts

@@ -1,3 +1,3 @@
 
 export { process_frontmatter } from './frontmatter';
-export { render_markdown_to_html, MarkdownExtension, MarkdownOptions } from './render';
+export { render_markdown_to_html, render_markdown_to_html_inline_sync, MarkdownExtension, MarkdownOptions } from './render';

src/pikchr.ts

@@ -0,0 +1,150 @@
+
+import { strip_svg } from './svg';
+
+export function post_process_pikchr_svg(svg: string, size?: string) {
+	svg = strip_svg(svg);
+
+	// text
+	for (const [ color, regex ] of Object.entries(text_regex)) {
+		svg = svg.replace(regex, (_, $1) => `<text${$1}fill="${text_css_vars[color]}"`);
+	}
+
+	// shape fill
+	for (const [ color, regex ] of Object.entries(shape_fill_regex)) {
+		svg = svg.replace(regex, (_, $1) => `<path${$1}style="fill:${fill_css_vars[color]}`);
+	}
+
+	// arrow heads
+	for (const [ color, regex ] of Object.entries(arrow_head_regex)) {
+		svg = svg.replace(regex, `fill:${line_css_vars[color]}`);
+	}
+
+	// lines / boxes
+	for (const [ color, regex ] of Object.entries(line_regex)) {
+		svg = svg.replace(regex, `stroke:${line_css_vars[color]}`);
+	}
+
+	return svg;
+}
+
+
+
+type CSSVarColor
+	= 'black'
+	| 'red'
+	| 'orange'
+	| 'yellow'
+	| 'green'
+	| 'teal'
+	| 'pink'
+	| 'purple'
+	| 'blue'
+	| 'indigo'
+	| 'magenta'
+	| 'brown'
+	;
+
+const text_regex: Record<CSSVarColor, RegExp> = {
+	black:   /<text( | [^>]+ )fill="rgb\(0,0,0\)"/gi,
+	red:     /<text( | [^>]+ )fill="rgb\(255,0,0\)"/gi,
+	orange:  /<text( | [^>]+ )fill="rgb\(255,165,0\)"/gi,
+	yellow:  /<text( | [^>]+ )fill="rgb\(255,255,0\)"/gi,
+	green:   /<text( | [^>]+ )fill="rgb\(0,128,0\)"/gi,
+	teal:    /<text( | [^>]+ )fill="rgb\(0,128,128\)"/gi,
+	pink:    /<text( | [^>]+ )fill="rgb\(255,192,203\)"/gi,
+	purple:  /<text( | [^>]+ )fill="rgb\(128,0,128\)"/gi,
+	blue:    /<text( | [^>]+ )fill="rgb\(0,0,255\)"/gi,
+	indigo:  /<text( | [^>]+ )fill="rgb\(75,0,130\)"/gi,
+	magenta: /<text( | [^>]+ )fill="rgb\(255,0,255\)"/gi,
+	brown:   /<text( | [^>]+ )fill="rgb\(165,42,42\)"/gi,
+};
+
+const line_regex: Record<CSSVarColor, RegExp> = {
+	black:   /stroke:rgb\(0,0,0\)/gi,
+	red:     /stroke:rgb\(255,0,0\)/gi,
+	orange:  /stroke:rgb\(255,165,0\)/gi,
+	yellow:  /stroke:rgb\(255,255,0\)/gi,
+	green:   /stroke:rgb\(0,128,0\)/gi,
+	teal:    /stroke:rgb\(0,128,128\)/gi,
+	pink:    /stroke:rgb\(255,192,203\)/gi,
+	purple:  /stroke:rgb\(128,0,128\)/gi,
+	blue:    /stroke:rgb\(0,0,255\)/gi,
+	indigo:  /stroke:rgb\(75,0,130\)/gi,
+	magenta: /stroke:rgb\(255,0,255\)/gi,
+	brown:   /stroke:rgb\(165,42,42\)/gi,
+};
+
+const shape_fill_regex = {
+	black:    /<path( | [^>]+ )style="fill:rgb\(0,0,0\)/gi,
+	red:      /<path( | [^>]+ )style="fill:rgb\(255,0,0\)/gi,
+	orange:   /<path( | [^>]+ )style="fill:rgb\(255,165,0\)/gi,
+	yellow:   /<path( | [^>]+ )style="fill:rgb\(255,255,0\)/gi,
+	green:    /<path( | [^>]+ )style="fill:rgb\(0,128,0\)/gi,
+	teal:     /<path( | [^>]+ )style="fill:rgb\(0,128,128\)/gi,
+	pink:     /<path( | [^>]+ )style="fill:rgb\(255,192,203\)/gi,
+	purple:   /<path( | [^>]+ )style="fill:rgb\(128,0,128\)/gi,
+	blue:     /<path( | [^>]+ )style="fill:rgb\(0,0,255\)/gi,
+	indigo:   /<path( | [^>]+ )style="fill:rgb\(75,0,130\)/gi,
+	magenta:  /<path( | [^>]+ )style="fill:rgb\(255,0,255\)/gi,
+	brown:    /<path( | [^>]+ )style="fill:rgb\(165,42,42\)/gi,
+}
+
+const arrow_head_regex: Record<CSSVarColor, RegExp> = {
+	black:    /fill:rgb\(0,0,0\)/gi,
+	red:      /fill:rgb\(255,0,0\)/gi,
+	orange:   /fill:rgb\(255,165,0\)/gi,
+	yellow:   /fill:rgb\(255,255,0\)/gi,
+	green:    /fill:rgb\(0,128,0\)/gi,
+	teal:     /fill:rgb\(0,128,128\)/gi,
+	pink:     /fill:rgb\(255,192,203\)/gi,
+	purple:   /fill:rgb\(128,0,128\)/gi,
+	blue:     /fill:rgb\(0,0,255\)/gi,
+	indigo:   /fill:rgb\(75,0,130\)/gi,
+	magenta:  /fill:rgb\(255,0,255\)/gi,
+	brown:    /fill:rgb\(165,42,42\)/gi,
+};
+
+const line_css_vars: Record<CSSVarColor, string> = {
+	black:    'var(--theme-line, currentcolor)',
+	red:      'var(--theme-chart-shape-red-line, currentcolor)',
+	orange:   'var(--theme-chart-shape-orange-line, currentcolor)',
+	yellow:   'var(--theme-chart-shape-yellow-line, currentcolor)',
+	green:    'var(--theme-chart-shape-green-line, currentcolor)',
+	teal:     'var(--theme-chart-shape-teal-line, currentcolor)',
+	pink:     'var(--theme-chart-shape-pink-line, currentcolor)',
+	purple:   'var(--theme-chart-shape-purple-line, currentcolor)',
+	blue:     'var(--theme-chart-shape-blue-line, currentcolor)',
+	indigo:   'var(--theme-chart-shape-indigo-line, currentcolor)',
+	magenta:  'var(--theme-chart-shape-magenta-line, currentcolor)',
+	brown:    'var(--theme-chart-shape-brown-line, currentcolor)',
+};
+
+const text_css_vars: Record<CSSVarColor, string> = {
+	black:    'var(--theme-text-body, currentcolor)',
+	red:      'var(--theme-chart-shape-red-line, currentcolor)',
+	orange:   'var(--theme-chart-shape-orange-line, currentcolor)',
+	yellow:   'var(--theme-chart-shape-yellow-line, currentcolor)',
+	green:    'var(--theme-chart-shape-green-line, currentcolor)',
+	teal:     'var(--theme-chart-shape-teal-line, currentcolor)',
+	pink:     'var(--theme-chart-shape-pink-line, currentcolor)',
+	purple:   'var(--theme-chart-shape-purple-line, currentcolor)',
+	blue:     'var(--theme-chart-shape-blue-line, currentcolor)',
+	indigo:   'var(--theme-chart-shape-indigo-line, currentcolor)',
+	magenta:  'var(--theme-chart-shape-magenta-line, currentcolor)',
+	brown:    'var(--theme-chart-shape-brown-line, currentcolor)',
+};
+
+const fill_css_vars: Record<CSSVarColor, string> = {
+	black:    'var(--theme-bg-heavy, transparent)',
+	red:      'var(--theme-chart-shape-red-fill, transparent)',
+	orange:   'var(--theme-chart-shape-orange-fill, transparent)',
+	yellow:   'var(--theme-chart-shape-yellow-fill, transparent)',
+	green:    'var(--theme-chart-shape-green-fill, transparent)',
+	teal:     'var(--theme-chart-shape-teal-fill, transparent)',
+	pink:     'var(--theme-chart-shape-pink-fill, transparent)',
+	purple:   'var(--theme-chart-shape-purple-fill, transparent)',
+	blue:     'var(--theme-chart-shape-blue-fill, transparent)',
+	indigo:   'var(--theme-chart-shape-indigo-fill, transparent)',
+	magenta:  'var(--theme-chart-shape-magenta-fill, transparent)',
+	brown:    'var(--theme-chart-shape-brown-fill, transparent)',
+};

src/prism/http.ts

@@ -0,0 +1,4 @@
+
+import { languages } from 'prismjs';
+
+languages.http['request-line'].pattern = /^(?:CONNECT|DELETE|GET|HEAD|QUERY|OPTIONS|PATCH|POST|PRI|PUT|SEARCH|TRACE)\s(?:https?:\/\/|\/)\S*\sHTTP\/[\d.]+/m;

src/prism/index.ts

@@ -4,6 +4,7 @@ import load_languages = require('prismjs/components/index');
 
 load_languages();
 require('./wasm');
+require('./http');
 
 // hooks.add('after-tokenize', (env) => {
 // 	// 

src/render.ts

@@ -4,11 +4,13 @@ import { create_renderer } from './renderer';
 import { mark_ext } from './mark';
 import { section_ext } from './section';
 import { icon_ext } from './icon';
-import { sanitize_html } from './html-sanitize';
+import { CustomElementHandling, sanitize_html } from './html-sanitize';
 import { katex_block_ext, katex_inline_ext } from './katex';
 import { footnote_list_ext, footnote_ref_ext } from './footnotes';
 import { description_list_ext } from './description-list';
 import { resolve_async_bindings } from './async-steps';
+import { base_url_walk_tokens } from './base-url';
+import { embed_ext } from './embed';
 
 export interface MarkdownOptions {
 	base_url?: string;
@@ -16,6 +18,7 @@ export interface MarkdownOptions {
 	inline?: boolean;
 	katex_macros?: Record<string, string>;
 	extensions?: MarkdownExtension[];
+	custom_elements?: CustomElementHandling;
 }
 
 export interface MarkdownExtension {
@@ -23,13 +26,36 @@ export interface MarkdownExtension {
 }
 
 export async function render_markdown_to_html(markdown: string, options: MarkdownOptions = { }) {
-	const marked_options: marked.MarkedOptions = {
+	const marked_options = marked_opts(options);
-		baseUrl: options.base_url,
+	setup_marked(options, marked_options);
+	const unsafe_html = options.inline
+		? marked.parseInline(markdown, marked_options)
+		: marked.parse(markdown, marked_options);
+
+	return sanitize_html(await resolve_async_bindings(unsafe_html), options.custom_elements);
+}
+
+export function render_markdown_to_html_inline_sync(markdown: string, options: MarkdownOptions = { }) {
+	const marked_options = marked_opts(options);
+	setup_marked(options, marked_options);
+	const unsafe_html = marked.parseInline(markdown, marked_options);
+	return sanitize_html(unsafe_html, options.custom_elements);
+}
+
+function marked_opts<T extends boolean>(options: MarkdownOptions) : marked.MarkedOptions {
+	return {
 		breaks: options.breaks || false,
 		renderer: create_renderer(options),
+		mangle: false,
+		headerIds: false,
 	};
+}
 
+function setup_marked(options: MarkdownOptions, marked_options: marked.MarkedOptions) {
 	marked.use({
+		walkTokens(token) {
+			base_url_walk_tokens(token, options);
+		},
 		extensions: [
 			katex_block_ext(marked_options.renderer, options),
 			katex_inline_ext(marked_options.renderer, options),
@@ -39,6 +65,7 @@ export async function render_markdown_to_html(markdown: string, options: Markdow
 			description_list_ext(marked_options.renderer, options),
 			section_ext(marked_options.renderer, options),
 			icon_ext(marked_options.renderer, options),
+			embed_ext(marked_options.renderer, options),
 			...(options.extensions || [ ]).map((ext) => {
 				return ext(marked_options.renderer, options);
 			}),
@@ -51,18 +78,4 @@ export async function render_markdown_to_html(markdown: string, options: Markdow
 			}
 		},
 	});
-	
-	const unsafe_html = options.inline
-		? marked.parseInline(markdown, marked_options)
-		: await new Promise<string>((resolve, reject) => {
-			marked.parse(markdown, marked_options, (error, unsafe_html) => {
-				if (error) {
-					return reject(error);
-				}
-
-				resolve_async_bindings(unsafe_html).then(resolve, reject);
-			});
-		});
-
-	return sanitize_html(unsafe_html);
 }

src/renderer.ts

@@ -14,6 +14,7 @@ import { bind_data_async } from './async-steps';
 import { render_vega_spec } from './vega';
 import { parse_attributes } from './attrs';
 import { MarkdownOptions } from './render';
+import { post_process_pikchr_svg } from './pikchr';
 
 export function create_renderer(opts: MarkdownOptions) {
 	const renderer = new marked.Renderer();
@@ -27,23 +28,16 @@ export function create_renderer(opts: MarkdownOptions) {
 }
 
 function heading(renderer: marked.Renderer, opts: MarkdownOptions) {
-	return function(orig_text: string, level: 1 | 2 | 3 | 4 | 5 | 6, raw: string, slugger) {
+	return function(orig_text: string, level: 1 | 2 | 3 | 4 | 5 | 6, raw: string) {
-		let { text, id, html_attrs } = parse_attributes(raw);
+		let { text, id, html_attrs } = parse_attributes(orig_text);
-	
+		if (id) {
-		if (! id) {
+			text += `\n<a class="heading-anchor" href="#${id}">`
-			id = slugger.slug(text);
+				+ `\n\t\t${icons.link}`
-			html_attrs.push(`id="${id}"`);
+				+ `\n\t\t<span style="display: none">Section titled ${text}</span>`
+				+ `\n\t</a>`;
 		}
-		
+
-		return `
+		return `\n<h${level} ${html_attrs.join(' ')}>\n\t${text}\n</h${level}>\n`;
-<h${level} ${html_attrs.join(' ')}>
-	${text}
-	<a class="heading-anchor" href="#${id}">
-		${icons.link}
-		<span style="display: none">Section titled ${text}</span>
-	</a>
-</h${level}>
-`;
 	};
 }
 
@@ -89,6 +83,8 @@ function code(renderer: marked.Renderer, opts: MarkdownOptions) {
 			case 'bash:samp': {
 				// Find the first newline that is not preceeded by a "\"
 				const end_of_input = /(?<!\\)(?:\r\n|\r|\n)/.exec(code);
+
+				// todo: handling for multi-line heredocs?
 	
 				// If there is no such newline, the whole content is input
 				if (! end_of_input) {
@@ -156,6 +152,10 @@ function code(renderer: marked.Renderer, opts: MarkdownOptions) {
 				const binding = bind_data_async(promise);
 				return figure(binding);
 			};
+
+			case 'yaml:calendar': {
+				// todo
+			};
 	
 			default:
 				return figure(`<pre class="language-${args[0] || 'txt'}">${render_prism(code, args[0])}</pre>`);
@@ -194,7 +194,7 @@ function code(renderer: marked.Renderer, opts: MarkdownOptions) {
 
 const arg_pattern = /^(?:[a-zA-Z0-9_:-]+|"(?:[^"\n]|(?<=\\)")*")/;
 
-function parse_code_args(text: string) {
+function parse_code_args(text = '') {
 	const args: string[] = [ ];
 
 	text = text.trim();
@@ -275,23 +275,3 @@ function post_process_nomnoml_svg(svg: string, size?: string) {
 
 	return svg;
 }
-
-const svg_text_fill_rgb_000 = /\b<text fill="rgb\(0,0,0\)"/gi;
-const svg_fill_rgb_000 = /fill:rgb\(0,0,0\)/gi;
-const svg_stroke_rgb_000 = /stroke:rgb\(0,0,0\)/gi;
-
-// todo: css variables
-function post_process_pikchr_svg(svg: string, size?: string) {
-	svg = strip_svg(svg);
-
-	// text
-	svg = svg.replace(svg_text_fill_rgb_000, '<text fill="var(--theme-text-body, currentcolor)"');
-
-	// arrow heads
-	svg = svg.replace(svg_fill_rgb_000, 'fill:var(--theme-line, currentcolor)');
-
-	// lines / boxes
-	svg = svg.replace(svg_stroke_rgb_000, 'stroke:var(--theme-line, currentcolor)');
-
-	return svg;
-}

src/vega/data-colors.ts

@@ -1,18 +1,27 @@
 
-// note: fallback colors come from "category10" scheme
-// https://vega.github.io/vega/docs/schemes/#category10
-// todo: css variables
 export const chart_data_colors = [
-	'var(--theme-chart-data-0, #1f77b4)',
+	'var(--theme-chart-shape-red-fill, #feaea5)',
-	'var(--theme-chart-data-1, #ff7f0e)',
+	'var(--theme-chart-shape-orange-fill, #fad6bc)',
-	'var(--theme-chart-data-2, #2ca02c)',
+	'var(--theme-chart-shape-yellow-fill, #fffec6)',
-	'var(--theme-chart-data-3, #d62728)',
+	'var(--theme-chart-shape-green-fill, #d6f9d5)',
-	'var(--theme-chart-data-4, #9467bd)',
+	'var(--theme-chart-shape-teal-fill, #b0ebe9)',
-	'var(--theme-chart-data-5, #8c564b)',
+	'var(--theme-chart-shape-pink-fill, #ffcae2)',
-	'var(--theme-chart-data-6, #e377c2)',
+	'var(--theme-chart-shape-purple-fill, #efdeff)',
-	'var(--theme-chart-data-7, #7f7f7f)',
+	'var(--theme-chart-shape-blue-fill, #caebff)',
-	'var(--theme-chart-data-8, #bcbd22)',
+	'var(--theme-chart-shape-indigo-fill, #c2d6f9)',
-	'var(--theme-chart-data-9, #17becf)',
+	'var(--theme-chart-shape-magenta-fill, #ebb5cd)',
+	'var(--theme-chart-shape-brown-fill, #e1c3b8)',
+  	'var(--theme-chart-shape-red-line, #ff806d)',
+  	'var(--theme-chart-shape-orange-line, #ffb780)',
+  	'var(--theme-chart-shape-yellow-line, #e1dc18)',
+  	'var(--theme-chart-shape-green-line, #66cc66)',
+  	'var(--theme-chart-shape-teal-line, #66cdcc)',
+  	'var(--theme-chart-shape-pink-line, #ff99cb)',
+  	'var(--theme-chart-shape-purple-line, #cc99fe)',
+  	'var(--theme-chart-shape-blue-line, #66cbff)',
+  	'var(--theme-chart-shape-indigo-line, #6295ee)',
+  	'var(--theme-chart-shape-magenta-line, #cc6698)',
+  	'var(--theme-chart-shape-brown-line, #bf8c71)',
 ];
 
 export function* chart_data_color_generator() : Generator<string, never> {