add support for embed, media elements

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@doc-utils/markdown2html",
-  "version": "0.2.3",
+  "version": "0.3.4",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@doc-utils/markdown2html",
-      "version": "0.2.3",
+      "version": "0.3.4",
       "dependencies": {
         "bytefield-svg": "^1.6.1",
         "dompurify": "^2.3.6",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@doc-utils/markdown2html",
-  "version": "0.2.3",
+  "version": "0.3.4",
   "publishConfig": {
     "registry": "https://gitea.home.jbrumond.me/api/packages/doc-utils/npm/"
   },

src/breadcrumb-nav.ts

@@ -1,64 +0,0 @@
-
-import { marked } from 'marked';
-import { ParsedAttributes, parse_attributes } from './attrs';
-import { MarkdownOptions } from './render';
-
-// todo: deprecate this
-
-export interface BreadcrumbNavToken extends marked.Tokens.Generic {
-	text: string;
-	attrs: ParsedAttributes;
-	items: marked.Token[][];
-}
-
-export function breadcrumb_nav_ext(renderer: marked.Renderer, opts: MarkdownOptions) : marked.TokenizerExtension & marked.RendererExtension {
-	return {
-		name: 'breadcrumb_nav',
-		level: 'block',
-		start: (src) => src.match(/^\/\/\//)?.index,
-		tokenizer(src, tokens) {
-			const rule = /^\/\/\/(\/*)([^\n]+)?(?:\n)((?:[^\/]|\/\/?(?!\/\1))+)\/\/\/\1/;
-			const match = rule.exec(src);
-
-			if (match) {
-				const token: BreadcrumbNavToken = {
-					type: 'breadcrumb_nav',
-					raw: match[0],
-					text: match[3],
-					attrs: parse_attributes(match[2] || ''),
-					tokens: [ ],
-					items: [ ],
-				};
-
-				const lines = match[3].trim().split('\n');
-
-				for (const line of lines) {
-					const tokens = this.lexer.inlineTokens(line, [ ]);
-					token.tokens.push(...tokens);
-					token.items.push(tokens);
-				}
-
-				return token;
-			}
-		},
-		renderer(token: BreadcrumbNavToken) {
-			return `<nav aria-label="breadcrumbs" ${token.attrs.html_attrs.join(' ')}>\n`
-				+ `\t<ol>\n`
-				+ '\t\t'
-					+ token.items.map((tokens, index) =>{ 
-						let item = '<li>\n';
-
-						if (index) {
-							item += '\t\t\t<span class="separator" aria-hidden="true">/</span>\n';
-						}
-
-						item += `\t\t\t${this.parser.parseInline(tokens, renderer)}\n`;
-
-						return item + '\t\t</li>';
-					}).join('\n\t\t')
-				+ '\n'
-				+ `\t</ol>\n`
-				+ `</nav>`;
-		}
-	};
-}

src/description-list.ts

@@ -3,14 +3,19 @@ import { marked } from 'marked';
 import { MarkdownOptions } from './render';
 
 export interface DescriptionListToken extends marked.Tokens.Generic {
+	type: 'description_list';
 	items: (DescriptionTermToken | DescriptionDetailToken)[];
 }
 
+export type DescriptionElemToken = DescriptionTermToken | DescriptionDetailToken;
+
 export interface DescriptionTermToken extends marked.Tokens.Generic {
+	type: 'description_term';
 	text: string;
 }
 
 export interface DescriptionDetailToken extends marked.Tokens.Generic {
+	type: 'description_detail';
 	text: string;
 }
 
@@ -18,72 +23,106 @@ export function description_list_ext(renderer: marked.Renderer, opts: MarkdownOp
 	return {
 		name: 'description_list',
 		level: 'block',
-		start: (src) => src.match(/^:[:#-]/)?.index,
+		start: (src) => src.match(/^: /)?.index,
 		tokenizer(src, tokens) {
-			const rule = /^(?::[:#-](?:\s[^\n]*)?(?:\n|$))+/;
+			const start = src.match(/^: /)?.index;
-			const match = rule.exec(src);
+			const lines = src.slice(start).split(/\n/g);
-
-			if (match) {
 			const token: DescriptionListToken = {
 				type: 'description_list',
-					raw: match[0],
+				raw: '',
 				items: [ ]
 			};
 
-				const items = token.raw.trim().split('\n');
+			let current: DescriptionElemToken;
-				const raw_buffer: string[] = [ ];
+			const render_current = () => {
-				const text_buffer: string[] = [ ];
+				if (current) {
+					this.lexer.blockTokens(current.text, current.tokens);
+					current = null;
+				}
+			};
 
-				const flush_buffer = () => {
+			for (const line of lines) {
-					if (! raw_buffer.length) {
+				// Skip empty lines
-						return;
+				if (! line.trim()) {
+					token.raw += line + '\n';
+
+					if (current) {
+						current.raw += line + '\n';
+						current.text += '\n';
 					}
 
-					// Grab the second character from the first line to determine the
+					continue;
-					// token type (should be "#" or "-")
+				}
-					const type = raw_buffer[0][1] === '#' ? 'description_term' : 'description_detail';
 
-					const sub_token: (DescriptionTermToken | DescriptionDetailToken) = {
+				// If the line starts immediately with a colon, it is a <dt>
-						type,
+				if (line.startsWith(': ')) {
-						raw: raw_buffer.join('\n'),
+					render_current();
-						text: text_buffer.join('\n'),
+					token.raw += line + '\n';
+					token.items.push(
+						current = {
+							type: 'description_term',
+							raw: line,
+							text: line.slice(2),
 							tokens: [ ],
-					};
+						}
-
+					);
-					raw_buffer.length = 0;
+					continue;
-					text_buffer.length = 0;
-
-					this.lexer.blockTokens(sub_token.text, sub_token.tokens);
-					token.items.push(sub_token);
-				};
-
-				for (const line of items) {
-					const rule = /^:([:#-])(?:\s([^\n]*))?(?:\n|$)/;
-					const match = rule.exec(line);
-		
-					if (match) {
-						if (match[1] !== ':') {
-							flush_buffer();
 				}
 
-						raw_buffer.push(match[0]);
+				// If the line starts with a colon after an indent, it is a <dd>
-						text_buffer.push(match[2]);
+				if (line.startsWith('  : ')) {
+					render_current();
+					token.raw += line + '\n';
+					token.items.push(
+						current = {
+							type: 'description_detail',
+							raw: line,
+							text: line.slice(4),
+							tokens: [ ],
 						}
+					);
+					continue;
 				}
 
-				flush_buffer();
+				// If the line starts with (at least) two indents, it is a child
+				// of the current element
+				if (line.startsWith('    ')) {
+					token.raw += line + '\n';
+					current.raw += '\n' + line;
+					current.text += '\n' + line.slice(current.type === 'description_term' ? 2 : 4);
+					continue;
+				}
 
+				// If the line starts with one indent, it is a child of the current
+				// <dt> (but is not allowed after a <dd>)
+				if (line.startsWith('  ')) {
+					if (current.type !== 'description_term') {
+						render_current();
+						break;
+					}
+
+					token.raw += line + '\n';
+					current.raw += '\n' + line;
+					current.text += '\n' + line.slice(2);
+					continue;
+				}
+
+				// If the line starts any other way, it is the start of new content
+				// and we are done parsing
+				render_current();
+				break;
+			}
+			
+			render_current();
+
+			if (token.items.length) {
 				return token;
 			}
 		},
 		renderer(token: DescriptionListToken) {
 			const items = token.items.map((item) => {
 				const tag = item.type === 'description_term' ? 'dt' : 'dd';
-				return `
+				return `<${tag}>${this.parser.parse(item.tokens)}</${tag}>`;
-					<${tag}>
-						${this.parser.parse(item.tokens)}
-					</${tag}>
-				`;
 			});
 
 			return `<dl>${items.join('')}</dl>`;

src/embed.ts

@@ -0,0 +1,33 @@
+
+import type { marked } from 'marked';
+import { MarkdownOptions } from './render';
+
+export interface EmbedToken extends marked.Tokens.Generic {
+	media_type: string;
+	href: string;
+}
+
+export function embed_ext(renderer: marked.Renderer, opts: MarkdownOptions) : marked.TokenizerExtension & marked.RendererExtension {
+	return {
+		name: 'embed',
+		level: 'inline',
+		start: (src) => src.match(/\{&/)?.index,
+		tokenizer(src, tokens) {
+			const rule = /^\{&\s+([^\s]+):\s*([^\s]+)\s+&\}/;
+			const match = rule.exec(src);
+
+			if (match) {
+				return {
+					type: 'embed',
+					raw: match[0],
+					media_type: match[1],
+					href: match[2],
+					tokens: [ ]
+				};
+			}
+		},
+		renderer(token: EmbedToken) {
+			return `<embed type="${token.media_type}" src="${token.href}"></embed>`;
+		}
+	};
+}

src/footnotes.ts

@@ -23,7 +23,7 @@ export function footnote_ref_ext(renderer: marked.Renderer, opts: MarkdownOption
 		level: 'inline',
 		start: (src) => src.match(/\[\^/)?.index,
 		tokenizer(src, tokens) {
-			const rule = /^\[\^([a-zA-Z0-9-._§]+)]/;
+			const rule = /^\[\^([a-zA-Z0-9-\._, §]+)]/;
 			const match = rule.exec(src);
 
 			if (match) {
@@ -68,7 +68,7 @@ export function footnote_list_ext(renderer: marked.Renderer, opts: MarkdownOptio
 			};
 
 			let remaining = src;
-			const prefix_rule = /^\[\^([a-zA-Z0-9\._§-]+)]:/;
+			const prefix_rule = /^\[\^([a-zA-Z0-9\., _§-]+)]:/;
 			const whitespace_rule = /^\s*(?:\n|$)/;
 
 			if (! prefix_rule.test(src)) {

src/html-sanitize.ts

@@ -9,5 +9,6 @@ export function sanitize_html(html: string, custom_elements?: CustomElementHandl
 	const dom_purify = createDOMPurify(window as any as Window);
 	return dom_purify.sanitize(html, {
 		CUSTOM_ELEMENT_HANDLING: custom_elements,
+		ADD_TAGS: [ 'embed', 'object', 'video', 'audio', 'iframe' ],
 	});
 }

src/prism/http.ts

@@ -0,0 +1,4 @@
+
+import { languages } from 'prismjs';
+
+languages.http['request-line'].pattern = /^(?:CONNECT|DELETE|GET|HEAD|QUERY|OPTIONS|PATCH|POST|PRI|PUT|SEARCH|TRACE)\s(?:https?:\/\/|\/)\S*\sHTTP\/[\d.]+/m;

src/prism/index.ts

@@ -4,6 +4,7 @@ import load_languages = require('prismjs/components/index');
 
 load_languages();
 require('./wasm');
+require('./http');
 
 // hooks.add('after-tokenize', (env) => {
 // 	// 

src/render.ts

@@ -9,8 +9,8 @@ import { katex_block_ext, katex_inline_ext } from './katex';
 import { footnote_list_ext, footnote_ref_ext } from './footnotes';
 import { description_list_ext } from './description-list';
 import { resolve_async_bindings } from './async-steps';
-import { breadcrumb_nav_ext } from './breadcrumb-nav';
 import { base_url_walk_tokens } from './base-url';
+import { embed_ext } from './embed';
 
 export interface MarkdownOptions {
 	base_url?: string;
@@ -26,24 +26,24 @@ export interface MarkdownExtension {
 }
 
 export async function render_markdown_to_html(markdown: string, options: MarkdownOptions = { }) {
-	const marked_options = marked_opts(true, options);
+	const marked_options = marked_opts(options);
+	setup_marked(options, marked_options);
 	const unsafe_html = options.inline
 		? marked.parseInline(markdown, marked_options)
-		: await marked.parse(markdown, marked_options).then(resolve_async_bindings);
+		: marked.parse(markdown, marked_options);
 
-	return sanitize_html(unsafe_html, options.custom_elements);
+	return sanitize_html(await resolve_async_bindings(unsafe_html), options.custom_elements);
 }
 
 export function render_markdown_to_html_inline_sync(markdown: string, options: MarkdownOptions = { }) {
-	const marked_options = marked_opts(false, options);
+	const marked_options = marked_opts(options);
 	setup_marked(options, marked_options);
 	const unsafe_html = marked.parseInline(markdown, marked_options);
 	return sanitize_html(unsafe_html, options.custom_elements);
 }
 
-function marked_opts<T extends boolean>(async: T, options: MarkdownOptions) : marked.MarkedOptions & { async: T } {
+function marked_opts<T extends boolean>(options: MarkdownOptions) : marked.MarkedOptions {
 	return {
-		async,
 		breaks: options.breaks || false,
 		renderer: create_renderer(options),
 		mangle: false,
@@ -65,7 +65,7 @@ function setup_marked(options: MarkdownOptions, marked_options: marked.MarkedOpt
 			description_list_ext(marked_options.renderer, options),
 			section_ext(marked_options.renderer, options),
 			icon_ext(marked_options.renderer, options),
-			breadcrumb_nav_ext(marked_options.renderer, options),
+			embed_ext(marked_options.renderer, options),
 			...(options.extensions || [ ]).map((ext) => {
 				return ext(marked_options.renderer, options);
 			}),

src/renderer.ts

@@ -29,17 +29,15 @@ export function create_renderer(opts: MarkdownOptions) {
 
 function heading(renderer: marked.Renderer, opts: MarkdownOptions) {
 	return function(orig_text: string, level: 1 | 2 | 3 | 4 | 5 | 6, raw: string) {
-		let { text, id, html_attrs } = parse_attributes(raw);
+		let { text, id, html_attrs } = parse_attributes(orig_text);
+		if (id) {
+			text += `\n<a class="heading-anchor" href="#${id}">`
+				+ `\n\t\t${icons.link}`
+				+ `\n\t\t<span style="display: none">Section titled ${text}</span>`
+				+ `\n\t</a>`;
+		}
 
-		return `
+		return `\n<h${level} ${html_attrs.join(' ')}>\n\t${text}\n</h${level}>`;
-<h${level} ${html_attrs.join(' ')}>
-	${text}
-	<a class="heading-anchor" href="#${id}">
-		${icons.link}
-		<span style="display: none">Section titled ${text}</span>
-	</a>
-</h${level}>
-`;
 	};
 }
 
@@ -155,6 +153,10 @@ function code(renderer: marked.Renderer, opts: MarkdownOptions) {
 				return figure(binding);
 			};
 
+			case 'yaml:calendar': {
+				// todo
+			};
+	
 			default:
 				return figure(`<pre class="language-${args[0] || 'txt'}">${render_prism(code, args[0])}</pre>`);
 		}