integrate jsonschema2markdown; add support for inline markdown rendering from templates

src/bin/docs2website.ts

@@ -5,6 +5,6 @@ import { build_docs_project } from '../build';
 main();
 
 async function main() {
-	const conf = await read_config('./sample-config.yaml');
+	const conf = await read_config(process.argv[2]);
 	await build_docs_project(conf);
 }

src/build.ts

@@ -1,13 +1,20 @@
 
+import { ColorTheme } from '@doc-utils/color-themes';
+import { build_markdown_from_json_schema } from '@doc-utils/jsonschema2markdown';
+import { render_markdown_to_html, render_markdown_to_html_inline_sync } from '@doc-utils/markdown2html';
+
 import { glob } from 'glob';
 import { Config } from './conf';
 import { promises as fs } from 'fs';
 import { dirname, join as path_join } from 'path';
 import { build_env_scope } from './env';
-import { process_frontmatter } from '@doc-utils/markdown2html';
-import { load_layout, Context, render_template, load_partials } from './template';
+import { load_layout, Context, render_template, load_partials, load_extras, FrontMatter } from './template';
 import { DateTime } from 'luxon';
 import assert = require('assert');
+import { load_themes, render_theme_css_properties } from './themes';
+import { icons } from './icons';
+import { mkdirp, read_json, read_text, read_yaml, write_text } from './fs';
+import { stringify as to_yaml } from 'yaml';
 
 interface BuildState {
 	conf: Config;
@@ -15,6 +22,9 @@ interface BuildState {
 	env: Record<string, string>;
 	partials?: Record<string, string>;
 	layouts: Record<string, string>;
+	themes: Record<string, ColorTheme>;
+	theme_groups: ThemeGroups;
+	extras: Record<string, string>;
 	made_directories: Set<string>;
 	build_time: {
 		iso: string;
@@ -22,13 +32,42 @@ interface BuildState {
 	};
 }
 
+export interface ThemeGroups {
+	all: ColorTheme[];
+	light: ColorTheme[];
+	dark: ColorTheme[];
+	high_contrast: ColorTheme[];
+	low_contrast: ColorTheme[];
+	monochrome: ColorTheme[];
+	greyscale: ColorTheme[];
+	protanopia_safe: ColorTheme[];
+	deuteranopia_safe: ColorTheme[];
+	tritanopia_safe: ColorTheme[];
+}
+
 export async function build_docs_project(conf: Config) {
 	const now = DateTime.now();
+	const themes = await load_themes(conf);
 	const state: BuildState = {
 		conf,
 		seen_files: new Set<string>(),
 		env: build_env_scope(conf),
 		layouts: Object.create(null),
+		themes: themes,
+		theme_groups: {
+			// fixme: this is horribly inefficient
+			all: Object.values(themes),
+			light: Object.values(themes).filter((theme) => theme.labels.includes('light')),
+			dark: Object.values(themes).filter((theme) => theme.labels.includes('dark')),
+			high_contrast: Object.values(themes).filter((theme) => theme.labels.includes('high_contrast')),
+			low_contrast: Object.values(themes).filter((theme) => theme.labels.includes('low_contrast')),
+			monochrome: Object.values(themes).filter((theme) => theme.labels.includes('monochrome')),
+			greyscale: Object.values(themes).filter((theme) => theme.labels.includes('greyscale')),
+			protanopia_safe: Object.values(themes).filter((theme) => theme.labels.includes('protanopia_safe')),
+			deuteranopia_safe: Object.values(themes).filter((theme) => theme.labels.includes('deuteranopia_safe')),
+			tritanopia_safe: Object.values(themes).filter((theme) => theme.labels.includes('tritanopia_safe')),
+		},
+		extras: await load_extras(),
 		made_directories: new Set<string>(),
 		build_time: {
 			iso: now.toISO(),
@@ -45,12 +84,20 @@ export async function build_docs_project(conf: Config) {
 	}
 
 	if (conf.input.markdown) {
-		// 
+		await render_markdown_files(state);
+	}
+
+	if (conf.input['schema+json'] || conf.input['schema+yaml']) {
+		await render_json_schema_files(state);
 	}
 
 	// todo...
 }
 
+
+
+// ===== Raw File Copy =====
+
 async function copy_raw_files(state: BuildState) {
 	const promises: Promise<any>[] = [ ];
 	const files = await glob(state.conf.input.raw, {
@@ -68,13 +115,17 @@ async function copy_raw_files(state: BuildState) {
 		const out_file = map_input_file_to_output_file(state, in_file);
 
 		promises.push(
-			fs.copyFile(in_file, await out_file, 0o600)
+			fs.copyFile(in_file, await out_file)
 		);
 	}
 
 	await Promise.all(promises);
 }
 
+
+
+// ===== File Renderers =====
+
 async function render_text_file_templates(state: BuildState) {
 	const promises: Promise<any>[] = [ ];
 	const files = await glob(state.conf.input.text, {
@@ -83,7 +134,7 @@ async function render_text_file_templates(state: BuildState) {
 	});
 
 	if (! state.partials) {
-		state.partials = await load_partials(state.conf);
+		await build_partials(state);
 	}
 
 	for (const in_file of files) {
@@ -103,7 +154,7 @@ async function render_text_file_templates(state: BuildState) {
 
 async function render_text_file_template(state: BuildState, in_file: string) {
 	const out_file = map_input_file_to_output_file(state, in_file);
-	const { frontmatter, document } = process_frontmatter(await fs.readFile(in_file, 'utf8'));
+	const { frontmatter, text } = await read_text(in_file);
 	
 	let layout: string;
 	const layout_file = frontmatter?.layout;
@@ -116,16 +167,202 @@ async function render_text_file_template(state: BuildState, in_file: string) {
 		layout = state.layouts[layout_file];
 	}
 
-	const context: Context = {
-		env: state.env,
-		page: frontmatter,
-		build_time: state.build_time,
-	};
-
-	const rendered = render_template(document, context, layout, structuredClone(state.partials));
-	await fs.writeFile(await out_file, rendered, 'utf8');
+	const tags = state.conf.templates?.tags;
+	const context = mustache_context(state, frontmatter);
+	const rendered = render_template(text, context, layout, structuredClone(state.partials), tags);
+	await write_text(await out_file, rendered);
 }
 
+async function render_markdown_files(state: BuildState) {
+	const promises: Promise<any>[] = [ ];
+	const files = await glob(state.conf.input.markdown, {
+		absolute: true,
+		cwd: state.conf.input.root,
+	});
+
+	if (! state.partials) {
+		await build_partials(state);
+	}
+
+	for (const in_file of files) {
+		if (state.seen_files.has(in_file)) {
+			continue;
+		}
+
+		state.seen_files.add(in_file);
+
+		promises.push(
+			render_markdown_file(state, in_file)
+		);
+	}
+
+	await Promise.all(promises);
+}
+
+async function render_markdown_file(state: BuildState, in_file: string) {
+	const out_file = map_input_file_to_output_file(state, in_file, [ '.md', '.markdown' ], '.html');
+	const { frontmatter, text } = await read_text(in_file);
+
+	const html = render_markdown_to_html(text, state.conf.markdown);
+	
+	let layout: string;
+	const layout_file = frontmatter?.layout;
+
+	if (layout_file) {
+		if (! state.layouts[layout_file]) {
+			state.layouts[layout_file] = await load_layout(state.conf, layout_file);
+		}
+
+		layout = state.layouts[layout_file];
+	}
+
+	const tags = state.conf.templates?.tags;
+	const context = mustache_context(state, frontmatter);
+	const rendered = render_template(await html, context, layout, structuredClone(state.partials), tags);
+	await write_text(await out_file, rendered);
+}
+
+async function render_json_schema_files(state: BuildState) {
+	const promises: Promise<any>[] = [ ];
+	const json_files = await glob(state.conf.input['schema+json'], {
+		absolute: true,
+		cwd: state.conf.input.root,
+	});
+	const yaml_files = await glob(state.conf.input['schema+yaml'], {
+		absolute: true,
+		cwd: state.conf.input.root,
+	});
+
+	if (! state.partials) {
+		await build_partials(state);
+	}
+
+	for (const in_file of json_files) {
+		if (state.seen_files.has(in_file)) {
+			continue;
+		}
+
+		state.seen_files.add(in_file);
+
+		promises.push(
+			handle_json_schema_json_file(state, in_file)
+		);
+	}
+
+	for (const in_file of yaml_files) {
+		if (state.seen_files.has(in_file)) {
+			continue;
+		}
+
+		state.seen_files.add(in_file);
+
+		promises.push(
+			handle_json_schema_yaml_file(state, in_file)
+		);
+	}
+
+	await Promise.all(promises);
+}
+
+async function handle_json_schema_json_file(state: BuildState, in_file: string) {
+	const promises: Promise<any>[] = [ ];
+
+	const out_file = map_input_file_to_output_file(state, in_file, [ '.json' ], '.html');
+	const { frontmatter, parsed, json } = await read_json(in_file, true);
+
+	promises.push(
+		render_json_schema(state, parsed, await out_file, frontmatter)
+	);
+
+	if (state.conf.output.include_inputs?.includes('schema+json')) {
+		const json_file = await map_input_file_to_output_file(state, in_file);
+
+		promises.push(
+			write_text(json_file, json)
+		);
+
+		if (state.conf.output.include_yaml_and_json) {
+			const yaml_file = await map_input_file_to_output_file(state, in_file, [ '.json' ], '.yaml');
+			
+			promises.push(
+				write_text(yaml_file, to_yaml(parsed))
+			);
+		}
+	}
+
+	await Promise.all(promises);
+}
+
+async function handle_json_schema_yaml_file(state: BuildState, in_file: string) {
+	const promises: Promise<any>[] = [ ];
+
+	const out_file = map_input_file_to_output_file(state, in_file, [ '.yaml', '.yml' ], '.html');
+	const { frontmatter, parsed, yaml } = await read_yaml(in_file, true);
+
+	promises.push(
+		render_json_schema(state, parsed, await out_file, frontmatter)
+	);
+
+	if (state.conf.output.include_inputs?.includes('schema+yaml')) {
+		const yaml_file = await map_input_file_to_output_file(state, in_file);
+
+		promises.push(
+			write_text(yaml_file, yaml)
+		);
+
+		if (state.conf.output.include_yaml_and_json) {
+			const json_file = await map_input_file_to_output_file(state, in_file, [ '.yaml', '.yml' ], '.json');
+			
+			promises.push(
+				write_text(json_file, JSON.stringify(parsed, null, '  '))
+			);
+		}
+	}
+
+	await Promise.all(promises);
+}
+
+async function render_json_schema(state: BuildState, schema: unknown, out_file: string, frontmatter?: any) {
+	const promises: Promise<any>[] = [ ];
+	const markdown = build_markdown_from_json_schema(schema);
+	
+	if (state.conf.output.include_intermediate_markdown) {
+		promises.push(
+			map_input_file_to_output_file(state, out_file, [ '.html' ], '.md')
+				.then((md_file) => write_text(md_file, markdown))
+		);
+	}
+
+	const html = render_markdown_to_html(markdown, state.conf.markdown);
+	
+	let layout: string;
+	const layout_file = frontmatter?.layout;
+
+	if (layout_file) {
+		if (! state.layouts[layout_file]) {
+			state.layouts[layout_file] = await load_layout(state.conf, layout_file);
+		}
+
+		layout = state.layouts[layout_file];
+	}
+
+	const tags = state.conf.templates?.tags;
+	const context = mustache_context(state, frontmatter);
+	const rendered = render_template(await html, context, layout, structuredClone(state.partials), tags);
+	
+	promises.push(
+		write_text(await out_file, rendered)
+	);
+
+	await Promise.all(promises);
+}
+
+
+
+
+
+// ===== Helpers =====
+
 async function map_input_file_to_output_file(state: BuildState, in_file: string, remove_exts?: string[], add_ext?: string) {
 	assert(in_file.startsWith(state.conf.input.root), 'input file expected to be inside input root');
 	let out_file = path_join(state.conf.output.root, in_file.slice(state.conf.input.root.length));
@@ -147,11 +384,38 @@ async function map_input_file_to_output_file(state: BuildState, in_file: string,
 	
 	if (! state.made_directories.has(dir)) {
 		state.made_directories.add(dir);
-		await fs.mkdir(dir, {
-			mode: 0o700,
-			recursive: true,
-		});
+		await mkdirp(dir);
 	}
 
 	return out_file;
 }
+
+async function build_partials(state: BuildState) {
+	state.partials = await load_partials(state.conf);
+
+	for (const [name, theme] of Object.entries(state.themes)) {
+		state.partials[`.themes/${name}`] = render_theme_css_properties(theme);
+	}
+
+	Object.assign(state.partials, state.extras);
+}
+
+function mustache_context(state: BuildState, frontmatter?: FrontMatter) : Context {
+	return {
+		env: state.env,
+		page: frontmatter,
+		build_time: state.build_time,
+		icons: icons,
+		themes: Object.values(state.themes),
+		theme_groups: structuredClone(state.theme_groups),
+		markdown: {
+			render_inline() {
+				return (text, render) => {
+					const md = render(text)
+					const html = render_markdown_to_html_inline_sync(md, state.conf.markdown);
+					return html;
+				};
+			},
+		}
+	};
+}

src/conf.ts

@@ -2,12 +2,14 @@
 import { promises as fs } from 'fs';
 import { parse as parse_yaml } from 'yaml';
 import { resolve as resolve_path, dirname } from 'path';
+import { MarkdownOptions } from '@doc-utils/markdown2html';
 
 export async function read_config(file: string) {
 	const path = resolve_path(process.cwd(), file);
 	const yaml = await fs.readFile(path, 'utf8');
 	const config = parse_yaml(yaml);
 	validate_config(config);
+	process_markdown_config(config);
 	resolve_paths(path, config);
 	return config;
 }
@@ -27,14 +29,17 @@ export interface Config {
 	templates?: {
 		layouts?: string;
 		partials?: string;
+		themes?: string;
+		tags?: [ string, string ];
 		env?: string[];
 	};
 	output: {
 		root: string;
+		include_inputs?: string[];
+		include_yaml_and_json?: boolean;
+		include_intermediate_markdown?: boolean;
 	};
-	markdown?: {
-		// 
-	};
+	markdown?: MarkdownOptions;
 	schema?: {
 		// 
 	};
@@ -61,3 +66,19 @@ function resolve_paths(file_path: string, config: Config) {
 		config.templates.partials = resolve_path(base_path, config.templates.partials);
 	}
 }
+
+function process_markdown_config(config: any) {
+	if (config?.markdown?.custom_elements) {
+		if (config.markdown.custom_elements.tag_names) {
+			const tags = new Set<string>(config.markdown.custom_elements.tag_names);
+			config.markdown.custom_elements.tagNameCheck = (tag_name) => tags.has(tag_name);
+			delete config.markdown.custom_elements.tag_names;
+		}
+
+		if (config.markdown.custom_elements.attribute_names) {
+			const attrs = new Set<string>(config.markdown.custom_elements.attribute_names);
+			config.markdown.custom_elements.attributeNameCheck = (attr_name) => attrs.has(attr_name);
+			delete config.markdown.custom_elements.attribute_names;
+		}
+	}
+}

src/fs.ts

@@ -0,0 +1,70 @@
+
+import { process_frontmatter } from '@doc-utils/markdown2html';
+import { promises as fs } from 'fs';
+import { resolve as resolve_path } from 'path';
+import { parse as parse_yaml, stringify as to_yaml } from 'yaml';
+
+export async function load_from_dir(dir: string, file: string) {
+	if (! dir) {
+		return null;
+	}
+
+	const rel_path = resolve_path('/', file);
+	const abs_path = resolve_path(dir, '.' + rel_path);
+	return await fs.readFile(abs_path, 'utf8');
+}
+
+export function mkdirp(dir: string, mode = 0o700) {
+	return fs.mkdir(dir, { mode, recursive: true });
+}
+
+export async function read_text(file: string, check_for_frontmatter = true) {
+	const text = await fs.readFile(file, 'utf8');
+
+	if (check_for_frontmatter) {
+		const { frontmatter, document } = process_frontmatter(text);
+		return { frontmatter, text: document };
+	}
+
+	return { text, frontmatter: null };
+}
+
+export async function read_json(file: string, check_for_frontmatter = true) {
+	const json = await fs.readFile(file, 'utf8');
+
+	if (check_for_frontmatter) {
+		const { frontmatter, document } = process_frontmatter(json);
+		const parsed = JSON.parse(document);
+		return { frontmatter, json: document, parsed };
+	}
+
+	const parsed = JSON.parse(json);
+	return { json, parsed, frontmatter: null };
+}
+
+export async function read_yaml(file: string, check_for_frontmatter = true) {
+	const yaml = await fs.readFile(file, 'utf8');
+
+	if (check_for_frontmatter) {
+		const { frontmatter, document } = process_frontmatter(yaml);
+		const parsed = parse_yaml(document);
+		return { frontmatter, yaml: document, parsed };
+	}
+
+	const parsed = parse_yaml(yaml);
+	return { yaml, parsed, frontmatter: null };
+}
+
+export function write_text(file: string, text: string) {
+	return fs.writeFile(file, text, 'utf8');
+}
+
+export function write_json(file: string, obj: any, pretty = true) {
+	const json = JSON.stringify(obj, null, pretty ? '  ' : null);
+	return fs.writeFile(file, json, 'utf8');
+}
+
+export function write_yaml(file: string, obj: any) {
+	const yaml = to_yaml(obj);
+	return fs.writeFile(file, yaml, 'utf8');
+}

src/icons.ts

@@ -0,0 +1,23 @@
+
+export const icons: Record<string, string> = Object.create(null);
+
+const whitespace = /[\s\t\n]+/g;
+const feather_icons: Record<string, string> = require('../vendor/feather-icons/icons.json');
+
+for (const [name, contents] of Object.entries(feather_icons)) {
+	icons[name] = `
+	<svg xmlns="http://www.w3.org/2000/svg"
+		class="icon ${name}"
+		aria-hidden="true"
+		style="width: var(--icon-size, 1rem); height: var(--icon-size, 1rem)"
+		viewBox="0 0 24 24"
+		fill="none"
+		stroke="currentcolor"
+		stroke-width="2"
+		stroke-linecap="round"
+		stroke-linejoin="round"
+	>${contents}</svg>
+	`.replace(whitespace, ' ').trim();
+}
+
+Object.freeze(icons);

src/index.ts

@@ -1,2 +1,3 @@
 
 export { read_config, Config } from './conf';
+export { build_docs_project, ThemeGroups } from './build';

src/template.ts

@@ -4,35 +4,57 @@ import { render as mustache_render } from 'mustache';
 import { promises as fs } from 'fs';
 import { resolve as resolve_path } from 'path';
 import { glob } from 'glob';
+import { load_from_dir } from './fs';
+import { ColorTheme } from '@doc-utils/color-themes';
+import { ThemeGroups } from './build';
 
 export interface Context {
 	env?: Record<string, string>;
-	page?: {
-		title?: string;
-		layout?: string;
-		[key: string]: string | number | boolean;
-	};
+	page?: FrontMatter;
+	icons: Record<string, string>;
+	themes: ColorTheme[];
+	theme_groups: ThemeGroups;
 	build_time: {
 		iso: string;
 		rfc2822: string;
 	};
+	markdown: {
+		render_inline(): MustacheRenderer;
+	}
 }
 
-export function render_template(template: string, context: Context, layout?: string, partials?: Record<string, string>) {
+export interface FrontMatter {
+	title?: string;
+	layout?: string;
+	[key: string]: unknown;
+}
+
+export function render_template(template: string, context: Context, layout?: string, partials: Record<string, string> = { }, tags?: [ string, string ]) {
 	partials['.content'] = template;
-	return mustache_render(layout || template, context, partials);
+	return mustache_render(layout || template, context, partials, tags);
+}
+
+export async function load_extras() {
+	const extras: Record<string, string> = Object.create(null);
+	const extras_dir = resolve_path(__dirname, '../extras');
+	const extras_files = [
+		'components/color-scheme-toggle-button.js',
+		'components/outline-button.js',
+		'components/outline-inline.js',
+		'prism.css',
+	];
+
+	const promises = extras_files.map((file) => load_from_dir(extras_dir, file));
+
+	for (let i = 0; i < extras_files.length; i++) {
+		extras[`.extras/${extras_files[i]}`] = await promises[i];
+	}
+
+	return extras;
 }
 
 export async function load_layout(conf: Config, file: string) {
-	const path = conf.templates?.layouts;
-
-	if (! path) {
-		return null;
-	}
-
-	const rel_path = resolve_path('/', file);
-	const abs_path = resolve_path(path, '.' + rel_path);
-	return await fs.readFile(abs_path, 'utf8');
+	return load_from_dir(conf.templates?.layouts, file);
 }
 
 export async function load_partials(conf: Config) {
@@ -55,3 +77,7 @@ export async function load_partials(conf: Config) {
 
 	return partials;
 }
+
+export interface MustacheRenderer {
+	(this: void, text: string, render: (text: string) => string): string;
+}

src/themes.ts

@@ -0,0 +1,64 @@
+
+import { glob } from 'glob';
+import { Config } from './conf';
+import { load_from_dir } from './fs';
+import { ColorTheme, themes as builtin_themes, load_theme as load_builtin_theme, validate_theme } from '@doc-utils/color-themes';
+
+export async function load_themes(conf: Config) {
+	const themes: Record<string, ColorTheme> = { };
+
+	for (const theme of builtin_themes) {
+		themes[theme] = load_builtin_theme(theme);
+	}
+
+	if (conf.templates?.themes) {
+		const local_themes = await glob(conf.templates.themes + '/**/theme.json');
+
+		for (const theme_path of local_themes) {
+			const segments = theme_path.split('/');
+			const theme = segments[segments.length - 2];
+			themes[theme] = await load_local_theme(conf, theme);
+		}
+	}
+
+	return themes;
+}
+
+export async function load_theme(conf: Config, name: string) {
+	if (name.includes('/')) {
+		return null;
+	}
+
+	const local_theme = await load_local_theme(conf, name);
+
+	if (local_theme) {
+		return local_theme;
+	}
+
+	return load_builtin_theme(name);
+}
+
+export async function load_local_theme(conf: Config, name: string) {
+	let json: string;
+
+	try {
+		json = await load_from_dir(conf.templates?.themes, `./${name}/theme.json`);
+	}
+	
+	catch (error) {
+		return null;
+	}
+
+	const theme = JSON.parse(json);
+	validate_theme(theme);
+	return theme;
+}
+
+export function render_theme_css_properties(theme: ColorTheme) {
+	return Object
+		.entries(theme.colors)
+		.map(([ name, color ]) => {
+			return `--theme-${name.replace(/_/g, '-')}: ${color};`;
+		})
+		.join('\n') + '\n';
+}