first pass at raw, text files

.gitignore

@@ -1,2 +1,4 @@
 node_modules
-build
+build
+docs
+www

package.json

@@ -5,25 +5,26 @@
     "registry": "https://gitea.home.jbrumond.me/api/packages/doc-utils/npm/"
   },
   "scripts": {
-    "tsc": "tsc"
+    "tsc": "tsc",
+    "clean": "rm -rf ./build"
   },
   "bin": {
     "docs2website": "./bin/docs2website"
   },
   "exports": "./build/index.js",
   "devDependencies": {
-    "@types/dompurify": "^2.3.3",
     "@types/jsdom": "^20.0.0",
-    "@types/katex": "^0.16.0",
     "@types/luxon": "^3.1.0",
-    "@types/marked": "^4.0.3",
+    "@types/mustache": "^4.2.2",
-    "@types/node": "^18.11.18",
+    "@types/node": "^18.16.3",
     "@types/prismjs": "^1.26.0",
-    "@types/qrcode": "^1.5.0",
     "typescript": "^5.0.4"
   },
   "dependencies": {
     "@doc-utils/markdown2html": "^0.1.0",
+    "glob": "^10.2.2",
+    "luxon": "^3.3.0",
+    "mustache": "^4.2.0",
     "yaml": "^2.2.2"
   }
 }

readme.md

@@ -0,0 +1,34 @@
+
+## Install from npm
+
+```bash
+# Update project npm config to refer to correct registry for the @doc-utils scope
+echo '@doc-utils:registry=https://gitea.jbrumond.me/api/packages/doc-utils/npm/' >> ./.npmrc
+
+# Install package for programatic use
+npm install --save @doc-utils/docs2website
+
+# Install globally for CLI usage
+npm install --global @doc-utils/docs2website
+```
+
+
+
+## Building from source
+
+```bash
+npm run tsc
+```
+
+
+## Command Line Use
+
+```
+docs2website <config file>
+```
+
+### Config File
+
+See [./sample-config.yaml](./sample-config.yaml)
+
+

sample-config.yaml

@@ -0,0 +1,62 @@
+
+# Input Configuration (where your documents are)
+input:
+  # The root directory for where to find input file. All other
+  # input paths are relative to this, and must be under this
+  root: ./docs
+
+  # Files identified in this section are copied over to the
+  # output directory unprocessed
+  raw:
+  - ./**/*.{png,jpg,jpeg,gif}
+
+  # Files in this section will be processed as mustache templates,
+  # but will receive no other processing
+  text:
+  - ./**/*.{css,html,js,txt}
+
+  # Files to be parsed as markdown (optionally with front matter)
+  # and rendered to HTML pages
+  markdown:
+  - ./**/*.md
+
+  # Files to be parsed as JSON Schema definitions and rendered
+  # to HTML documentation. Additionally, the original JSON / Yaml
+  # file will also be copied to the output directory, unaltered
+  schema+json:
+  - ./**/*.schema.json
+  schema+yaml:
+  - ./**/*.schema.{yaml,yml}
+
+  # Files to be parsed as OpenAPI V3 specifications and rendered
+  # to HTML documentation. Additionally, the original JSON / Yaml
+  # file will also be copied to the output directory, unaltered
+  openapi+json:
+  - ./**/*.openapi.json
+  openapi+yaml:
+  - ./**/*.openapi.{yaml,yml}
+
+# Template Configuration (used by mustache to actually render pages)
+templates:
+  # Root directory where layout files are stored
+  layouts: ./layouts
+
+  # Root directory where partial files are stored
+  partials: ./partials
+
+  # (Optional) whitelist of environment variables to be made accessible
+  # under `env` when processing templates
+  env:
+  - EXAMPLE_ENVIRONMENT_VARIABLE
+  - FOO_BAR_BAZ
+
+# Output Configuration (where to put your website)
+output:
+  # The root directory to output your website at. The path of an
+  # input file relative to $.input.root will match (aside from file
+  # extension) the path of the output file relative to $.output.root
+  root: ./www
+
+# Markdown-to-HTML Configuration
+markdown:
+  # 

src/bin/docs2website.ts

@@ -0,0 +1,10 @@
+
+import { read_config } from '../conf';
+import { build_docs_project } from '../build';
+
+main();
+
+async function main() {
+	const conf = await read_config('./sample-config.yaml');
+	await build_docs_project(conf);
+}

src/build.ts

@@ -0,0 +1,157 @@
+
+import { glob } from 'glob';
+import { Config } from './conf';
+import { promises as fs } from 'fs';
+import { dirname, join as path_join } from 'path';
+import { build_env_scope } from './env';
+import { process_frontmatter } from '@doc-utils/markdown2html';
+import { load_layout, Context, render_template, load_partials } from './template';
+import { DateTime } from 'luxon';
+import assert = require('assert');
+
+interface BuildState {
+	conf: Config;
+	seen_files: Set<string>;
+	env: Record<string, string>;
+	partials?: Record<string, string>;
+	layouts: Record<string, string>;
+	made_directories: Set<string>;
+	build_time: {
+		iso: string;
+		rfc2822: string;
+	};
+}
+
+export async function build_docs_project(conf: Config) {
+	const now = DateTime.now();
+	const state: BuildState = {
+		conf,
+		seen_files: new Set<string>(),
+		env: build_env_scope(conf),
+		layouts: Object.create(null),
+		made_directories: new Set<string>(),
+		build_time: {
+			iso: now.toISO(),
+			rfc2822: now.toRFC2822(),
+		},
+	}
+
+	if (conf.input.raw) {
+		await copy_raw_files(state);
+	}
+
+	if (conf.input.text) {
+		await render_text_file_templates(state);
+	}
+
+	if (conf.input.markdown) {
+		// 
+	}
+
+	// todo...
+}
+
+async function copy_raw_files(state: BuildState) {
+	const promises: Promise<any>[] = [ ];
+	const files = await glob(state.conf.input.raw, {
+		absolute: true,
+		cwd: state.conf.input.root,
+	});
+
+	for (const in_file of files) {
+		if (state.seen_files.has(in_file)) {
+			continue;
+		}
+
+		state.seen_files.add(in_file);
+
+		const out_file = map_input_file_to_output_file(state, in_file);
+
+		promises.push(
+			fs.copyFile(in_file, await out_file, 0o600)
+		);
+	}
+
+	await Promise.all(promises);
+}
+
+async function render_text_file_templates(state: BuildState) {
+	const promises: Promise<any>[] = [ ];
+	const files = await glob(state.conf.input.text, {
+		absolute: true,
+		cwd: state.conf.input.root,
+	});
+
+	if (! state.partials) {
+		state.partials = await load_partials(state.conf);
+	}
+
+	for (const in_file of files) {
+		if (state.seen_files.has(in_file)) {
+			continue;
+		}
+
+		state.seen_files.add(in_file);
+
+		promises.push(
+			render_text_file_template(state, in_file)
+		);
+	}
+
+	await Promise.all(promises);
+}
+
+async function render_text_file_template(state: BuildState, in_file: string) {
+	const out_file = map_input_file_to_output_file(state, in_file);
+	const { frontmatter, document } = process_frontmatter(await fs.readFile(in_file, 'utf8'));
+	
+	let layout: string;
+	const layout_file = frontmatter?.layout;
+
+	if (layout_file) {
+		if (! state.layouts[layout_file]) {
+			state.layouts[layout_file] = await load_layout(state.conf, layout_file);
+		}
+
+		layout = state.layouts[layout_file];
+	}
+
+	const context: Context = {
+		env: state.env,
+		page: frontmatter,
+		build_time: state.build_time,
+	};
+
+	const rendered = render_template(document, context, layout, structuredClone(state.partials));
+	await fs.writeFile(await out_file, rendered, 'utf8');
+}
+
+async function map_input_file_to_output_file(state: BuildState, in_file: string, remove_exts?: string[], add_ext?: string) {
+	assert(in_file.startsWith(state.conf.input.root), 'input file expected to be inside input root');
+	let out_file = path_join(state.conf.output.root, in_file.slice(state.conf.input.root.length));
+
+	if (remove_exts) {
+		for (const ext of remove_exts) {
+			if (out_file.endsWith(ext)) {
+				out_file = out_file.slice(0, -ext.length);
+				break;
+			}
+		}
+	}
+
+	if (add_ext) {
+		out_file += add_ext;
+	}
+
+	const dir = dirname(out_file);
+	
+	if (! state.made_directories.has(dir)) {
+		state.made_directories.add(dir);
+		await fs.mkdir(dir, {
+			mode: 0o700,
+			recursive: true,
+		});
+	}
+
+	return out_file;
+}

src/conf.ts

@@ -0,0 +1,63 @@
+
+import { promises as fs } from 'fs';
+import { parse as parse_yaml } from 'yaml';
+import { resolve as resolve_path, dirname } from 'path';
+
+export async function read_config(file: string) {
+	const path = resolve_path(process.cwd(), file);
+	const yaml = await fs.readFile(path, 'utf8');
+	const config = parse_yaml(yaml);
+	validate_config(config);
+	resolve_paths(path, config);
+	return config;
+}
+
+export interface Config {
+	input: {
+		root: string;
+		raw?: string[];
+		text?: string[];
+		markdown?: string[];
+		'schema+json'?: string[];
+		'schema+yaml'?: string[];
+		'openapi+json'?: string[];
+		'openapi+yaml'?: string[];
+		// ...
+	};
+	templates?: {
+		layouts?: string;
+		partials?: string;
+		env?: string[];
+	};
+	output: {
+		root: string;
+	};
+	markdown?: {
+		// 
+	};
+	schema?: {
+		// 
+	};
+	openapi?: {
+		// ;
+	}
+	// ...
+}
+
+function validate_config(config: unknown) : asserts config is Config {
+	// todo: validate config
+}
+
+function resolve_paths(file_path: string, config: Config) {
+	const base_path = dirname(file_path);
+	config.input.root = resolve_path(base_path, config.input.root);
+	config.output.root = resolve_path(base_path, config.output.root);
+
+	if (config.templates?.layouts) {
+		config.templates.layouts = resolve_path(base_path, config.templates.layouts);
+	}
+
+	if (config.templates?.partials) {
+		config.templates.partials = resolve_path(base_path, config.templates.partials);
+	}
+}

src/env.ts

@@ -0,0 +1,13 @@
+
+import { Config } from './conf';
+
+export function build_env_scope(conf: Config) {
+	const whitelist = conf.templates?.env ?? [ ];
+	const scope: Record<string, string> = Object.create(null);
+
+	for (const variable of whitelist) {
+		scope[variable] = process.env[variable];
+	}
+
+	return scope;
+}

src/index.ts

@@ -0,0 +1,2 @@
+
+export { read_config, Config } from './conf';

src/template.ts

@@ -0,0 +1,57 @@
+
+import { Config } from './conf';
+import { render as mustache_render } from 'mustache';
+import { promises as fs } from 'fs';
+import { resolve as resolve_path } from 'path';
+import { glob } from 'glob';
+
+export interface Context {
+	env?: Record<string, string>;
+	page?: {
+		title?: string;
+		layout?: string;
+		[key: string]: string | number | boolean;
+	};
+	build_time: {
+		iso: string;
+		rfc2822: string;
+	};
+}
+
+export function render_template(template: string, context: Context, layout?: string, partials?: Record<string, string>) {
+	partials['.content'] = template;
+	return mustache_render(layout || template, context, partials);
+}
+
+export async function load_layout(conf: Config, file: string) {
+	const path = conf.templates?.layouts;
+
+	if (! path) {
+		return null;
+	}
+
+	const rel_path = resolve_path('/', file);
+	const abs_path = resolve_path(path, '.' + rel_path);
+	return await fs.readFile(abs_path, 'utf8');
+}
+
+export async function load_partials(conf: Config) {
+	const path = conf.templates?.partials;
+
+	if (! path) {
+		return { };
+	}
+
+	const partials: Record<string, string> = { };
+	const partial_files = await glob(path + '/**/*', {
+		cwd: path,
+		absolute: false,
+	});
+
+	for (const file of partial_files) {
+		const abs_file = resolve_path(path, file);
+		partials[file] = await fs.readFile(abs_file, 'utf8');
+	}
+
+	return partials;
+}