doc-utils/docs2website
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

work on splitting up extras css; start handling build metadata; start building sitemaps

Browse Source
This commit is contained in:
James Brumond
2023-05-19 00:00:15 -07:00
parent bf390c324f
commit 5689c64c4e
21 changed files with 1015 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/bin/docs2website.ts
View File

@@ -1,6 +1,6 @@
import { read_config } from '../conf';
import { build_docs_project } from '../build';
import { build_docs_project } from '../build-files';
main();

230
src/build-files/helpers.ts Normal file
View File

@@ -0,0 +1,230 @@
import { dirname, join as path_join } from 'path';
import { mkdirp, write_text } from '../fs';
import { icons } from '../icons';
import { BuildState } from './state';
import { load_partials, FrontMatter, Context, load_layout, render_template } from '../template';
import { render_theme_css_properties } from '../themes';
import { render_markdown_to_html, render_markdown_to_html_inline_sync } from '@doc-utils/markdown2html';
import { CalendarConfig, RSSConfig } from '../conf';
export interface OutFileURL {
base_url: string;
rel_path: string;
dir_url: string;
abs_url: string;
}
export function map_output_file_to_url(state: BuildState, out_file: string, index_file?: string) : OutFileURL {
let rel_path = out_file.slice(state.conf.input.root.length);
if (index_file && rel_path.endsWith('/' + index_file)) {
rel_path = rel_path.slice(0, -(index_file.length + 1));
}
if (! rel_path.startsWith('/')) {
rel_path = '/' + rel_path;
}
const base_url = state.conf.base_url.endsWith('/')
? state.conf.base_url.slice(0, -1)
: state.conf.base_url;
return {
base_url,
rel_path,
dir_url: base_url + dirname(rel_path),
abs_url: base_url + rel_path
};
}
export async function map_input_file_to_output_file(state: BuildState, in_file: string, remove_exts?: string[], add_ext?: string) {
if (! in_file.startsWith(state.conf.input.root)) {
throw new Error('input file expected to be inside input root');
}
let out_file = path_join(state.conf.output.root, in_file.slice(state.conf.input.root.length));
if (remove_exts) {
for (const ext of remove_exts) {
if (out_file.endsWith(ext)) {
out_file = out_file.slice(0, -ext.length);
break;
}
}
}
if (add_ext) {
out_file += add_ext;
}
const dir = dirname(out_file);
if (! state.made_directories.has(dir)) {
state.made_directories.add(dir);
await mkdirp(dir);
}
return out_file;
}
export async function build_partials(state: BuildState) {
state.partials = await load_partials(state.conf);
for (const [name, theme] of Object.entries(state.themes)) {
state.partials[`.themes/${name}`] = render_theme_css_properties(theme);
}
Object.assign(state.partials, state.extras);
}
export function mustache_context(state: BuildState, page_url: string, frontmatter?: FrontMatter) : Context {
return {
env: state.env,
page: frontmatter,
base_url: state.conf.base_url,
page_url: page_url,
build_time: state.build_time,
icons: icons,
themes: Object.values(state.themes),
theme_groups: structuredClone(state.theme_groups),
markdown: {
render_inline() {
return (text, render) => {
const md = render(text)
const html = render_markdown_to_html_inline_sync(md,
Object.assign({ }, state.conf.markdown, { base_url: page_url })
);
return html;
};
},
}
};
}
export async function render_page(state: BuildState, out_file: string, out_url: OutFileURL, text: string, render_as_markdown: boolean, frontmatter?: any) {
if (render_as_markdown) {
const opts = Object.assign({ }, state.conf.markdown, {
base_url: out_url.abs_url
});
text = await render_markdown_to_html(text, opts);
}
let layout: string;
const layout_file = frontmatter?.layout;
if (layout_file) {
if (! state.layouts[layout_file]) {
state.layouts[layout_file] = await load_layout(state.conf, layout_file);
}
layout = state.layouts[layout_file];
}
const tags = state.conf.templates?.tags;
const context = mustache_context(state, out_url.abs_url, frontmatter);
const rendered = render_template(text, context, layout, structuredClone(state.partials), tags);
await write_text(out_file, rendered);
handle_page_side_effects(state, out_file, out_url, frontmatter);
}
function handle_page_side_effects(state: BuildState, out_file: string, out_url: OutFileURL, frontmatter?: any) {
// Only actual HTML webpages are registered with things like the RSS feed; This is
// to prevent things like CSS files showing up, which may be templated (and therefore
// pass through this function), but are not really "pages"
if (frontmatter && typeof frontmatter === 'object' && out_file.endsWith('.html')) {
if (state.conf.rss) {
if (Array.isArray(state.conf.rss)) {
for (const rss_conf of state.conf.rss) {
handle_rss(state, rss_conf, out_url, frontmatter);
}
}
else {
handle_rss(state, { }, out_url, frontmatter);
}
}
handle_sitemap(state, out_url, frontmatter);
handle_event(state, out_url, frontmatter);
if (state.conf.calendars) {
for (const cal_conf of state.conf.calendars) {
handle_calendar(state, cal_conf, out_url, frontmatter);
}
}
}
}
function handle_rss(state: BuildState, rss_conf: RSSConfig, out_url: OutFileURL, frontmatter: any) {
// const field = rss_conf
// const rss_frontmatter = state.
// //
}
function handle_sitemap(state: BuildState, out_url: OutFileURL, frontmatter: any) {
if (! state.conf.sitemap) {
return;
}
const sitemap_frontmatter = state.conf.sitemap.front_matter_field
? frontmatter?.[state.conf.sitemap.front_matter_field] || { }
: frontmatter?.sitemap || { };
state.sitemap.push({
url: out_url.abs_url,
lastmod: state.build_time.iso,
change_freq: sitemap_frontmatter?.change_freq,
priority: sitemap_frontmatter?.priority,
});
}
function handle_event(state: BuildState, out_url: OutFileURL, frontmatter: any) {
//
}
function handle_calendar(state: BuildState, cal_conf: CalendarConfig, out_url: OutFileURL, frontmatter: any) {
//
}
export function file_hash_matches(state: BuildState, in_file: string, new_hash: string) {
const { old_metadata, new_metadata } = state;
if (! old_metadata.last_build?.config_hash) {
return false;
}
if (old_metadata.last_build.config_hash !== new_metadata.last_build.config_hash) {
return false;
}
in_file = in_file.slice(state.conf.input.root.length);
const old_hash = state.old_metadata.files[in_file]?.last_build_hash;
if (! old_hash) {
return false;
}
if (old_hash !== new_hash) {
return false;
}
return true;
}
export function skip_file(state: BuildState, in_file: string, out_file: string, out_url: OutFileURL, frontmatter?: any) {
in_file = in_file.slice(state.conf.input.root.length);
state.new_metadata.files[in_file] = structuredClone(state.old_metadata?.files?.[in_file]);
handle_page_side_effects(state, out_file, out_url, frontmatter);
}
export function update_metadata(state: BuildState, in_file: string, hash: string) {
in_file = in_file.slice(state.conf.input.root.length);
state.new_metadata.files[in_file] = {
first_seen_time: state.old_metadata?.files?.[in_file]?.first_seen_time || state.build_time.iso,
last_build_hash: hash,
last_updated_time: state.build_time.iso,
};
}

96
src/build-files/index.ts Normal file
View File

@@ -0,0 +1,96 @@
import { DateTime } from 'luxon';
import { Config } from '../conf';
import { build_env_scope } from '../env';
import { load_themes } from '../themes';
import { load_extras } from '../template';
import { Metadata } from '../metadata';
import { hash_obj } from '../hash';
import { BuildState } from './state';
import { read_json, with_default_if_missing, write_json } from '../fs';
import { copy_raw_files } from './raw';
import { render_text_file_templates } from './mustache';
import { render_markdown_files } from './markdown';
import { render_json_schema_files } from './jsonschema';
import { write_sitemap_if_needed } from './sitemap';
export { BuildState, ThemeGroups } from './state';
export async function build_docs_project(conf: Config) {
const now = DateTime.now();
const conf_hash = hash_obj(conf);
const themes = await load_themes(conf);
const get_metadata = read_json(conf.metadata, false).then(({ parsed }) => parsed);
const metadata = await with_default_if_missing<Metadata>(get_metadata, {
last_build: null,
files: { },
});
const state: BuildState = {
conf,
seen_files: new Set<string>(),
env: build_env_scope(conf),
layouts: Object.create(null),
themes: themes,
theme_groups: {
all: Object.values(themes),
light: [ ],
dark: [ ],
high_contrast: [ ],
low_contrast: [ ],
monochrome: [ ],
greyscale: [ ],
protanopia_safe: [ ],
deuteranopia_safe: [ ],
tritanopia_safe: [ ],
},
old_metadata: metadata,
new_metadata: {
last_build: {
time: now.toISO(),
config_hash: conf_hash,
},
files: { }
},
extras: await load_extras(),
made_directories: new Set<string>(),
sitemap: [ ],
build_time: {
iso: now.toISO(),
rfc2822: now.toRFC2822(),
},
};
for (const theme of Object.values(themes)) {
for (const label of theme.labels) {
state.theme_groups[label].push(theme);
}
}
if (conf.input.raw) {
await copy_raw_files(state);
}
if (conf.input.text) {
await render_text_file_templates(state);
}
if (conf.input.markdown) {
await render_markdown_files(state);
}
if (conf.input['schema+json'] || conf.input['schema+yaml']) {
await render_json_schema_files(state);
}
// todo: other file types...
await write_sitemap_if_needed(state);
// todo: rss
// todo: events
// Write the updated metadata file
await write_json(conf.metadata, state.new_metadata, true);
}

136
src/build-files/jsonschema.ts Normal file
View File

@@ -0,0 +1,136 @@
import { glob } from 'glob';
import { BuildState } from './state';
import { read_json, write_text, read_yaml } from '../fs';
import { build_markdown_from_json_schema } from '@doc-utils/jsonschema2markdown';
import { stringify as to_yaml } from 'yaml';
import { build_partials, file_hash_matches, map_input_file_to_output_file, map_output_file_to_url, render_page, skip_file, update_metadata } from './helpers';
export async function render_json_schema_files(state: BuildState) {
const promises: Promise<any>[] = [ ];
const json_files = await glob(state.conf.input['schema+json'], {
absolute: true,
cwd: state.conf.input.root,
});
const yaml_files = await glob(state.conf.input['schema+yaml'], {
absolute: true,
cwd: state.conf.input.root,
});
if (! state.partials) {
await build_partials(state);
}
for (const in_file of json_files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
promises.push(
handle_json_schema_json_file(state, in_file)
);
}
for (const in_file of yaml_files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
promises.push(
handle_json_schema_yaml_file(state, in_file)
);
}
await Promise.all(promises);
}
export async function handle_json_schema_json_file(state: BuildState, in_file: string) {
const promises: Promise<any>[] = [ ];
const out_file = map_input_file_to_output_file(state, in_file, [ '.json' ], '.html');
const { frontmatter, parsed, json, hash } = await read_json(in_file, true);
promises.push(
render_json_schema(state, parsed, in_file, await out_file, hash, frontmatter)
);
if (state.conf.output.include_inputs?.includes('schema+json')) {
const json_file = await map_input_file_to_output_file(state, in_file);
promises.push(
write_text(json_file, json)
);
if (state.conf.output.include_yaml_and_json) {
const yaml_file = await map_input_file_to_output_file(state, in_file, [ '.json' ], '.yaml');
promises.push(
write_text(yaml_file, to_yaml(parsed))
);
}
}
await Promise.all(promises);
}
export async function handle_json_schema_yaml_file(state: BuildState, in_file: string) {
const promises: Promise<any>[] = [ ];
const out_file = map_input_file_to_output_file(state, in_file, [ '.yaml', '.yml' ], '.html');
const { frontmatter, parsed, yaml, hash } = await read_yaml(in_file, true);
promises.push(
render_json_schema(state, parsed, in_file, await out_file, hash, frontmatter)
);
if (state.conf.output.include_inputs?.includes('schema+yaml')) {
const yaml_file = await map_input_file_to_output_file(state, in_file);
promises.push(
write_text(yaml_file, yaml)
);
if (state.conf.output.include_yaml_and_json) {
const json_file = await map_input_file_to_output_file(state, in_file, [ '.yaml', '.yml' ], '.json');
promises.push(
write_text(json_file, JSON.stringify(parsed, null, ' '))
);
}
}
await Promise.all(promises);
}
export async function render_json_schema(state: BuildState, schema: unknown, in_file: string, out_file: string, hash: string, frontmatter?: any) {
if (frontmatter?.skip) {
return;
}
const out_url = map_output_file_to_url(state, out_file);
if (file_hash_matches(state, in_file, hash)) {
return skip_file(state, in_file, out_file, out_url, frontmatter);
}
const promises: Promise<any>[] = [ ];
const markdown = build_markdown_from_json_schema(schema);
if (state.conf.output.include_intermediate_markdown) {
promises.push(
map_input_file_to_output_file(state, out_file, [ '.html' ], '.md')
.then((md_file) => write_text(md_file, markdown))
);
}
promises.push(
render_page(state, out_file, out_url, markdown, true, frontmatter)
);
await Promise.all(promises);
update_metadata(state, in_file, hash);
}

48
src/build-files/markdown.ts Normal file
View File

@@ -0,0 +1,48 @@
import { glob } from 'glob';
import { read_text } from '../fs';
import { BuildState } from './state';
import { build_partials, file_hash_matches, map_input_file_to_output_file, map_output_file_to_url, render_page, skip_file, update_metadata } from './helpers';
export async function render_markdown_files(state: BuildState) {
const promises: Promise<any>[] = [ ];
const files = await glob(state.conf.input.markdown, {
absolute: true,
cwd: state.conf.input.root,
});
if (! state.partials) {
await build_partials(state);
}
for (const in_file of files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
promises.push(
render_markdown_file(state, in_file)
);
}
await Promise.all(promises);
}
export async function render_markdown_file(state: BuildState, in_file: string) {
const out_file = await map_input_file_to_output_file(state, in_file, [ '.md', '.markdown' ], '.html');
const out_url = map_output_file_to_url(state, out_file);
const { frontmatter, text, hash } = await read_text(in_file);
if (frontmatter?.skip) {
return;
}
if (file_hash_matches(state, in_file, hash)) {
return skip_file(state, in_file, out_file, out_url, frontmatter);
}
await render_page(state, out_file, out_url, text, true, frontmatter);
update_metadata(state, in_file, hash);
}

48
src/build-files/mustache.ts Normal file
View File

@@ -0,0 +1,48 @@
import { glob } from 'glob';
import { read_text } from '../fs';
import { BuildState } from './state';
import { build_partials, file_hash_matches, map_input_file_to_output_file, map_output_file_to_url, render_page, skip_file, update_metadata } from './helpers';
export async function render_text_file_templates(state: BuildState) {
const promises: Promise<any>[] = [ ];
const files = await glob(state.conf.input.text, {
absolute: true,
cwd: state.conf.input.root,
});
if (! state.partials) {
await build_partials(state);
}
for (const in_file of files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
promises.push(
render_text_file_template(state, in_file)
);
}
await Promise.all(promises);
}
export async function render_text_file_template(state: BuildState, in_file: string) {
const out_file = await map_input_file_to_output_file(state, in_file);
const out_url = map_output_file_to_url(state, out_file);
const { frontmatter, text, hash } = await read_text(in_file);
if (frontmatter?.skip) {
return;
}
if (file_hash_matches(state, in_file, hash)) {
return skip_file(state, in_file, out_file, out_url, frontmatter);
}
await render_page(state, out_file, out_url, text, false, frontmatter);
update_metadata(state, in_file, hash);
}

32
src/build-files/raw.ts Normal file
View File

@@ -0,0 +1,32 @@
import { glob } from 'glob';
import { BuildState } from './state';
import { promises as fs } from 'fs';
import { map_input_file_to_output_file } from './helpers';
export async function copy_raw_files(state: BuildState) {
const promises: Promise<any>[] = [ ];
const files = await glob(state.conf.input.raw, {
absolute: true,
cwd: state.conf.input.root,
});
for (const in_file of files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
const out_file = map_input_file_to_output_file(state, in_file);
// todo: check hashes to see if we can skip
promises.push(
fs.copyFile(in_file, await out_file)
);
}
await Promise.all(promises);
// todo: update metadata
}

57
src/build-files/sitemap.ts Normal file
View File

@@ -0,0 +1,57 @@
import type { XMLBuilder } from 'xmlbuilder2/lib/interfaces';
import { create as create_xml } from 'xmlbuilder2';
import { BuildState } from './state';
import { write_text } from '../fs';
import { resolve as path_resolve } from 'path';
export async function write_sitemap_if_needed(state: BuildState) {
if (! state.conf.sitemap) {
return;
}
const doc = create_xml({ version: '1.0', encoding: 'UTF-8' });
const urlset = doc.ele('urlset', {
'xmlns': 'http://www.sitemaps.org/schemas/sitemap/0.9',
'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
'xsi:schemaLocation': 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd'
});
for (const entry of state.sitemap) {
url_elem(urlset, entry.url, entry.lastmod, entry.change_freq, entry.priority);
}
const out_file = state.conf.sitemap.out_file || path_resolve(state.conf.output.root, 'sitemap.xml');
const xml = doc.toString({
indent: ' ',
prettyPrint: true,
});
await write_text(out_file, xml);
}
export interface SitemapEntry {
url: string;
lastmod: string;
change_freq?: ChangeFreq;
priority?: number;
}
export type ChangeFreq = 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
function url_elem(urlset: XMLBuilder, loc: string, lastmod: string, change_freq?: ChangeFreq, priority?: number) {
const url = urlset.ele('url');
url.ele('loc').txt(loc);
url.ele('lastmod').txt(lastmod);
if (change_freq) {
url.ele('changefreq').txt(change_freq);
}
if (priority != null) {
url.ele('priority').txt(priority.toFixed(1));
}
return url;
}

42
src/build-files/state.ts Normal file
View File

@@ -0,0 +1,42 @@
import type { Config } from '../conf';
import type { Metadata } from '../metadata';
import type { ColorTheme } from '@doc-utils/color-themes';
import type { SitemapEntry } from './sitemap';
export interface BuildState {
conf: Config;
seen_files: Set<string>;
env: Record<string, string>;
partials?: Record<string, string>;
layouts: Record<string, string>;
themes: Record<string, ColorTheme>;
theme_groups: ThemeGroups;
extras: Record<string, string>;
made_directories: Set<string>;
old_metadata: Metadata;
new_metadata: Metadata;
// rss: {
// url: string;
// last_updated: string;
// }[];
sitemap: SitemapEntry[];
// events: EventEntry[];
build_time: {
iso: string;
rfc2822: string;
};
}
export interface ThemeGroups {
all: ColorTheme[];
light: ColorTheme[];
dark: ColorTheme[];
high_contrast: ColorTheme[];
low_contrast: ColorTheme[];
monochrome: ColorTheme[];
greyscale: ColorTheme[];
protanopia_safe: ColorTheme[];
deuteranopia_safe: ColorTheme[];
tritanopia_safe: ColorTheme[];
}

425
src/build.ts
View File

@@ -1,425 +0,0 @@
import { ColorTheme } from '@doc-utils/color-themes';
import { build_markdown_from_json_schema } from '@doc-utils/jsonschema2markdown';
import { render_markdown_to_html, render_markdown_to_html_inline_sync } from '@doc-utils/markdown2html';
import { glob } from 'glob';
import { DateTime } from 'luxon';
import { stringify as to_yaml } from 'yaml';
import { promises as fs } from 'fs';
import { dirname, join as path_join } from 'path';
import { icons } from './icons';
import { Config } from './conf';
import { build_env_scope } from './env';
import { load_themes, render_theme_css_properties } from './themes';
import { mkdirp, read_json, read_text, read_yaml, write_text } from './fs';
import { load_layout, Context, render_template, load_partials, load_extras, FrontMatter } from './template';
interface BuildState {
conf: Config;
seen_files: Set<string>;
env: Record<string, string>;
partials?: Record<string, string>;
layouts: Record<string, string>;
themes: Record<string, ColorTheme>;
theme_groups: ThemeGroups;
extras: Record<string, string>;
made_directories: Set<string>;
build_time: {
iso: string;
rfc2822: string;
};
}
export interface ThemeGroups {
all: ColorTheme[];
light: ColorTheme[];
dark: ColorTheme[];
high_contrast: ColorTheme[];
low_contrast: ColorTheme[];
monochrome: ColorTheme[];
greyscale: ColorTheme[];
protanopia_safe: ColorTheme[];
deuteranopia_safe: ColorTheme[];
tritanopia_safe: ColorTheme[];
}
export async function build_docs_project(conf: Config) {
const now = DateTime.now();
const themes = await load_themes(conf);
const state: BuildState = {
conf,
seen_files: new Set<string>(),
env: build_env_scope(conf),
layouts: Object.create(null),
themes: themes,
theme_groups: {
// fixme: this is horribly inefficient
all: Object.values(themes),
light: Object.values(themes).filter((theme) => theme.labels.includes('light')),
dark: Object.values(themes).filter((theme) => theme.labels.includes('dark')),
high_contrast: Object.values(themes).filter((theme) => theme.labels.includes('high_contrast')),
low_contrast: Object.values(themes).filter((theme) => theme.labels.includes('low_contrast')),
monochrome: Object.values(themes).filter((theme) => theme.labels.includes('monochrome')),
greyscale: Object.values(themes).filter((theme) => theme.labels.includes('greyscale')),
protanopia_safe: Object.values(themes).filter((theme) => theme.labels.includes('protanopia_safe')),
deuteranopia_safe: Object.values(themes).filter((theme) => theme.labels.includes('deuteranopia_safe')),
tritanopia_safe: Object.values(themes).filter((theme) => theme.labels.includes('tritanopia_safe')),
},
extras: await load_extras(),
made_directories: new Set<string>(),
build_time: {
iso: now.toISO(),
rfc2822: now.toRFC2822(),
},
}
if (conf.input.raw) {
await copy_raw_files(state);
}
if (conf.input.text) {
await render_text_file_templates(state);
}
if (conf.input.markdown) {
await render_markdown_files(state);
}
if (conf.input['schema+json'] || conf.input['schema+yaml']) {
await render_json_schema_files(state);
}
// todo...
}
// ===== Raw File Copy =====
async function copy_raw_files(state: BuildState) {
const promises: Promise<any>[] = [ ];
const files = await glob(state.conf.input.raw, {
absolute: true,
cwd: state.conf.input.root,
});
for (const in_file of files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
const out_file = map_input_file_to_output_file(state, in_file);
promises.push(
fs.copyFile(in_file, await out_file)
);
}
await Promise.all(promises);
}
// ===== File Renderers =====
async function render_text_file_templates(state: BuildState) {
const promises: Promise<any>[] = [ ];
const files = await glob(state.conf.input.text, {
absolute: true,
cwd: state.conf.input.root,
});
if (! state.partials) {
await build_partials(state);
}
for (const in_file of files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
promises.push(
render_text_file_template(state, in_file)
);
}
await Promise.all(promises);
}
async function render_text_file_template(state: BuildState, in_file: string) {
const out_file = map_input_file_to_output_file(state, in_file);
const { frontmatter, text } = await read_text(in_file);
let layout: string;
const layout_file = frontmatter?.layout;
if (layout_file) {
if (! state.layouts[layout_file]) {
state.layouts[layout_file] = await load_layout(state.conf, layout_file);
}
layout = state.layouts[layout_file];
}
const tags = state.conf.templates?.tags;
const context = mustache_context(state, frontmatter);
const rendered = render_template(text, context, layout, structuredClone(state.partials), tags);
await write_text(await out_file, rendered);
}
async function render_markdown_files(state: BuildState) {
const promises: Promise<any>[] = [ ];
const files = await glob(state.conf.input.markdown, {
absolute: true,
cwd: state.conf.input.root,
});
if (! state.partials) {
await build_partials(state);
}
for (const in_file of files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
promises.push(
render_markdown_file(state, in_file)
);
}
await Promise.all(promises);
}
async function render_markdown_file(state: BuildState, in_file: string) {
const out_file = map_input_file_to_output_file(state, in_file, [ '.md', '.markdown' ], '.html');
const { frontmatter, text } = await read_text(in_file);
const html = render_markdown_to_html(text, state.conf.markdown);
let layout: string;
const layout_file = frontmatter?.layout;
if (layout_file) {
if (! state.layouts[layout_file]) {
state.layouts[layout_file] = await load_layout(state.conf, layout_file);
}
layout = state.layouts[layout_file];
}
const tags = state.conf.templates?.tags;
const context = mustache_context(state, frontmatter);
const rendered = render_template(await html, context, layout, structuredClone(state.partials), tags);
await write_text(await out_file, rendered);
}
async function render_json_schema_files(state: BuildState) {
const promises: Promise<any>[] = [ ];
const json_files = await glob(state.conf.input['schema+json'], {
absolute: true,
cwd: state.conf.input.root,
});
const yaml_files = await glob(state.conf.input['schema+yaml'], {
absolute: true,
cwd: state.conf.input.root,
});
if (! state.partials) {
await build_partials(state);
}
for (const in_file of json_files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
promises.push(
handle_json_schema_json_file(state, in_file)
);
}
for (const in_file of yaml_files) {
if (state.seen_files.has(in_file)) {
continue;
}
state.seen_files.add(in_file);
promises.push(
handle_json_schema_yaml_file(state, in_file)
);
}
await Promise.all(promises);
}
async function handle_json_schema_json_file(state: BuildState, in_file: string) {
const promises: Promise<any>[] = [ ];
const out_file = map_input_file_to_output_file(state, in_file, [ '.json' ], '.html');
const { frontmatter, parsed, json } = await read_json(in_file, true);
promises.push(
render_json_schema(state, parsed, await out_file, frontmatter)
);
if (state.conf.output.include_inputs?.includes('schema+json')) {
const json_file = await map_input_file_to_output_file(state, in_file);
promises.push(
write_text(json_file, json)
);
if (state.conf.output.include_yaml_and_json) {
const yaml_file = await map_input_file_to_output_file(state, in_file, [ '.json' ], '.yaml');
promises.push(
write_text(yaml_file, to_yaml(parsed))
);
}
}
await Promise.all(promises);
}
async function handle_json_schema_yaml_file(state: BuildState, in_file: string) {
const promises: Promise<any>[] = [ ];
const out_file = map_input_file_to_output_file(state, in_file, [ '.yaml', '.yml' ], '.html');
const { frontmatter, parsed, yaml } = await read_yaml(in_file, true);
promises.push(
render_json_schema(state, parsed, await out_file, frontmatter)
);
if (state.conf.output.include_inputs?.includes('schema+yaml')) {
const yaml_file = await map_input_file_to_output_file(state, in_file);
promises.push(
write_text(yaml_file, yaml)
);
if (state.conf.output.include_yaml_and_json) {
const json_file = await map_input_file_to_output_file(state, in_file, [ '.yaml', '.yml' ], '.json');
promises.push(
write_text(json_file, JSON.stringify(parsed, null, ' '))
);
}
}
await Promise.all(promises);
}
async function render_json_schema(state: BuildState, schema: unknown, out_file: string, frontmatter?: any) {
const promises: Promise<any>[] = [ ];
const markdown = build_markdown_from_json_schema(schema);
if (state.conf.output.include_intermediate_markdown) {
promises.push(
map_input_file_to_output_file(state, out_file, [ '.html' ], '.md')
.then((md_file) => write_text(md_file, markdown))
);
}
const html = render_markdown_to_html(markdown, state.conf.markdown);
let layout: string;
const layout_file = frontmatter?.layout;
if (layout_file) {
if (! state.layouts[layout_file]) {
state.layouts[layout_file] = await load_layout(state.conf, layout_file);
}
layout = state.layouts[layout_file];
}
const tags = state.conf.templates?.tags;
const context = mustache_context(state, frontmatter);
const rendered = render_template(await html, context, layout, structuredClone(state.partials), tags);
promises.push(
write_text(await out_file, rendered)
);
await Promise.all(promises);
}
// ===== Helpers =====
async function map_input_file_to_output_file(state: BuildState, in_file: string, remove_exts?: string[], add_ext?: string) {
if (! in_file.startsWith(state.conf.input.root)) {
throw new Error('input file expected to be inside input root');
}
let out_file = path_join(state.conf.output.root, in_file.slice(state.conf.input.root.length));
if (remove_exts) {
for (const ext of remove_exts) {
if (out_file.endsWith(ext)) {
out_file = out_file.slice(0, -ext.length);
break;
}
}
}
if (add_ext) {
out_file += add_ext;
}
const dir = dirname(out_file);
if (! state.made_directories.has(dir)) {
state.made_directories.add(dir);
await mkdirp(dir);
}
return out_file;
}
async function build_partials(state: BuildState) {
state.partials = await load_partials(state.conf);
for (const [name, theme] of Object.entries(state.themes)) {
state.partials[`.themes/${name}`] = render_theme_css_properties(theme);
}
Object.assign(state.partials, state.extras);
}
function mustache_context(state: BuildState, frontmatter?: FrontMatter) : Context {
return {
env: state.env,
page: frontmatter,
build_time: state.build_time,
icons: icons,
themes: Object.values(state.themes),
theme_groups: structuredClone(state.theme_groups),
markdown: {
render_inline() {
return (text, render) => {
const md = render(text)
const html = render_markdown_to_html_inline_sync(md, state.conf.markdown);
return html;
};
},
}
};
}

56
src/conf.ts
View File

@@ -14,7 +14,21 @@ export async function read_config(file: string) {
return config;
}
export interface RSSConfig {
dir_path?: string;
out_file?: string;
front_matter_field?: string;
}
export interface CalendarConfig {
dir_path?: string;
out_file?: string;
front_matter_field?: string;
}
export interface Config {
metadata: string;
base_url: string;
input: {
root: string;
raw?: string[];
@@ -39,7 +53,16 @@ export interface Config {
include_yaml_and_json?: boolean;
include_intermediate_markdown?: boolean;
};
markdown?: MarkdownOptions;
sitemap?: false | {
out_file?: string;
front_matter_field?: string;
};
rss?: false | RSSConfig[];
events?: false | {
front_matter_field?: string;
};
calendars?: false | CalendarConfig[];
markdown?: Omit<MarkdownOptions, 'base_url' | 'inline' | 'extensions'>;
schema?: {
//
};
@@ -57,6 +80,7 @@ function resolve_paths(file_path: string, config: Config) {
const base_path = dirname(file_path);
config.input.root = resolve_path(base_path, config.input.root);
config.output.root = resolve_path(base_path, config.output.root);
config.metadata = resolve_path(base_path, config.metadata);
if (config.templates?.layouts) {
config.templates.layouts = resolve_path(base_path, config.templates.layouts);
@@ -65,6 +89,36 @@ function resolve_paths(file_path: string, config: Config) {
if (config.templates?.partials) {
config.templates.partials = resolve_path(base_path, config.templates.partials);
}
if ((config.sitemap as boolean) === true) {
config.sitemap = { };
}
if ((config.events as boolean) === true) {
config.events = { };
}
if (Array.isArray(config.rss)) {
for (const rss_conf of config.rss) {
if (rss_conf.dir_path) {
rss_conf.dir_path = resolve_path(config.input.root, rss_conf.dir_path);
rss_conf.out_file = resolve_path(config.output.root, rss_conf.out_file);
}
}
}
if (config.sitemap && typeof config.sitemap === 'object' && config.sitemap.out_file) {
config.sitemap.out_file = resolve_path(config.output.root, config.sitemap.out_file);
}
if (Array.isArray(config.calendars)) {
for (const cal_conf of config.calendars) {
if (cal_conf.dir_path) {
cal_conf.dir_path = resolve_path(config.input.root, cal_conf.dir_path);
cal_conf.out_file = resolve_path(config.output.root, cal_conf.out_file);
}
}
}
}
function process_markdown_config(config: any) {

36
src/fs.ts
View File

@@ -3,6 +3,7 @@ import { process_frontmatter } from '@doc-utils/markdown2html';
import { promises as fs } from 'fs';
import { resolve as resolve_path } from 'path';
import { parse as parse_yaml, stringify as to_yaml } from 'yaml';
import { hash_str } from './hash';
export async function load_from_dir(dir: string, file: string) {
if (! dir) {
@@ -18,41 +19,58 @@ export function mkdirp(dir: string, mode = 0o700) {
return fs.mkdir(dir, { mode, recursive: true });
}
export async function read_text(file: string, check_for_frontmatter = true) {
export async function with_default_if_missing<T>(read_promise: Promise<T>, default_value: T) : Promise<T> {
try {
return await read_promise;
}
catch (error) {
if (error?.code === 'ENOENT') {
return default_value;
}
throw error;
}
}
export async function read_text(file: string, check_for_frontmatter = true, compute_hash = true) {
const text = await fs.readFile(file, 'utf8');
const hash = compute_hash ? hash_str(text) : null;
if (check_for_frontmatter) {
const { frontmatter, document } = process_frontmatter(text);
return { frontmatter, text: document };
return { frontmatter, text: document, hash };
}
return { text, frontmatter: null };
return { text, frontmatter: null, hash };
}
export async function read_json(file: string, check_for_frontmatter = true) {
export async function read_json(file: string, check_for_frontmatter = true, compute_hash = true) {
const json = await fs.readFile(file, 'utf8');
const hash = compute_hash ? hash_str(json) : null;
if (check_for_frontmatter) {
const { frontmatter, document } = process_frontmatter(json);
const parsed = JSON.parse(document);
return { frontmatter, json: document, parsed };
return { frontmatter, json: document, parsed, hash };
}
const parsed = JSON.parse(json);
return { json, parsed, frontmatter: null };
return { json, parsed, frontmatter: null, hash };
}
export async function read_yaml(file: string, check_for_frontmatter = true) {
export async function read_yaml(file: string, check_for_frontmatter = true, compute_hash = true) {
const yaml = await fs.readFile(file, 'utf8');
const hash = compute_hash ? hash_str(yaml) : null;
if (check_for_frontmatter) {
const { frontmatter, document } = process_frontmatter(yaml);
const parsed = parse_yaml(document);
return { frontmatter, yaml: document, parsed };
return { frontmatter, yaml: document, parsed, hash };
}
const parsed = parse_yaml(yaml);
return { yaml, parsed, frontmatter: null };
return { yaml, parsed, frontmatter: null, hash };
}
export function write_text(file: string, text: string) {

14
src/hash.ts Normal file
View File

@@ -0,0 +1,14 @@
import { createHash } from 'crypto';
export function hash_str(str: string) {
const hash = createHash('sha512');
hash.update(str);
return hash.digest('base64');
}
export function hash_obj(obj: any) {
// todo: use something more stable
const str = JSON.stringify(obj);
return hash_str(str);
}

2
src/index.ts
View File

@@ -1,3 +1,3 @@
export { read_config, Config } from './conf';
export { build_docs_project, ThemeGroups } from './build';
export { build_docs_project, ThemeGroups } from './build-files';

16
src/metadata.ts Normal file
View File

@@ -0,0 +1,16 @@
export interface Metadata {
last_build: {
time: string;
config_hash: string;
};
files: {
[file_path: string]: {
first_seen_time: string;
last_build_hash: string;
last_updated_time: string;
};
};
}
//

8
src/template.ts
View File

@@ -6,11 +6,13 @@ import { resolve as resolve_path } from 'path';
import { glob } from 'glob';
import { load_from_dir } from './fs';
import { ColorTheme } from '@doc-utils/color-themes';
import { ThemeGroups } from './build';
import { ThemeGroups } from './build-files';
export interface Context {
env?: Record<string, string>;
page?: FrontMatter;
base_url: string;
page_url: string;
icons: Record<string, string>;
themes: ColorTheme[];
theme_groups: ThemeGroups;
@@ -43,11 +45,11 @@ export async function load_extras() {
'components/outline-inline.js',
'prism.css',
'typography/spacious.css',
'typography/dense.css',
'typography/compact.css',
'typography/general.css',
'theme-animation.css',
'forms-inputs/spacious.css',
'forms-inputs/dense.css',
'forms-inputs/compact.css',
'forms-inputs/general.css',
];