move over to sqlite3 storage

conf/00-default.yaml

@@ -24,7 +24,8 @@ oidc:
   server_url: https://oauth.example.com
   signing_algorithm: ES512
   client_id: your-client-id
-  client_secret: your-client-secret
+  client_secret:
+    from_env: OAUTH_CLIENT_SECRET
 pkce_cookie:
   name: app_pkce_code
   secure: true
@@ -34,12 +35,15 @@ session_cookie:
   name: app_session_key
   secure: true
   ttl: 7200
-  pepper: secret-pepper-value
+  pepper:
+    from_env: SESSION_HASH_PEPPER
 snowflake_uid:
   epoch: 1577836800000
   instance: 0  # todo: This should be populated by a StatefulSet ordinal in k8s; Need to prototype
 storage:
-  engine: file
+  engine: sqlite3
+  pool_min: 2
+  pool_max: 10
 argon2:
   # Using the argon2id variant with a time cost of 3 and memory cost 64MiB (65536)
   # is the recommendation for memory constrained environments, according to RFC 9106. If

schemas/config.json

@@ -145,14 +145,18 @@
 					]
 				},
 				"client_id": {
-					"title": "",
+					"title": "OAuth2 client ID",
 					"description": "",
 					"type": "string"
 				},
 				"client_secret": {
-					"title": "",
+					"allOf": [
-					"description": "",
+            {
-					"type": "string"
+              "title": "OAuth2 client secret",
+              "description": ""
+            },
+            { "$ref": "#/$defs/secret_value" }
+          ]
 				}
 			}
 		},
@@ -212,6 +216,15 @@
 					"description": "Time-to-live for the session key cookie (in seconds)",
 					"type": "integer",
 					"default": 7200
+				},
+				"pepper": {
+					"allOf": [
+            {
+              "title": "Session key hashing pepper",
+              "description": "Cryptographic 'pepper' (or 'secret salt') value, appended to sessions keys before hashing"
+            },
+            { "$ref": "#/$defs/secret_value" }
+          ]
 				}
 			}
 		},
@@ -273,6 +286,29 @@
 			"required": [
 				"engine"
 			]
+		},
+    "secret_value": {
+      "oneOf": [
+        { "$ref": "#/$defs/env_var" },
+        {
+          "title": "",
+          "description": "",
+          "type": "string"
+        }
+      ]
+    },
+    "env_var": {
+      "type": "object",
+      "properties": {
+        "from_env": {
+          "title": "",
+          "description": "",
+          "type": "string"
+        }
+      },
+      "required": [
+        "from_env"
+      ]
     }
 	}
 }

src/http-web/authentication/login-callback.ts

@@ -8,7 +8,7 @@ import type { HttpConfig } from '../../http/server';
 import type { HttpWebDependencies } from '../server';
 import type { HttpURL, Locale, Timezone } from '../../utilities/types';
 import type { FastifyInstance, FastifyRequest, RouteShorthandOptions } from 'fastify';
-import { UserinfoResponse } from 'openid-client';
+import type { UserinfoResponse } from 'openid-client';
 
 export function register_login_callback_endpoint(http_server: FastifyInstance, conf: HttpConfig, deps: HttpWebDependencies) {
 	const { logger, pkce_cookie, oidc, storage, snowflake, session, argon2 } = deps;

src/storage/config.ts

@@ -1,10 +1,8 @@
 
-import type { FileStorageConfig } from './file/config';
 import type { SQLite3StorageConfig } from './sqlite3/config';
 // import type { MariaDBStorageConfig } from './mariadb/config';
 
 export type StorageConfig
-	= FileStorageConfig
+	= SQLite3StorageConfig
-	| SQLite3StorageConfig
 	// | MariaDBStorageConfig
 	;

src/storage/file/config.ts

@@ -1,4 +0,0 @@
-
-export interface FileStorageConfig {
-	engine: 'file';
-}

src/storage/file/index.ts

@@ -1,44 +0,0 @@
-
-import { pino } from 'pino';
-import { make_data_dir } from '../files';
-import { StorageProvider } from '../provider';
-import { FileStorageConfig } from './config';
-import { create_session, get_session, delete_session, cleanup_old_sessions } from './sessions';
-import { get_user, create_user, update_user, delete_user, get_user_by_oidc_sub } from './users';
-
-export function create_file_storage_provider(conf: FileStorageConfig, logger: pino.Logger) : StorageProvider {
-	let status: 'ok' | 'warning' | 'updating' | 'unavailable' = 'unavailable';
-
-	const init_steps = [
-		make_data_dir('sessions'),
-		make_data_dir('users'),
-	];
-
-	const ready = Promise.all(init_steps).then(() => {
-		status = 'ok';
-	});
-
-	return {
-		ready,
-
-		status() {
-			return {
-				engine: 'file',
-				status: status,
-			};
-		},
-		
-		// Login Sessions
-		get_session,
-		create_session,
-		delete_session,
-		cleanup_old_sessions,
-
-		// Users
-		get_user,
-		get_user_by_oidc_sub,
-		create_user,
-		update_user,
-		delete_user,
-	};
-}

src/storage/file/sessions.ts

@@ -1,69 +0,0 @@
-
-import type { SessionData } from '../provider';
-import type { Snowflake } from '../../utilities/snowflake-uid';
-import { read_data_file, write_data_file, delete_data_file, read_data_dir } from '../files';
-
-export async function create_session(data: SessionData) {
-	const json = JSON.stringify(to_json(data), null, '  ');
-	await write_data_file(`sessions/${data.prefix}`, json, 'wx');
-}
-
-export async function get_session(prefix: string) {
-	const data = await read_data_file<SessionJson>(`sessions/${prefix}`, 'json');
-	return from_json(data);
-}
-
-export async function delete_session(prefix: string) {
-	await delete_data_file(`sessions/${prefix}`);
-}
-
-export async function cleanup_old_sessions() {
-	const files = await read_data_dir('sessions');
-	const promises = files.map((file) => check_session_for_expiration(file));
-	await Promise.all(promises);
-}
-
-async function check_session_for_expiration(prefix: string) {
-	let session: SessionData;
-
-	try {
-		session = await get_session(prefix);
-	}
-
-	catch (error) {
-		// todo: if the session doesn't exist, skip
-		throw error;
-	}
-
-	if (session.expires.getTime() <= Date.now()) {
-		await delete_session(prefix);
-	}
-}
-
-interface SessionJson {
-	prefix: string;
-	key_hash: string;
-	user_id: Snowflake;
-	started: string;
-	expires: string;
-}
-
-function to_json(session: SessionData) : SessionJson {
-	return {
-		prefix: session.prefix,
-		key_hash: session.key_hash,
-		user_id: session.user_id,
-		started: session.started.toISOString(),
-		expires: session.expires.toISOString(),
-	};
-}
-
-function from_json(json: SessionJson) : SessionData {
-	return {
-		prefix: json.prefix,
-		key_hash: json.key_hash,
-		user_id: json.user_id,
-		started: new Date(Date.parse(json.started)),
-		expires: new Date(Date.parse(json.expires)),
-	};
-}

src/storage/file/users.ts

@@ -1,70 +0,0 @@
-
-import type { UserData } from '../provider';
-import type { Snowflake } from '../../utilities/snowflake-uid';
-import { delete_data_file, read_data_file, write_data_file } from '../files';
-import { memo } from '../../utilities/memo';
-import { HttpURL, Locale, Timezone } from '../../utilities/types';
-
-export async function get_user(user_id: Snowflake) : Promise<UserData> {
-	const data = await read_data_file<UserJson>(`users/${user_id}`, 'json');
-	return from_json(data);
-}
-
-export async function get_user_by_oidc_sub(sub: string) : Promise<UserData> {
-	// todo: get_user_by_oidc_sub
-	return null;
-}
-
-export async function create_user(data: UserData) {
-	const json = JSON.stringify(to_json(data), null, '  ');
-	await write_data_file(`users/${data.id}`, json, 'wx');
-}
-
-export async function update_user(user_id: Snowflake, data: UserData) {
-	// todo: update_user
-}
-
-export async function delete_user(user_id: Snowflake) {
-	await delete_data_file(`users/${user_id}`);
-	// todo: other cleanup
-}
-
-interface UserJson {
-	id: Snowflake;
-	username: string;
-	oidc_subject?: string;
-	name?: string;
-	website?: HttpURL;
-	profile?: HttpURL;
-	picture?: HttpURL;
-	locale: Locale;
-	time_zone: Timezone;
-}
-
-function to_json(user: UserData) : UserJson {
-	return {
-		id: user.id,
-		username: user.username,
-		oidc_subject: user.oidc_subject,
-		name: user.name,
-		locale: user.locale,
-		time_zone: user.timezone,
-		picture: user.picture,
-		profile: user.profile,
-		website: user.website,
-	};
-}
-
-function from_json(json: UserJson) : UserData {
-	return {
-		id: json.id,
-		username: json.username,
-		oidc_subject: json.oidc_subject,
-		name: json.name,
-		locale: json.locale,
-		timezone: json.time_zone,
-		picture: json.picture,
-		profile: json.profile,
-		website: json.website,
-	};
-}

src/storage/index.ts

@@ -1,7 +1,6 @@
 
 import { StorageConfig } from './config';
 import { StorageProvider } from './provider';
-import { create_file_storage_provider } from './file';
 import { create_sqlite3_storage_provider } from './sqlite3';
 // import { create_mariadb_storage_provider } from './mariadb';
 import { pino } from 'pino';
@@ -10,7 +9,6 @@ export type * from './provider';
 
 export function create_storage_provider(conf: StorageConfig, logger: pino.Logger) : StorageProvider {
 	switch (conf.engine) {
-		case 'file': return create_file_storage_provider(conf, logger);
 		case 'sqlite3': return create_sqlite3_storage_provider(conf, logger);
 		// case 'mariadb': return create_mariadb_storage_provider(conf, logger);
 	}

src/storage/provider.ts

@@ -24,7 +24,7 @@ export interface StorageProvider {
 }
 
 export interface StorageStatus {
-	engine: 'file' | 'sqlite3' | 'mysql';
+	engine: 'sqlite3' | 'mariadb';
 	status: 'ok' | 'warning' | 'updating' | 'unavailable';
 }
 

src/storage/sqlite3/config.ts

@@ -3,4 +3,5 @@ export interface SQLite3StorageConfig {
 	engine: 'sqlite3';
 	pool_min: number;
 	pool_max: number;
+	no_migrate: boolean;
 }

src/storage/sqlite3/migrate.ts

@@ -18,7 +18,7 @@ interface Migration {
 /**
  * Ensures that the database files exist and are updated to their latest schema versions
  */
-export async function bring_db_schema_up_to_date(db: DB, no_update = false) {
+export async function bring_db_schema_up_to_date(db: DB, no_migrate = false) {
 	updating = true;
 	const log = db.logger.child({ source: 'storage/sqlite3/migrate' });
 	const conn = await db.settings.acquire();
@@ -38,15 +38,15 @@ export async function bring_db_schema_up_to_date(db: DB, no_update = false) {
 		return;
 	}
 
-	if (db_version > supported_db_version || db_version < 0) {
+	if (db_version > supported_db_version || db_version < 0 || (db_version | 0) !== db_version) {
-		console.error('settings.db seems to contain an unknown or unsupported version');
+		log.fatal('settings.db seems to contain an unknown or unsupported version');
 		await commit(db, conn);
 		await db.settings.release(conn);
 		process.exit(1);
 	}
 
-	if (no_update) {
+	if (no_migrate) {
-		console.error('settings.db is out of date, but no_update flag is set');
+		log.fatal('settings.db is out of date, but no_migrate flag is set');
 		await commit(db, conn);
 		await db.settings.release(conn);
 		process.exit(1);
@@ -85,14 +85,15 @@ create table if not exists migration_history (
 	id ${type.rowid} primary key,
 	version int not null,
 	migrated_at datetime not null
-)`;
+)
+`;
 
 async function create_table_migration_history(db: DB, conn: sqlite3.Database) {
 	await run(db, conn, sql_create_table_migration_history);
 }
 
 const sql_begin_transaction = `
-begin transaction
+begin exclusive transaction
 `;
 
 async function begin_transaction(db: DB, conn: sqlite3.Database) {