From 12a2ccaed31de21b758adac476b9a8bc58ebaf00 Mon Sep 17 00:00:00 2001 From: James Brumond Date: Fri, 28 Jul 2023 20:02:34 -0700 Subject: [PATCH] move over to sqlite3 storage --- conf/00-default.yaml | 10 ++- schemas/config.json | 46 ++++++++++-- src/http-web/authentication/login-callback.ts | 2 +- src/storage/config.ts | 4 +- src/storage/file/config.ts | 4 -- src/storage/file/index.ts | 44 ------------ src/storage/file/sessions.ts | 69 ------------------ src/storage/file/users.ts | 70 ------------------- src/storage/index.ts | 2 - src/storage/provider.ts | 2 +- src/storage/sqlite3/config.ts | 1 + src/storage/sqlite3/migrate.ts | 15 ++-- 12 files changed, 60 insertions(+), 209 deletions(-) delete mode 100644 src/storage/file/config.ts delete mode 100644 src/storage/file/index.ts delete mode 100644 src/storage/file/sessions.ts delete mode 100644 src/storage/file/users.ts diff --git a/conf/00-default.yaml b/conf/00-default.yaml index 31bc853..75ad5c3 100644 --- a/conf/00-default.yaml +++ b/conf/00-default.yaml @@ -24,7 +24,8 @@ oidc: server_url: https://oauth.example.com signing_algorithm: ES512 client_id: your-client-id - client_secret: your-client-secret + client_secret: + from_env: OAUTH_CLIENT_SECRET pkce_cookie: name: app_pkce_code secure: true @@ -34,12 +35,15 @@ session_cookie: name: app_session_key secure: true ttl: 7200 - pepper: secret-pepper-value + pepper: + from_env: SESSION_HASH_PEPPER snowflake_uid: epoch: 1577836800000 instance: 0 # todo: This should be populated by a StatefulSet ordinal in k8s; Need to prototype storage: - engine: file + engine: sqlite3 + pool_min: 2 + pool_max: 10 argon2: # Using the argon2id variant with a time cost of 3 and memory cost 64MiB (65536) # is the recommendation for memory constrained environments, according to RFC 9106. If diff --git a/schemas/config.json b/schemas/config.json index 5a99767..f6a5ddf 100644 --- a/schemas/config.json +++ b/schemas/config.json @@ -145,14 +145,18 @@ ] }, "client_id": { - "title": "", + "title": "OAuth2 client ID", "description": "", "type": "string" }, "client_secret": { - "title": "", - "description": "", - "type": "string" + "allOf": [ + { + "title": "OAuth2 client secret", + "description": "" + }, + { "$ref": "#/$defs/secret_value" } + ] } } }, @@ -212,6 +216,15 @@ "description": "Time-to-live for the session key cookie (in seconds)", "type": "integer", "default": 7200 + }, + "pepper": { + "allOf": [ + { + "title": "Session key hashing pepper", + "description": "Cryptographic 'pepper' (or 'secret salt') value, appended to sessions keys before hashing" + }, + { "$ref": "#/$defs/secret_value" } + ] } } }, @@ -273,6 +286,29 @@ "required": [ "engine" ] - } + }, + "secret_value": { + "oneOf": [ + { "$ref": "#/$defs/env_var" }, + { + "title": "", + "description": "", + "type": "string" + } + ] + }, + "env_var": { + "type": "object", + "properties": { + "from_env": { + "title": "", + "description": "", + "type": "string" + } + }, + "required": [ + "from_env" + ] + } } } \ No newline at end of file diff --git a/src/http-web/authentication/login-callback.ts b/src/http-web/authentication/login-callback.ts index e6f8a4e..42c04ea 100644 --- a/src/http-web/authentication/login-callback.ts +++ b/src/http-web/authentication/login-callback.ts @@ -8,7 +8,7 @@ import type { HttpConfig } from '../../http/server'; import type { HttpWebDependencies } from '../server'; import type { HttpURL, Locale, Timezone } from '../../utilities/types'; import type { FastifyInstance, FastifyRequest, RouteShorthandOptions } from 'fastify'; -import { UserinfoResponse } from 'openid-client'; +import type { UserinfoResponse } from 'openid-client'; export function register_login_callback_endpoint(http_server: FastifyInstance, conf: HttpConfig, deps: HttpWebDependencies) { const { logger, pkce_cookie, oidc, storage, snowflake, session, argon2 } = deps; diff --git a/src/storage/config.ts b/src/storage/config.ts index 5ee820e..4985b60 100644 --- a/src/storage/config.ts +++ b/src/storage/config.ts @@ -1,10 +1,8 @@ -import type { FileStorageConfig } from './file/config'; import type { SQLite3StorageConfig } from './sqlite3/config'; // import type { MariaDBStorageConfig } from './mariadb/config'; export type StorageConfig - = FileStorageConfig - | SQLite3StorageConfig + = SQLite3StorageConfig // | MariaDBStorageConfig ; diff --git a/src/storage/file/config.ts b/src/storage/file/config.ts deleted file mode 100644 index 12e8541..0000000 --- a/src/storage/file/config.ts +++ /dev/null @@ -1,4 +0,0 @@ - -export interface FileStorageConfig { - engine: 'file'; -} diff --git a/src/storage/file/index.ts b/src/storage/file/index.ts deleted file mode 100644 index 988907b..0000000 --- a/src/storage/file/index.ts +++ /dev/null @@ -1,44 +0,0 @@ - -import { pino } from 'pino'; -import { make_data_dir } from '../files'; -import { StorageProvider } from '../provider'; -import { FileStorageConfig } from './config'; -import { create_session, get_session, delete_session, cleanup_old_sessions } from './sessions'; -import { get_user, create_user, update_user, delete_user, get_user_by_oidc_sub } from './users'; - -export function create_file_storage_provider(conf: FileStorageConfig, logger: pino.Logger) : StorageProvider { - let status: 'ok' | 'warning' | 'updating' | 'unavailable' = 'unavailable'; - - const init_steps = [ - make_data_dir('sessions'), - make_data_dir('users'), - ]; - - const ready = Promise.all(init_steps).then(() => { - status = 'ok'; - }); - - return { - ready, - - status() { - return { - engine: 'file', - status: status, - }; - }, - - // Login Sessions - get_session, - create_session, - delete_session, - cleanup_old_sessions, - - // Users - get_user, - get_user_by_oidc_sub, - create_user, - update_user, - delete_user, - }; -} diff --git a/src/storage/file/sessions.ts b/src/storage/file/sessions.ts deleted file mode 100644 index 8e27129..0000000 --- a/src/storage/file/sessions.ts +++ /dev/null @@ -1,69 +0,0 @@ - -import type { SessionData } from '../provider'; -import type { Snowflake } from '../../utilities/snowflake-uid'; -import { read_data_file, write_data_file, delete_data_file, read_data_dir } from '../files'; - -export async function create_session(data: SessionData) { - const json = JSON.stringify(to_json(data), null, ' '); - await write_data_file(`sessions/${data.prefix}`, json, 'wx'); -} - -export async function get_session(prefix: string) { - const data = await read_data_file(`sessions/${prefix}`, 'json'); - return from_json(data); -} - -export async function delete_session(prefix: string) { - await delete_data_file(`sessions/${prefix}`); -} - -export async function cleanup_old_sessions() { - const files = await read_data_dir('sessions'); - const promises = files.map((file) => check_session_for_expiration(file)); - await Promise.all(promises); -} - -async function check_session_for_expiration(prefix: string) { - let session: SessionData; - - try { - session = await get_session(prefix); - } - - catch (error) { - // todo: if the session doesn't exist, skip - throw error; - } - - if (session.expires.getTime() <= Date.now()) { - await delete_session(prefix); - } -} - -interface SessionJson { - prefix: string; - key_hash: string; - user_id: Snowflake; - started: string; - expires: string; -} - -function to_json(session: SessionData) : SessionJson { - return { - prefix: session.prefix, - key_hash: session.key_hash, - user_id: session.user_id, - started: session.started.toISOString(), - expires: session.expires.toISOString(), - }; -} - -function from_json(json: SessionJson) : SessionData { - return { - prefix: json.prefix, - key_hash: json.key_hash, - user_id: json.user_id, - started: new Date(Date.parse(json.started)), - expires: new Date(Date.parse(json.expires)), - }; -} diff --git a/src/storage/file/users.ts b/src/storage/file/users.ts deleted file mode 100644 index e791bdf..0000000 --- a/src/storage/file/users.ts +++ /dev/null @@ -1,70 +0,0 @@ - -import type { UserData } from '../provider'; -import type { Snowflake } from '../../utilities/snowflake-uid'; -import { delete_data_file, read_data_file, write_data_file } from '../files'; -import { memo } from '../../utilities/memo'; -import { HttpURL, Locale, Timezone } from '../../utilities/types'; - -export async function get_user(user_id: Snowflake) : Promise { - const data = await read_data_file(`users/${user_id}`, 'json'); - return from_json(data); -} - -export async function get_user_by_oidc_sub(sub: string) : Promise { - // todo: get_user_by_oidc_sub - return null; -} - -export async function create_user(data: UserData) { - const json = JSON.stringify(to_json(data), null, ' '); - await write_data_file(`users/${data.id}`, json, 'wx'); -} - -export async function update_user(user_id: Snowflake, data: UserData) { - // todo: update_user -} - -export async function delete_user(user_id: Snowflake) { - await delete_data_file(`users/${user_id}`); - // todo: other cleanup -} - -interface UserJson { - id: Snowflake; - username: string; - oidc_subject?: string; - name?: string; - website?: HttpURL; - profile?: HttpURL; - picture?: HttpURL; - locale: Locale; - time_zone: Timezone; -} - -function to_json(user: UserData) : UserJson { - return { - id: user.id, - username: user.username, - oidc_subject: user.oidc_subject, - name: user.name, - locale: user.locale, - time_zone: user.timezone, - picture: user.picture, - profile: user.profile, - website: user.website, - }; -} - -function from_json(json: UserJson) : UserData { - return { - id: json.id, - username: json.username, - oidc_subject: json.oidc_subject, - name: json.name, - locale: json.locale, - timezone: json.time_zone, - picture: json.picture, - profile: json.profile, - website: json.website, - }; -} diff --git a/src/storage/index.ts b/src/storage/index.ts index 3ea8a82..75181ea 100644 --- a/src/storage/index.ts +++ b/src/storage/index.ts @@ -1,7 +1,6 @@ import { StorageConfig } from './config'; import { StorageProvider } from './provider'; -import { create_file_storage_provider } from './file'; import { create_sqlite3_storage_provider } from './sqlite3'; // import { create_mariadb_storage_provider } from './mariadb'; import { pino } from 'pino'; @@ -10,7 +9,6 @@ export type * from './provider'; export function create_storage_provider(conf: StorageConfig, logger: pino.Logger) : StorageProvider { switch (conf.engine) { - case 'file': return create_file_storage_provider(conf, logger); case 'sqlite3': return create_sqlite3_storage_provider(conf, logger); // case 'mariadb': return create_mariadb_storage_provider(conf, logger); } diff --git a/src/storage/provider.ts b/src/storage/provider.ts index 5dac8b3..5117b2c 100644 --- a/src/storage/provider.ts +++ b/src/storage/provider.ts @@ -24,7 +24,7 @@ export interface StorageProvider { } export interface StorageStatus { - engine: 'file' | 'sqlite3' | 'mysql'; + engine: 'sqlite3' | 'mariadb'; status: 'ok' | 'warning' | 'updating' | 'unavailable'; } diff --git a/src/storage/sqlite3/config.ts b/src/storage/sqlite3/config.ts index 08fe6bb..4bb4594 100644 --- a/src/storage/sqlite3/config.ts +++ b/src/storage/sqlite3/config.ts @@ -3,4 +3,5 @@ export interface SQLite3StorageConfig { engine: 'sqlite3'; pool_min: number; pool_max: number; + no_migrate: boolean; } diff --git a/src/storage/sqlite3/migrate.ts b/src/storage/sqlite3/migrate.ts index a08d3e1..aba34a7 100644 --- a/src/storage/sqlite3/migrate.ts +++ b/src/storage/sqlite3/migrate.ts @@ -18,7 +18,7 @@ interface Migration { /** * Ensures that the database files exist and are updated to their latest schema versions */ -export async function bring_db_schema_up_to_date(db: DB, no_update = false) { +export async function bring_db_schema_up_to_date(db: DB, no_migrate = false) { updating = true; const log = db.logger.child({ source: 'storage/sqlite3/migrate' }); const conn = await db.settings.acquire(); @@ -38,15 +38,15 @@ export async function bring_db_schema_up_to_date(db: DB, no_update = false) { return; } - if (db_version > supported_db_version || db_version < 0) { - console.error('settings.db seems to contain an unknown or unsupported version'); + if (db_version > supported_db_version || db_version < 0 || (db_version | 0) !== db_version) { + log.fatal('settings.db seems to contain an unknown or unsupported version'); await commit(db, conn); await db.settings.release(conn); process.exit(1); } - if (no_update) { - console.error('settings.db is out of date, but no_update flag is set'); + if (no_migrate) { + log.fatal('settings.db is out of date, but no_migrate flag is set'); await commit(db, conn); await db.settings.release(conn); process.exit(1); @@ -85,14 +85,15 @@ create table if not exists migration_history ( id ${type.rowid} primary key, version int not null, migrated_at datetime not null -)`; +) +`; async function create_table_migration_history(db: DB, conn: sqlite3.Database) { await run(db, conn, sql_create_table_migration_history); } const sql_begin_transaction = ` -begin transaction +begin exclusive transaction `; async function begin_transaction(db: DB, conn: sqlite3.Database) {