minimal/dashboard
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move over to sqlite3 storage

Browse Source
This commit is contained in:
James Brumond 2023-07-28 20:02:34 -07:00
parent dc6e01db14
commit 12a2ccaed3
Signed by: james
GPG Key ID: E8F2FC44BAA3357A
12 changed files with 60 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
conf/00-default.yaml
View File

@ -24,7 +24,8 @@ oidc:
server_url: https://oauth.example.com
signing_algorithm: ES512
client_id: your-client-id
client_secret: your-client-secret
client_secret:
from_env: OAUTH_CLIENT_SECRET
pkce_cookie:
name: app_pkce_code
secure: true
@ -34,12 +35,15 @@ session_cookie:
name: app_session_key
secure: true
ttl: 7200
pepper: secret-pepper-value
pepper:
from_env: SESSION_HASH_PEPPER
snowflake_uid:
epoch: 1577836800000
instance: 0 # todo: This should be populated by a StatefulSet ordinal in k8s; Need to prototype
storage:
engine: file
engine: sqlite3
pool_min: 2
pool_max: 10
argon2:
# Using the argon2id variant with a time cost of 3 and memory cost 64MiB (65536)
# is the recommendation for memory constrained environments, according to RFC 9106. If

46
schemas/config.json
View File

@ -145,14 +145,18 @@
]
},
"client_id": {
"title": "",
"title": "OAuth2 client ID",
"description": "",
"type": "string"
},
"client_secret": {
"title": "",
"description": "",
"type": "string"
"allOf": [
{
"title": "OAuth2 client secret",
"description": ""
},
{ "$ref": "#/$defs/secret_value" }
]
}
}
},
@ -212,6 +216,15 @@
"description": "Time-to-live for the session key cookie (in seconds)",
"type": "integer",
"default": 7200
},
"pepper": {
"allOf": [
{
"title": "Session key hashing pepper",
"description": "Cryptographic 'pepper' (or 'secret salt') value, appended to sessions keys before hashing"
},
{ "$ref": "#/$defs/secret_value" }
]
}
}
},
@ -273,6 +286,29 @@
"required": [
"engine"
]
}
},
"secret_value": {
"oneOf": [
{ "$ref": "#/$defs/env_var" },
{
"title": "",
"description": "",
"type": "string"
}
]
},
"env_var": {
"type": "object",
"properties": {
"from_env": {
"title": "",
"description": "",
"type": "string"
}
},
"required": [
"from_env"
]
}
}
}

2
src/http-web/authentication/login-callback.ts
View File

@ -8,7 +8,7 @@ import type { HttpConfig } from '../../http/server';
import type { HttpWebDependencies } from '../server';
import type { HttpURL, Locale, Timezone } from '../../utilities/types';
import type { FastifyInstance, FastifyRequest, RouteShorthandOptions } from 'fastify';
import { UserinfoResponse } from 'openid-client';
import type { UserinfoResponse } from 'openid-client';
export function register_login_callback_endpoint(http_server: FastifyInstance, conf: HttpConfig, deps: HttpWebDependencies) {
const { logger, pkce_cookie, oidc, storage, snowflake, session, argon2 } = deps;

4
src/storage/config.ts
View File

@ -1,10 +1,8 @@
import type { FileStorageConfig } from './file/config';
import type { SQLite3StorageConfig } from './sqlite3/config';
// import type { MariaDBStorageConfig } from './mariadb/config';
export type StorageConfig
= FileStorageConfig
| SQLite3StorageConfig
= SQLite3StorageConfig
// | MariaDBStorageConfig
;

4
src/storage/file/config.ts
View File

@ -1,4 +0,0 @@
export interface FileStorageConfig {
engine: 'file';
}

44
src/storage/file/index.ts
View File

@ -1,44 +0,0 @@
import { pino } from 'pino';
import { make_data_dir } from '../files';
import { StorageProvider } from '../provider';
import { FileStorageConfig } from './config';
import { create_session, get_session, delete_session, cleanup_old_sessions } from './sessions';
import { get_user, create_user, update_user, delete_user, get_user_by_oidc_sub } from './users';
export function create_file_storage_provider(conf: FileStorageConfig, logger: pino.Logger) : StorageProvider {
let status: 'ok' | 'warning' | 'updating' | 'unavailable' = 'unavailable';
const init_steps = [
make_data_dir('sessions'),
make_data_dir('users'),
];
const ready = Promise.all(init_steps).then(() => {
status = 'ok';
});
return {
ready,
status() {
return {
engine: 'file',
status: status,
};
},
// Login Sessions
get_session,
create_session,
delete_session,
cleanup_old_sessions,
// Users
get_user,
get_user_by_oidc_sub,
create_user,
update_user,
delete_user,
};
}

69
src/storage/file/sessions.ts
View File

@ -1,69 +0,0 @@
import type { SessionData } from '../provider';
import type { Snowflake } from '../../utilities/snowflake-uid';
import { read_data_file, write_data_file, delete_data_file, read_data_dir } from '../files';
export async function create_session(data: SessionData) {
const json = JSON.stringify(to_json(data), null, ' ');
await write_data_file(`sessions/${data.prefix}`, json, 'wx');
}
export async function get_session(prefix: string) {
const data = await read_data_file<SessionJson>(`sessions/${prefix}`, 'json');
return from_json(data);
}
export async function delete_session(prefix: string) {
await delete_data_file(`sessions/${prefix}`);
}
export async function cleanup_old_sessions() {
const files = await read_data_dir('sessions');
const promises = files.map((file) => check_session_for_expiration(file));
await Promise.all(promises);
}
async function check_session_for_expiration(prefix: string) {
let session: SessionData;
try {
session = await get_session(prefix);
}
catch (error) {
// todo: if the session doesn't exist, skip
throw error;
}
if (session.expires.getTime() <= Date.now()) {
await delete_session(prefix);
}
}
interface SessionJson {
prefix: string;
key_hash: string;
user_id: Snowflake;
started: string;
expires: string;
}
function to_json(session: SessionData) : SessionJson {
return {
prefix: session.prefix,
key_hash: session.key_hash,
user_id: session.user_id,
started: session.started.toISOString(),
expires: session.expires.toISOString(),
};
}
function from_json(json: SessionJson) : SessionData {
return {
prefix: json.prefix,
key_hash: json.key_hash,
user_id: json.user_id,
started: new Date(Date.parse(json.started)),
expires: new Date(Date.parse(json.expires)),
};
}

70
src/storage/file/users.ts
View File

@ -1,70 +0,0 @@
import type { UserData } from '../provider';
import type { Snowflake } from '../../utilities/snowflake-uid';
import { delete_data_file, read_data_file, write_data_file } from '../files';
import { memo } from '../../utilities/memo';
import { HttpURL, Locale, Timezone } from '../../utilities/types';
export async function get_user(user_id: Snowflake) : Promise<UserData> {
const data = await read_data_file<UserJson>(`users/${user_id}`, 'json');
return from_json(data);
}
export async function get_user_by_oidc_sub(sub: string) : Promise<UserData> {
// todo: get_user_by_oidc_sub
return null;
}
export async function create_user(data: UserData) {
const json = JSON.stringify(to_json(data), null, ' ');
await write_data_file(`users/${data.id}`, json, 'wx');
}
export async function update_user(user_id: Snowflake, data: UserData) {
// todo: update_user
}
export async function delete_user(user_id: Snowflake) {
await delete_data_file(`users/${user_id}`);
// todo: other cleanup
}
interface UserJson {
id: Snowflake;
username: string;
oidc_subject?: string;
name?: string;
website?: HttpURL;
profile?: HttpURL;
picture?: HttpURL;
locale: Locale;
time_zone: Timezone;
}
function to_json(user: UserData) : UserJson {
return {
id: user.id,
username: user.username,
oidc_subject: user.oidc_subject,
name: user.name,
locale: user.locale,
time_zone: user.timezone,
picture: user.picture,
profile: user.profile,
website: user.website,
};
}
function from_json(json: UserJson) : UserData {
return {
id: json.id,
username: json.username,
oidc_subject: json.oidc_subject,
name: json.name,
locale: json.locale,
timezone: json.time_zone,
picture: json.picture,
profile: json.profile,
website: json.website,
};
}

2
src/storage/index.ts
View File

@ -1,7 +1,6 @@
import { StorageConfig } from './config';
import { StorageProvider } from './provider';
import { create_file_storage_provider } from './file';
import { create_sqlite3_storage_provider } from './sqlite3';
// import { create_mariadb_storage_provider } from './mariadb';
import { pino } from 'pino';
@ -10,7 +9,6 @@ export type * from './provider';
export function create_storage_provider(conf: StorageConfig, logger: pino.Logger) : StorageProvider {
switch (conf.engine) {
case 'file': return create_file_storage_provider(conf, logger);
case 'sqlite3': return create_sqlite3_storage_provider(conf, logger);
// case 'mariadb': return create_mariadb_storage_provider(conf, logger);
}

2
src/storage/provider.ts
View File

@ -24,7 +24,7 @@ export interface StorageProvider {
}
export interface StorageStatus {
engine: 'file' | 'sqlite3' | 'mysql';
engine: 'sqlite3' | 'mariadb';
status: 'ok' | 'warning' | 'updating' | 'unavailable';
}

1
src/storage/sqlite3/config.ts
View File

@ -3,4 +3,5 @@ export interface SQLite3StorageConfig {
engine: 'sqlite3';
pool_min: number;
pool_max: number;
no_migrate: boolean;
}

15
src/storage/sqlite3/migrate.ts
View File

@ -18,7 +18,7 @@ interface Migration {
/**
* Ensures that the database files exist and are updated to their latest schema versions
*/
export async function bring_db_schema_up_to_date(db: DB, no_update = false) {
export async function bring_db_schema_up_to_date(db: DB, no_migrate = false) {
updating = true;
const log = db.logger.child({ source: 'storage/sqlite3/migrate' });
const conn = await db.settings.acquire();
@ -38,15 +38,15 @@ export async function bring_db_schema_up_to_date(db: DB, no_update = false) {
return;
}
if (db_version > supported_db_version || db_version < 0) {
console.error('settings.db seems to contain an unknown or unsupported version');
if (db_version > supported_db_version || db_version < 0 || (db_version | 0) !== db_version) {
log.fatal('settings.db seems to contain an unknown or unsupported version');
await commit(db, conn);
await db.settings.release(conn);
process.exit(1);
}
if (no_update) {
console.error('settings.db is out of date, but no_update flag is set');
if (no_migrate) {
log.fatal('settings.db is out of date, but no_migrate flag is set');
await commit(db, conn);
await db.settings.release(conn);
process.exit(1);
@ -85,14 +85,15 @@ create table if not exists migration_history (
id ${type.rowid} primary key,
version int not null,
migrated_at datetime not null
)`;
)
`;
async function create_table_migration_history(db: DB, conn: sqlite3.Database) {
await run(db, conn, sql_create_table_migration_history);
}
const sql_begin_transaction = `
begin transaction
begin exclusive transaction
`;
async function begin_transaction(db: DB, conn: sqlite3.Database) {