work on weather services; outbound http request caching; styles updates and icons

This commit is contained in:
2023-08-12 19:19:20 -07:00
parent 7c205632cb
commit 85d72b43d2
55 changed files with 2593 additions and 234 deletions

View File

@@ -2,7 +2,7 @@
import { promises as fs } from 'fs';
import { join as path_join } from 'path';
import { parse as parse_yaml } from 'yaml';
import { deep_merge } from './utilities/deep-merge';
import { deep_merge } from './utilities/deep';
import { OIDCConfig, validate_oidc_conf } from './security/openid-connect';
import { PKCECookieConfig, validate_pkce_cookie_conf } from './security/pkce-cookie';
import { SnowflakeConfig, validate_snowflake_conf } from './utilities/snowflake-uid';
@@ -12,6 +12,8 @@ import { Argon2HashConfig } from './security/argon-hash';
import { SessionCookieConfig, validate_session_cookie_conf } from './security/session';
import { HttpConfig } from './http/server';
import { ColorThemeConfig } from './utilities/color-themes';
import { ServicesConfig, validate_services_conf } from './services';
import { OutboundHttpConfig, validate_outbound_http_conf } from './http/outbound';
const conf_dir = process.env.CONF_PATH;
@@ -34,6 +36,22 @@ export interface Conf {
storage: StorageConfig;
argon2: Argon2HashConfig;
color_themes: ColorThemeConfig;
outbound_http: OutboundHttpConfig;
services: ServicesConfig;
}
export type SecretValue = string | { from_env: string; };
export function resolve_secret_value(value: SecretValue) {
if (typeof value === 'string') {
return value;
}
if (! value?.from_env) {
return null;
}
return process.env[value.from_env];
}
export async function load_conf() : Promise<unknown> {
@@ -100,4 +118,20 @@ export function validate_conf(conf: unknown) : asserts conf is Conf {
else {
throw new Error('`conf.session_cookie` is missing');
}
if ('outbound_http' in conf) {
validate_outbound_http_conf(conf.outbound_http);
}
else {
throw new Error('`conf.outbound_http` is missing');
}
if ('services' in conf) {
validate_services_conf(conf.services);
}
else {
throw new Error('`conf.services` is missing');
}
}

41
src/http-web/assets.ts Normal file
View File

@@ -0,0 +1,41 @@
import { Req } from '../http/request';
import { HttpConfig } from '../http/server';
import { HttpWebDependencies } from './server';
import { csp_headers } from '../http/content-security-policy';
import { FastifyInstance, RouteShorthandOptions } from 'fastify';
import { load_template } from '../utilities/mustache';
export function register_asset_endpoints(http_server: FastifyInstance, conf: HttpConfig, { color_themes }: HttpWebDependencies) {
const opts: RouteShorthandOptions = {
schema: { },
};
endpoint('forms.css', 'text/css; charset=utf-8');
endpoint('popup.css', 'text/css; charset=utf-8');
endpoint('structure.css', 'text/css; charset=utf-8');
endpoint('typography.css', 'text/css; charset=utf-8');
endpoint('color-theme-controls.css', 'text/css; charset=utf-8');
endpoint('color-theme-controls.js', 'text/javascript');
endpoint('weather.gov/styles.css', 'text/css; charset=utf-8');
endpoint('weatherapi.com/styles.css', 'text/css; charset=utf-8');
http_server.get('/themes.css', opts, async (req: Req, res) => {
res.status(200);
res.header('content-type', 'text/css; charset=utf-8');
res.header('cache-control', 'public, max-age=3600');
csp_headers(res, conf.exposed_url);
return color_themes.css;
});
function endpoint(path: string, media_type: string) {
http_server.get('/' + path, opts, async (req: Req, res) => {
res.status(200);
res.header('content-type', media_type);
res.header('cache-control', 'public, max-age=3600');
csp_headers(res, conf.exposed_url);
return load_template(path);
});
}
}

View File

@@ -1,20 +0,0 @@
import { Req } from '../../http/request';
import { HttpConfig } from '../../http/server';
import { HttpWebDependencies } from '../server';
import { csp_headers } from '../../http/content-security-policy';
import { FastifyInstance, RouteShorthandOptions } from 'fastify';
export function register_color_themes_endpoint(http_server: FastifyInstance, conf: HttpConfig, { color_themes }: HttpWebDependencies) {
const opts: RouteShorthandOptions = {
schema: { },
};
http_server.get('/themes.css', opts, async (req: Req, res) => {
res.status(200);
res.header('content-type', 'text/css; charset=utf-8');
res.header('cache-control', 'public, max-age=3600');
csp_headers(res, conf.exposed_url);
return color_themes.css;
});
}

View File

@@ -1,22 +0,0 @@
import { Req } from '../../http/request';
import { HttpConfig } from '../../http/server';
import { HttpWebDependencies } from '../server';
import { csp_headers } from '../../http/content-security-policy';
import { FastifyInstance, RouteShorthandOptions } from 'fastify';
import { load_template } from '../../utilities/mustache';
export function register_typography_endpoint(http_server: FastifyInstance, conf: HttpConfig, { }: HttpWebDependencies) {
const opts: RouteShorthandOptions = {
schema: { },
};
http_server.get('/typography.css', opts, async (req: Req, res) => {
res.status(200);
res.header('content-type', 'text/css; charset=utf-8');
res.header('cache-control', 'no-store');
// res.header('cache-control', 'public, max-age=3600');
csp_headers(res, conf.exposed_url);
return load_template('typography.css');
});
}

View File

@@ -1,17 +1,18 @@
import * as sch from '../../utilities/json-schema';
import { render_login_page } from './login-page';
import { send_html_error } from '../../http/send-error';
import { redirect_200_refresh } from '../../http/redirects';
import { render_template } from '../../utilities/mustache';
import type { HttpConfig } from '../../http/server';
import type { HttpWebDependencies } from '../server';
import type { HttpURL, Locale, Timezone } from '../../utilities/types';
import type { FastifyInstance, FastifyRequest, RouteShorthandOptions } from 'fastify';
import type { UserinfoResponse } from 'openid-client';
import type { ErrorCode, ErrorInfo } from '../../http/send-error';
export function register_login_callback_endpoint(http_server: FastifyInstance, conf: HttpConfig, deps: HttpWebDependencies) {
const { logger, pkce_cookie, oidc, storage, snowflake, session, argon2 } = deps;
const { logger, pkce_cookie, oidc, storage, snowflake, session } = deps;
const opts: RouteShorthandOptions = {
schema: {
response: {
@@ -54,13 +55,13 @@ export function register_login_callback_endpoint(http_server: FastifyInstance, c
if (! pkce_code_verifier) {
log.debug('no pkce code verifier provided');
return send_html_error(res, 'oidc_no_pkce_code_verifier', render_login_page);
return send_html_error(res, 'oidc_no_pkce_code_verifier', render_login_failed_page);
}
const params = oidc.parse_callback_params(req.url);
if (! params) {
return send_html_error(res, 'oidc_callback_params_invalid', render_login_page);
return send_html_error(res, 'oidc_callback_params_invalid', render_login_failed_page);
}
log.debug({ callback_params: params }, 'received callback params');
@@ -68,7 +69,7 @@ export function register_login_callback_endpoint(http_server: FastifyInstance, c
const token_set = await oidc.fetch_token_set(`${conf.exposed_url}/login-callback`, params, pkce_code_verifier);
if (! token_set) {
return send_html_error(res, 'oidc_token_fetch_failed', render_login_page);
return send_html_error(res, 'oidc_token_fetch_failed', render_login_failed_page);
}
log.debug('fetched token set; requesting user info');
@@ -76,7 +77,7 @@ export function register_login_callback_endpoint(http_server: FastifyInstance, c
const user_info = await oidc.fetch_user_info(token_set);
if (! user_info) {
return send_html_error(res, 'oidc_userinfo_fetch_failed', render_login_page);
return send_html_error(res, 'oidc_userinfo_fetch_failed', render_login_failed_page);
}
log.debug({ oidc_subject: user_info.sub }, 'fetched user info; looking up local user data');
@@ -124,3 +125,19 @@ function get_zoneinfo(user_info: UserinfoResponse) : Timezone {
const from_userinfo = user_info.zoneinfo as Timezone;
return from_userinfo || 'Africa/Abidjan';
}
function render_login_failed_page(error_code?: ErrorCode, error?: ErrorInfo) {
const view = {
login_error_code: error_code,
login_error: error,
page_title: 'Login Failed',
};
const partial_files = {
controls: 'controls.html.mustache',
color_theme_controls: 'color-theme-controls.html.mustache',
page_content: 'login-failed.html.mustache',
};
return render_template('page.html.mustache', view, { }, partial_files);
}

View File

@@ -1,43 +0,0 @@
import { HttpConfig } from '../../http/server';
import { HttpWebDependencies } from '../server';
import { ErrorCode, ErrorInfo } from '../../http/send-error';
import { redirect_303_see_other } from '../../http/redirects';
import { FastifyInstance, FastifyReply, RouteShorthandOptions } from 'fastify';
import { csp_headers } from '../../http/content-security-policy';
import { render_template } from '../../utilities/mustache';
export function register_login_page_endpoint(http_server: FastifyInstance, conf: HttpConfig, { pkce_cookie, session }: HttpWebDependencies) {
const opts: RouteShorthandOptions = {
schema: { },
};
http_server.get('/login', opts, async (req, res) => {
try {
await session.check_login(req);
return redirect_303_see_other(res, conf.exposed_url, 'Already logged in');
}
catch (error) {
return send_login_page(res);
}
});
function send_login_page(res: FastifyReply) {
res.status(200);
res.header('content-type', 'text/html; charset=utf-8');
csp_headers(res, conf.exposed_url);
session.reset(res);
pkce_cookie.reset(res);
return render_login_page();
}
}
export function render_login_page(error_code?: ErrorCode, error?: ErrorInfo) {
const view = {
error_code,
error,
};
return render_template('login.html.mustache', view);
}

View File

@@ -6,8 +6,9 @@ import { HttpWebDependencies } from './server';
import { csp_headers } from '../http/content-security-policy';
import { FastifyInstance, RouteShorthandOptions } from 'fastify';
import { render_template } from '../utilities/mustache';
import type { NamedLocation } from '../storage/named-location';
export function register_root_page_endpoint(http_server: FastifyInstance, conf: HttpConfig, { session, logger }: HttpWebDependencies) {
export function register_root_page_endpoint(http_server: FastifyInstance, conf: HttpConfig, { session, services }: HttpWebDependencies) {
const opts: RouteShorthandOptions = {
schema: { },
};
@@ -21,17 +22,47 @@ export function register_root_page_endpoint(http_server: FastifyInstance, conf:
session.reset(res);
}
const location: NamedLocation = {
name: 'Home',
latitude: 45.4961,
longitude: -122.6714,
};
const [
{ value: forecast },
{ value: alerts },
{ value: weather }
] = await Promise.all([
services.weather_gov.get_forecast_for_location(location, 'us'),
services.weather_gov.get_alerts_for_location(location),
services.weatherapi_com.get_current_for_location(location),
]);
const rendered_widgets = await Promise.all([
services.weatherapi_com.render_current_weather(location, weather),
services.weather_gov.render_forecast(location, forecast, alerts),
]);
res.status(200);
res.header('cache-control', 'private, no-cache');
res.header('content-type', 'text/html; charset=utf-8');
csp_headers(res, conf.exposed_url);
return render_root_page(req.session?.user);
return render_root_page(req.session?.user, rendered_widgets);
});
}
function render_root_page(user: UserData) {
function render_root_page(user: UserData, rendered_widgets: string[]) {
const view = {
user,
page_title: 'Dashboard',
rendered_widgets,
};
return render_template('root.html.mustache', view);
const partial_files = {
controls: 'controls.html.mustache',
color_theme_controls: 'color-theme-controls.html.mustache',
page_content: 'root.html.mustache',
};
return render_template('page.html.mustache', view, { }, partial_files);
}

View File

@@ -9,14 +9,15 @@ import { BaseHttpDependencies, HttpConfig, create_http_server } from '../http/se
import { SnowflakeProvider } from '../utilities/snowflake-uid';
import { register_csp_report_endpoint } from '../http/content-security-policy';
import { register_asset_endpoints } from './assets';
import { register_root_page_endpoint } from './root-page';
import { register_login_page_endpoint } from './authentication/login-page';
import { register_submit_login_endpoint } from './authentication/submit-login';
import { register_login_callback_endpoint } from './authentication/login-callback';
import { register_logout_endpoint } from './authentication/logout';
import { ColorThemeProvider } from '../utilities/color-themes';
import { register_color_themes_endpoint } from './assets/color-themes';
import { register_typography_endpoint } from './assets/typography';
import { OutboundHttpProvider } from '../http/outbound';
import { ServicesProvider } from '../services';
import { RSSFeedReaderProvider } from '../utilities/rss-feeds';
export interface HttpWebDependencies extends BaseHttpDependencies {
oidc: OIDCProvider;
@@ -26,25 +27,24 @@ export interface HttpWebDependencies extends BaseHttpDependencies {
session: SessionProvider;
snowflake: SnowflakeProvider;
color_themes: ColorThemeProvider;
outbound_http: OutboundHttpProvider;
rss_feed_reader: RSSFeedReaderProvider;
services: ServicesProvider;
}
export function create_http_web_server(conf: HttpConfig, deps: HttpWebDependencies) {
return create_http_server(conf, deps, {
endpoints: [
register_csp_report_endpoint,
// Shared Assets
register_color_themes_endpoint,
register_typography_endpoint,
// Root page
register_root_page_endpoint,
register_asset_endpoints,
// Login/logout
register_login_page_endpoint,
register_submit_login_endpoint,
register_login_callback_endpoint,
register_logout_endpoint,
// Pages
register_root_page_endpoint,
],
content_parsers: {
// 'application/ld+json': json_content_parser,

200
src/http/outbound.ts Normal file
View File

@@ -0,0 +1,200 @@
import pino from 'pino';
import { URL } from 'url';
import { request as http_request, OutgoingHttpHeaders, IncomingHttpHeaders } from 'http';
import { request as https_request } from 'https';
import { create_request_cache_provider } from './request-cache';
import { HttpURL } from '../utilities/types';
import { deep_copy } from '../utilities/deep';
let next_id = 1;
const timeout = 5000;
export interface HttpResult {
url: URL;
// req: ClientRequest;
// res: IncomingMessage;
status: number;
body: string;
headers: IncomingHttpHeaders;
}
export interface OutboundHttpConfig {
https_only: boolean;
}
export function validate_outbound_http_conf(conf: unknown) : asserts conf is OutboundHttpConfig {
// todo: validate config
}
export type OutboundHttpProvider = ReturnType<typeof create_outbound_http_provider>;
export function create_outbound_http_provider(conf: OutboundHttpConfig, logger: pino.Logger) {
const http_log = logger.child({ logger: 'outbound-http' });
const response_cache = create_request_cache_provider({ }, http_log);
return {
req: http_req,
get(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) {
return http_req('GET', url_str, headers, null, skip_cache);
},
head(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) {
return http_req('HEAD', url_str, headers, null, skip_cache);
},
post(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
return http_req('POST', url_str, headers, body, skip_cache);
},
put(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
return http_req('PUT', url_str, headers, body, skip_cache);
},
patch(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
return http_req('PATCH', url_str, headers, body, skip_cache);
},
delete(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
return http_req('DELETE', url_str, headers, body, skip_cache);
},
};
async function http_req(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
const url = new URL(url_str);
const make_request
= url.protocol === 'https:' ? https_request
: url.protocol === 'http:' ? http_request
: null;
if (! make_request) {
throw new Error('illegal protocol');
}
if (url.protocol !== 'https:' && conf.https_only) {
throw new Error('non-secure http requests not allowed (by configuration)');
}
if (url.username || url.password) {
throw new Error('urls containing user credentials not allowed');
}
const req_headers = headers ? deep_copy(headers) : { };
const result: HttpResult = {
url,
// req: null,
// res: null,
status: null,
body: null,
headers: null,
};
const path = url.pathname + (url.search || '');
const port = url.port ? parseInt(url.port, 10) : (url.protocol === 'https:' ? 443 : 80);
const log = http_log.child({
req_id: next_id++,
method: method,
protocol: url.protocol,
hostname: url.hostname,
port: port,
path: path,
});
const cached = skip_cache
? null
: response_cache.check_for_cached_response(method, url_str, req_headers);
if (cached) {
const revalidation = response_cache.determine_revalidation_needed(cached);
if (! revalidation.must_revalidate) {
log.info('serving request from cache (no revalidation required)');
result.status = cached.status;
result.body = cached.body;
result.headers = deep_copy(cached.res_headers);
return Promise.resolve(result);
}
log.info('cached response found, but requires revalidation');
Object.assign(req_headers, revalidation.headers);
}
log.info('starting outbound http request');
return new Promise<HttpResult>((resolve, reject) => {
const start = Date.now();
const req = make_request({
method: method,
protocol: url.protocol,
hostname: url.hostname,
port: port,
path: path,
headers: req_headers
}, (res) => {
// result.res = res;
result.status = res.statusCode;
log.info({
status: res.statusCode,
duration: Date.now() - start,
}, `headers received`);
switch (result.status) {
case 304:
log.info({ status: cached.status }, 're-using cached response (not modified)');
result.status = cached.status;
result.body = cached.body;
result.headers = deep_copy(cached.res_headers);
response_cache.freshen_cache(cached, res.headers);
resolve(result);
break;
case 500:
case 502:
case 503:
case 504:
if (cached?.cache_info?.cache_control?.stale_if_error) {
log.info({ status: cached.status }, 're-using cached response (response returned an error status, and `stale-if-error` directive was provided)');
result.status = cached.status;
result.body = cached.body;
result.headers = deep_copy(cached.res_headers);
resolve(result);
break;
}
// nobreak
default: {
const start = Date.now();
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
log.info({ duration: Date.now() - start }, `response finished`);
result.body = data;
result.headers = res.headers;
response_cache.add_response_to_cache(method, url_str, req_headers, result);
resolve(result);
});
};
}
});
// result.req = req;
req.setTimeout(timeout, () => {
log.error('request timeout');
req.destroy();
});
req.on('error', (error) => {
log.error('request error ' + error.stack);
reject(error);
});
if (body) {
req.write(body);
}
req.end();
});
}
}

View File

@@ -17,8 +17,13 @@ export interface CacheControl {
no_store?: boolean;
no_cache?: boolean;
max_age?: number;
s_max_age?: number;
must_revalidate?: boolean;
proxy_revalidate?: boolean;
immutable?: boolean;
stale_while_revalidate?: boolean;
stale_if_error?: boolean;
must_understand?: boolean;
}
export function parse_cache_headers(headers: IncomingHttpHeaders) {
@@ -62,12 +67,29 @@ export function parse_cache_headers(headers: IncomingHttpHeaders) {
case 'proxy-revalidate':
result.cache_control.proxy_revalidate = true;
break;
case 'immutable':
result.cache_control.immutable = true;
break;
case 'stale-while-revalidate':
result.cache_control.stale_while_revalidate = true;
break;
case 'stale-if-error':
result.cache_control.stale_if_error = true;
break;
case 'must-understand':
result.cache_control.must_understand = true;
break;
default:
if (directive.startsWith('max-age=')) {
result.cache_control.max_age = parseInt(directive.slice(8), 10);
break;
}
if (directive.startsWith('s-max-age=')) {
result.cache_control.s_max_age = parseInt(directive.slice(10), 10);
break;
}
// todo: log something here about unknown directive
}
}

228
src/http/request-cache.ts Normal file
View File

@@ -0,0 +1,228 @@
import { pino } from 'pino';
import { DateTime } from 'luxon';
import { deep_copy } from '../utilities/deep';
import { OutgoingHttpHeaders, IncomingHttpHeaders } from 'http';
import { parse_cache_headers, ParsedCacheHeaders } from './parse-cache-headers';
import type { HttpURL } from '../utilities/types';
import type { HttpResult } from './outbound';
export interface RequestCacheConfig {
// todo: support for configurable limits:
max_records?: number; // max number of cached responses
max_length?: number; // max content-length for one response
evict_after?: number; // period of time unused after which a response should be evicted
}
export interface CachedResponse {
cache_key: string;
cached_time: DateTime;
is_stale: boolean;
url: HttpURL;
method: string;
req_headers: OutgoingHttpHeaders;
status: number;
body: string;
res_headers: IncomingHttpHeaders;
cache_info: ParsedCacheHeaders;
expire_timer: NodeJS.Timeout;
}
export function validate_request_cache_conf(conf: unknown) : asserts conf is RequestCacheConfig {
// todo: validate config
}
export type RequestCacheProvider = ReturnType<typeof create_request_cache_provider>;
export function create_request_cache_provider(conf: RequestCacheConfig, logger: pino.Logger) {
type URLCache = Record<string, CachedResponse>;
const cached_responses: Record<HttpURL, URLCache> = Object.create(null);
const cacheable_methods = new Set([ 'GET', 'HEAD' ]);
const cacheable_status_codes = new Set([
200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501
]);
return {
/**
* Should be called before making a new request to check if a cached response already exists
*/
check_for_cached_response(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders) : Readonly<CachedResponse> {
if (! cached_responses[url_str]) {
return;
}
for (const [cache_key, cached] of Object.entries(cached_responses[url_str])) {
if (cache_key === header_cache_key(method, cached.cache_info.vary, headers)) {
return cached;
}
}
},
/**
* Should be called when a cached response is found, to determine what if any revalidation is
* required before re-using the response
*/
determine_revalidation_needed(cached: CachedResponse) {
const { cache_control, etag, date, last_modified } = cached.cache_info;
const must_revalidate = Boolean(cache_control?.must_revalidate || cache_control?.no_cache);
const headers: OutgoingHttpHeaders = { };
if (etag) {
headers['if-none-match'] = etag;
}
if (last_modified || date) {
headers['if-modified-since'] = last_modified?.toHTTP() ?? date?.toHTTP();
}
else {
headers['if-modified-since'] = cached.cached_time.toHTTP();
}
return {
must_revalidate,
headers,
};
},
/**
* Should be called when receiving a non-`304 Not Modified` response from a request, either
* as a novel request, or during a revalidation request
*/
add_response_to_cache(method: string, url_str: HttpURL, headers: OutgoingHttpHeaders, result: HttpResult) {
if (! cacheable_methods.has(method) || ! cacheable_status_codes.has(result.status)) {
logger.info('method/status not cacheable');
return;
}
const cache_info = parse_cache_headers(result.headers);
if (cache_info.cache_control?.no_store || cache_info.cache_control?.private) {
logger.info(cache_info, 'response contains cache-control directives that prevent caching');
return;
}
const cache_key = header_cache_key(method, cache_info.vary, headers);
unset_expire_timer(url_str, cache_key);
const cached: CachedResponse = {
cache_key,
cached_time: DateTime.now(),
is_stale: false,
url: url_str,
method,
req_headers: deep_copy(headers),
status: result.status,
body: result.body,
res_headers: deep_copy(result.headers),
cache_info,
expire_timer: null,
};
logger.info(cache_info, 'caching response');
store_to_cache(cached, cache_info);
},
/**
* Should be called when receiving a `304 Not Modified` response during a revalidation request
* to update the cache with more recent freshness info
*/
freshen_cache(cached: CachedResponse, headers: IncomingHttpHeaders) {
const cache_info = parse_cache_headers(headers);
unset_expire_timer(cached.url, cached.cache_key);
store_to_cache(cached, cache_info);
},
};
function store_to_cache(cached: CachedResponse, cache_info: ParsedCacheHeaders) {
if (! cached_responses[cached.url]) {
cached_responses[cached.url] = Object.create(null);
}
cached_responses[cached.url][cached.cache_key] = cached;
cached.is_stale = false;
if (cache_info.cache_control?.max_age) {
let ttl_ms = cache_info.cache_control.max_age * 1000;
if (cache_info.age) {
ttl_ms -= cache_info.age * 1000;
}
if (ttl_ms <= 0) {
cached.is_stale = true;
return;
}
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
}
else if (cache_info.expires) {
const ttl_ms = cache_info.expires.toUnixInteger() - Date.now();
if (ttl_ms <= 0) {
cached.is_stale = true;
return;
}
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
}
// If no cache-control or expires header is given, default to the
// recommended 10% of time since last-modified
else if (cache_info.last_modified) {
const last_modified_ago_ms = Date.now() - cache_info.last_modified.toUnixInteger();
if (last_modified_ago_ms <= 0) {
cached.is_stale = true;
return;
}
const ttl_ms = (last_modified_ago_ms / 10) | 0;
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
}
}
function unset_expire_timer(url_str: HttpURL, cache_key: string) {
const cached = cached_responses[url_str]?.[cache_key];
if (cached?.expire_timer) {
clearTimeout(cached.expire_timer);
}
}
function set_expire_timer(url_str: HttpURL, cache_key: string, ttl_ms: number) {
const timer = setTimeout(expire, ttl_ms);
timer.unref();
return timer;
function expire() {
if (cached_responses[url_str]?.[cache_key]) {
cached_responses[url_str][cache_key].is_stale = true;
}
};
}
}
function header_cache_key(method: string, vary: string[], headers: OutgoingHttpHeaders) {
const parts: string[] = [ method ];
if (vary?.length) {
for (const header of vary) {
if (Array.isArray(headers[header])) {
const values = (headers[header] as string[]).sort();
for (const value of values) {
parts.push(`${header}: ${value}`);
}
}
else if (headers[header]) {
parts.push(`${header}: ${headers[header]}`);
}
}
}
return parts.join('\0');
}

View File

@@ -4,6 +4,7 @@ import { BaseClient, CallbackParamsType, Issuer, TokenSet, TypeOfGenericClient }
import { PKCECookieConfig } from './pkce-cookie';
import { redirect_302_found } from '../http/redirects';
import pino from 'pino';
import { SecretValue, resolve_secret_value } from '../conf';
const scopes = 'openid profile email';
@@ -11,7 +12,7 @@ export interface OIDCConfig {
server_url: string;
signing_algorithm: 'ES512';
client_id: string;
client_secret: string;
client_secret: SecretValue;
pkce: PKCECookieConfig;
}
@@ -33,7 +34,7 @@ export function create_oidc_provider(conf: OIDCConfig, logger: pino.Logger) {
Client = issuer.Client;
client = new Client({
client_id: conf.client_id,
client_secret: conf.client_secret,
client_secret: resolve_secret_value(conf.client_secret),
id_token_signed_response_alg: conf.signing_algorithm,
authorization_signed_response_alg: conf.signing_algorithm,
});

View File

@@ -1,12 +1,13 @@
import { pino } from 'pino';
import { rand } from '../utilities/rand';
import { FastifyReply } from 'fastify';
import { Req } from '../http/request';
import { resolve_secret_value, SecretValue } from '../conf';
import { invalidate_cookie, parse_req_cookies, set_cookie } from '../http/cookies';
import type { Req } from '../http/request';
import type { FastifyReply } from 'fastify';
import type { Snowflake } from '../utilities/snowflake-uid';
import type { Argon2HashProvider } from './argon-hash';
import type { SessionData, StorageProvider, UserData } from '../storage';
import { Snowflake } from '../utilities/snowflake-uid';
export interface SessionKey {
full_key: string;
@@ -20,7 +21,7 @@ export interface SessionCookieConfig {
name: string;
secure: boolean;
ttl: number;
pepper: string;
pepper: SecretValue;
}
export function validate_session_cookie_conf(conf: unknown) : asserts conf is SessionCookieConfig {
@@ -29,6 +30,7 @@ export function validate_session_cookie_conf(conf: unknown) : asserts conf is Se
export function create_session_provider(conf: SessionCookieConfig, logger: pino.Logger, argon2: Argon2HashProvider, storage: StorageProvider) {
const session_logger = logger.child({ logger: 'session' });
const pepper = resolve_secret_value(conf.pepper);
const self = {
async generate_key() : Promise<SessionKey> {
const bytes = await rand(48);
@@ -40,7 +42,7 @@ export function create_session_provider(conf: SessionCookieConfig, logger: pino.
return { prefix, raw_key, full_key };
},
hash_key(key: SessionKey) : Promise<string> {
return argon2.hash(conf.pepper + key.raw_key);
return argon2.hash(pepper + key.raw_key);
},
parse_key(full_key: string) : SessionKey {
const [ prefix, raw_key, ...rest ] = full_key.split('.');
@@ -52,7 +54,7 @@ export function create_session_provider(conf: SessionCookieConfig, logger: pino.
return { prefix, raw_key, full_key };
},
verify_key(key: SessionKey, key_hash: string) : Promise<boolean> {
return argon2.verify(key_hash, conf.pepper + key.raw_key);
return argon2.verify(key_hash, pepper + key.raw_key);
},
write_to_cookie(res: FastifyReply, key: SessionKey) {
const session_expire = new Date(Date.now() + (conf.ttl * 1000));

38
src/services/index.ts Normal file
View File

@@ -0,0 +1,38 @@
import { pino } from 'pino';
import { create_weather_gov_provider, validate_weather_gov_conf, WeatherGovConfig } from './weather.gov';
import type { SecretValue } from '../conf';
import type { OutboundHttpProvider } from '../http/outbound';
import { create_weatherapi_com_provider, validate_weatherapi_com_conf, WeatherAPIComConfig } from './weatherapi.com';
export interface ServicesConfig {
'weather.gov'?: WeatherGovConfig;
'weatherapi.com'?: WeatherAPIComConfig;
'openweathermap.org'?: {
enabled: boolean;
api_key: SecretValue;
};
}
export type ServicesProvider = ReturnType<typeof create_services_provider>;
export function validate_services_conf(conf: unknown) : asserts conf is ServicesConfig {
validate_weather_gov_conf(conf['weather.gov']);
validate_weatherapi_com_conf(conf['weatherapi.com']);
}
export function create_services_provider(conf: ServicesConfig, logger: pino.Logger, outbound_http: OutboundHttpProvider) {
const weather_gov = create_weather_gov_provider(conf['weather.gov'], logger, outbound_http);
const weatherapi_com = create_weatherapi_com_provider(conf['weatherapi.com'], logger, outbound_http);
const ready = Promise.all([
weather_gov.ready,
weatherapi_com.ready,
]);
return {
ready,
weather_gov,
weatherapi_com,
};
}

View File

@@ -0,0 +1,184 @@
import pino from 'pino';
import { memo } from '../../utilities/memo';
import { deep_freeze } from '../../utilities/deep';
import { render_forecast } from './render-forecast';
import type { NamedLocation } from '../../storage/named-location';
import type { OutboundHttpProvider } from '../../http/outbound';
import type { WeatherGovPoints, WeatherGovForecast, WeatherGovAlerts } from './interface';
import { HttpURL } from '../../utilities/types';
export interface WeatherGovConfig {
enabled: boolean;
user_agent: string;
}
export function validate_weather_gov_conf(conf: unknown) : asserts conf is WeatherGovConfig {
// todo: validate config
}
export type WeatherGovProvider = ReturnType<typeof create_weather_gov_provider>;
export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino.Logger, outbound_http: OutboundHttpProvider) {
const log = logger.child({ logger: 'weather.gov' });
const get_points_ttl = 1000 * 60 * 60 * 24;
const get_points = memo(get_points_ttl, get_points_nocache, { });
const get_forecast_ttl = 1000 * 60 * 30;
const get_forecast = memo(get_forecast_ttl, get_forecast_nocache, { });
const get_alerts_ttl = 1000 * 30;
const get_alerts = memo(get_alerts_ttl, get_alerts_nocache, { });
return {
ready: Promise.resolve(),
get enabled() {
return conf?.enabled;
},
get_points: if_enabled(get_points),
get_forecast: if_enabled(get_forecast),
get_alerts: if_enabled(get_alerts),
async get_forecast_for_lat_long(latitude: number, longitude: number, units: 'us' | 'si') {
const { value: points } = await get_points(latitude, longitude);
return get_forecast(points.gridId, points.gridX, points.gridY, units, false);
},
async get_hourly_forecast_for_lat_long(latitude: number, longitude: number, units: 'us' | 'si') {
const { value: points } = await get_points(latitude, longitude);
return get_forecast(points.gridId, points.gridX, points.gridY, units, true);
},
async get_forecast_for_location(location: NamedLocation, units: 'us' | 'si') {
const { value: points } = await get_points(location.latitude, location.longitude);
return get_forecast(points.gridId, points.gridX, points.gridY, units, false);
},
async get_hourly_forecast_for_location(location: NamedLocation, units: 'us' | 'si') {
const { value: points } = await get_points(location.latitude, location.longitude);
return get_forecast(points.gridId, points.gridX, points.gridY, units, true);
},
async get_alerts_for_location(location: NamedLocation) {
return get_alerts(location.latitude, location.longitude);
},
render_forecast,
};
function if_enabled<T>(func: T) : T {
if (conf?.enabled) {
return func;
}
return (() => {
throw new Error('weather.gov service not enabled');
}) as T;
}
async function get_points_nocache(latitude: number, longitude: number) : Promise<Cachable<WeatherGovPoints>> {
log.info({ latitude, longitude }, 'fetching points data');
let { status, headers, body } = await outbound_http.get(`https://api.weather.gov/points/${latitude},${longitude}`, {
'accept': 'application/ld+json',
'user-agent': conf.user_agent,
});
// Follow a single redirect if given
if (status === 301) {
({ status, headers, body } = await outbound_http.get(headers.location as HttpURL, {
'accept': 'application/ld+json',
'user-agent': conf.user_agent,
}));
}
if (status === 200) {
const data = JSON.parse(body);
return cacheable(get_points_ttl, data as WeatherGovPoints);
}
const data = headers['content-type'].startsWith('application/problem+json')
? JSON.parse(body)
: { status };
log.error(data, 'failed to fetch points data');
throw new Error('failed to fetch points data');
}
async function get_forecast_nocache(gridId: string, gridX: number, gridY: number, units: 'us' | 'si', hourly = false) : Promise<Cachable<WeatherGovForecast>> {
log.info({ gridId, gridX, gridY }, 'fetching forecast info');
const url = `https://api.weather.gov/gridpoints/${gridId}/${gridX},${gridY}/forecast${hourly ? '/hourly' : ''}?units=${units}`;
let { status, headers, body } = await outbound_http.get(url as HttpURL, {
'accept': 'application/ld+json',
'user-agent': conf.user_agent,
});
// Follow a single redirect if given
if (status === 301) {
({ status, headers, body } = await outbound_http.get(headers.location as HttpURL, {
'accept': 'application/ld+json',
'user-agent': conf.user_agent,
}));
}
if (status === 200) {
const data = JSON.parse(body);
return cacheable(get_forecast_ttl, data as WeatherGovForecast);
}
const data = headers['content-type'].startsWith('application/problem+json')
? JSON.parse(body)
: { status };
log.error(data, 'failed to fetch forecast data');
throw new Error('failed to fetch forecast data');
}
async function get_alerts_nocache(latitude: number, longitude: number) : Promise<Cachable<WeatherGovAlerts>> {
log.info({ latitude, longitude }, 'fetching alerts info');
let { status, headers, body } = await outbound_http.get(`https://api.weather.gov/alerts?active=1&point=${latitude},${longitude}`, {
'accept': 'application/ld+json',
'user-agent': conf.user_agent,
});
// Follow a single redirect if given
if (status === 301) {
({ status, headers, body } = await outbound_http.get(headers.location as HttpURL, {
'accept': 'application/ld+json',
'user-agent': conf.user_agent,
}));
}
if (status === 200) {
const data = JSON.parse(body);
return cacheable(get_alerts_ttl, data as WeatherGovAlerts);
}
const data = headers['content-type'].startsWith('application/problem+json')
? JSON.parse(body)
: { status };
log.error(data, 'failed to fetch alerts data');
throw new Error('failed to fetch alerts data');
}
}
function cacheable<T>(ttl: number, value: T) : Cachable<T> {
const now = new Date();
return deep_freeze({
fetched: now,
expires: new Date(now.getTime() + ttl),
value,
});
}
interface Cachable<T> {
readonly fetched: Date;
readonly expires: Date;
readonly value: Readonly<T>;
}

View File

@@ -0,0 +1,103 @@
import { ContextDefinition } from 'jsonld';
import { HttpURL, ISOTimestamp, Timezone, URN } from '../../utilities/types';
export interface WeatherGovMeasure<Unit extends `wmoUnit:${string}` = `wmoUnit:${string}`> {
unitCode: Unit;
value: number;
}
export interface WeatherGovPoints {
'@context': ContextDefinition;
geometry: string;
cwa: string;
forecastOffice: HttpURL;
gridId: string;
gridX: number;
gridY: number;
forecast: HttpURL;
forecastHourly: HttpURL;
forecastGridData: HttpURL;
forecastStations: HttpURL;
relativeLocation: {
city: string;
state: string;
geometry: string;
distance: WeatherGovMeasure<'wmoUnit:m'>;
bearing: WeatherGovMeasure<'wmoUnit:degree_(angle)'>;
};
forecastZone: HttpURL;
county: HttpURL;
fireWeatherZone: HttpURL;
timeZone: Timezone;
radarStation: string;
}
export interface WeatherGovForecast {
'@context': ContextDefinition;
geometry: string;
updated: ISOTimestamp;
units: 'us' | 'si';
forecastGenerator: 'BaselineForecastGenerator' | 'HourlyForecastGenerator';
generatedAt: ISOTimestamp;
updateTime: ISOTimestamp;
validTimes: string;
elevation: WeatherGovMeasure<'wmoUnit:m'>;
periods: WeatherGovForecastPeriod[];
}
export interface WeatherGovForecastPeriod {
number: number;
name: string;
startTime: ISOTimestamp;
endTime: ISOTimestamp;
isDaytime: boolean;
temperature: number;
temperatureUnit: 'F' | 'C';
temperatureTrend: unknown; // todo: what is this?
probabilityOfPrecipitation: WeatherGovMeasure<'wmoUnit:percent'>;
dewPoint: WeatherGovMeasure<'wmoUnit:degC'>;
relativeHumidity: WeatherGovMeasure<'wmoUnit:percent'>;
windSpeed: `${number} mph` | `${number} km/h`;
windDirection: 'N' | 'NNW' | 'NW' | 'WNW' | 'W' | 'NNE' | 'NE' | 'ENE' | 'E'
| 'S' | 'SSW' | 'SW' | 'WSW' | 'W' | 'SSE' | 'SE' | 'ESE' | 'E';
icon: HttpURL;
shortForecast: string;
detailedForecast: string;
}
export interface WeatherGovAlerts {
'@context': ContextDefinition;
'@graph': {
'@id': HttpURL;
'@type': 'wx:Alert',
id: URN<`oid:${string}`>;
areaDesc: string;
geometry: string;
geocode: {
SAME: `${number}`[];
UGC: string[];
};
affectedZones: HttpURL[];
references: unknown[]; // todo: what is this?
sent: ISOTimestamp;
effective: ISOTimestamp;
onset: ISOTimestamp;
expires: ISOTimestamp;
ends: ISOTimestamp;
status: 'Actual' | 'Exercise' | 'System' | 'Test' | 'Draft';
messageType: 'Alert' | 'Update' | 'Cancel';
category: string;
severity: 'Extreme' | 'Severe' | 'Moderate' | 'Minor' | 'Unknown';
certainty: 'Observed' | 'Likely' | 'Possible' | 'Unlikely' | 'Unknown';
urgency: 'Immediate' | 'Expected' | 'Future' | 'Past' | 'Unknown';
event: string;
sender: string;
senderName: string;
headline: string;
description: string;
instruction: string;
response: string;
parameters: Record<string, string[]>;
}[];
}

View File

@@ -0,0 +1,44 @@
import { icons } from '../../utilities/icons';
import { render_template } from '../../utilities/mustache';
import type { NamedLocation } from '../../storage/named-location';
import type { WeatherGovAlerts, WeatherGovForecast, WeatherGovForecastPeriod } from './interface';
export function render_forecast(location: NamedLocation, forecast: WeatherGovForecast, alerts: WeatherGovAlerts) {
let forecast_today: WeatherGovForecastPeriod[];
let forecast_future: WeatherGovForecastPeriod[];
const forecast_days: WeatherGovForecastPeriod[] = [ ];
const forecast_nights: WeatherGovForecastPeriod[] = [ ];
if (forecast.periods[0].isDaytime) {
forecast_today = forecast.periods.slice(0, 2);
forecast_future = forecast.periods.slice(2);
}
else {
forecast_today = forecast.periods.slice(0, 1);
forecast_future = forecast.periods.slice(1);
}
for (const period of forecast_future) {
if (period.isDaytime) {
forecast_days.push(period);
}
else {
forecast_nights.push(period);
}
}
const view = {
location,
forecast_today,
forecast_future,
forecast_days,
forecast_nights,
icons,
alerts: alerts['@graph'],
};
return render_template('weather.gov/forecast.html.mustache', view);
}

View File

@@ -0,0 +1,96 @@
import pino from 'pino';
import { memo } from '../../utilities/memo';
import { deep_freeze } from '../../utilities/deep';
// import { render_forecast } from './render-forecast';
import { render_current_weather } from './render-current-weather';
import { SecretValue, resolve_secret_value } from '../../conf';
import type { OutboundHttpProvider } from '../../http/outbound';
import type { WeatherAPIComCurrentWeather, WeatherAPIComLocation } from './interface';
import type { NamedLocation } from '../../storage/named-location';
export interface WeatherAPIComConfig {
enabled: boolean;
api_key: SecretValue;
}
export function validate_weatherapi_com_conf(conf: unknown) : asserts conf is WeatherAPIComConfig {
// todo: validate config
}
export type WeatherAPIComProvider = ReturnType<typeof create_weatherapi_com_provider>;
export function create_weatherapi_com_provider(conf: WeatherAPIComConfig, logger: pino.Logger, outbound_http: OutboundHttpProvider) {
const log = logger.child({ logger: 'weatherapi.com' });
const api_key = resolve_secret_value(conf.api_key);
const get_current_ttl = 1000;
// const get_current_ttl = 1000 * 60 * 3;
const get_current = memo(get_current_ttl, get_current_nocache, { });
// const get_forecast_ttl = 1000 * 60 * 30;
// const get_forecast = memo(get_forecast_ttl, get_forecast_nocache, { });
// const get_alerts_ttl = 1000 * 30;
// const get_alerts = memo(get_alerts_ttl, get_alerts_nocache, { });
return {
ready: Promise.resolve(),
get enabled() {
return conf?.enabled;
},
get_current: if_enabled(get_current),
get_current_for_location(location: NamedLocation) {
return get_current(`${location.latitude},${location.longitude}`);
},
render_current_weather,
};
function if_enabled<T>(func: T) : T {
if (conf?.enabled) {
return func;
}
return (() => {
throw new Error('weatherapi.com service not enabled');
}) as T;
}
async function get_current_nocache(query: string) : Promise<Cachable<WeatherAPIComCurrentWeather>> {
log.info({ query }, 'fetching current weather');
let { status, headers, body } = await outbound_http.get(`https://api.weatherapi.com/v1/current.json?key=${api_key}&q=${query}`, {
'accept': 'application/json',
});
if (status === 200) {
const data = JSON.parse(body);
return cacheable(get_current_ttl, data as WeatherAPIComCurrentWeather);
}
const data = headers['content-type'].startsWith('application/json')
? JSON.parse(body)
: { status };
log.error(data, 'failed to fetch current weather');
throw new Error('failed to fetch current weather');
}
}
function cacheable<T>(ttl: number, value: T) : Cachable<T> {
const now = new Date();
return deep_freeze({
fetched: now,
expires: new Date(now.getTime() + ttl),
value,
});
}
interface Cachable<T> {
readonly fetched: Date;
readonly expires: Date;
readonly value: Readonly<T>;
}

View File

@@ -0,0 +1,108 @@
import { HttpURLSchemeless, ISODate, LocalDateTime, Timezone } from '../../utilities/types';
export interface WeatherAPIComLocation {
name: string;
region: string;
country: string
lat: number;
lon: number;
tz_id: Timezone;
localtime_epoch: number;
localtime: LocalDateTime;
}
export interface WeatherAPIComCurrentWeather {
location: WeatherAPIComLocation;
current: WeatherAPIComWeather & {
last_updated_epoch: number;
last_updated: LocalDateTime;
};
}
export interface WeatherAPIComForecast extends WeatherAPIComCurrentWeather {
forecast: {
forecastday: {
date: ISODate;
date_epoch: number;
day: {
maxtemp_c: number;
maxtemp_f: number;
mintemp_c: number;
mintemp_f: number;
avgtemp_c: number;
avgtemp_f: number;
maxwind_mph: number;
maxwind_kph: number;
totalprecip_mm: number;
totalprecip_in: number;
totalsnow_cm: number;
avgvis_km: number;
avgvis_miles: number;
avghumidity: number;
daily_will_it_rain: number;
daily_chance_of_rain: number;
daily_will_it_snow: number;
daily_chance_of_snow: number;
condition: WeatherAPIComCondition;
uv: number;
};
astro: {
sunrise: string;
sunset: string;
moonrise: string;
moonset: string;
moon_phase: string;
moon_illumination: `${bigint}`;
is_moon_up: 0 | 1;
is_sun_up: 0 | 1;
};
hour: WeatherAPIComHourForecast[];
}[];
};
}
export interface WeatherAPIComCondition {
text: string;
icon: HttpURLSchemeless;
code: number;
}
export interface WeatherAPIComWeather {
temp_c: number;
temp_f: number;
is_day: number;
condition: WeatherAPIComCondition;
wind_mph: number;
wind_kph: number;
wind_degree: number;
wind_dir: string;
pressure_mb: number;
pressure_in: number;
precip_mm: number;
precip_in: number;
humidity: number;
cloud: number;
feelslike_c: number;
feelslike_f: number;
vis_km: number;
vis_miles: number;
gust_mph: number;
gust_kph: number;
uv: number;
}
export interface WeatherAPIComHourForecast extends WeatherAPIComWeather {
time_epoch: number;
time: LocalDateTime;
windchill_c: number;
windchill_f: number;
heatindex_c: number;
heatindex_f: number;
dewpoint_c: number;
dewpoint_f: number;
will_it_rain: number;
chance_of_rain: number;
will_it_snow: number;
chance_of_snow: number;
}

View File

@@ -0,0 +1,15 @@
import { icons } from '../../utilities/icons';
import { render_template } from '../../utilities/mustache';
import type { NamedLocation } from '../../storage/named-location';
import type { WeatherAPIComCurrentWeather } from './interface';
export function render_current_weather(location: NamedLocation, weather: WeatherAPIComCurrentWeather) {
const view = {
location,
weather,
icons,
};
return render_template('weatherapi.com/current.html.mustache', view);
}

View File

@@ -12,6 +12,9 @@ import { create_session_provider } from './security/session';
import { create_http_metadata_server } from './http-metadata/server';
import { HttpWebDependencies, create_http_web_server } from './http-web/server';
import { create_color_theme_provider } from './utilities/color-themes';
import { create_outbound_http_provider } from './http/outbound';
import { create_services_provider } from './services';
import { create_rss_feed_reader_provider } from './utilities/rss-feeds';
main();
@@ -39,11 +42,15 @@ async function main() {
const argon2 = create_argon_hash_provider(conf.argon2);
const session = create_session_provider(conf.session_cookie, logger, argon2, storage);
const color_themes = create_color_theme_provider(conf.color_themes);
const outbound_http = create_outbound_http_provider(conf.outbound_http, logger);
const rss_feed_reader = create_rss_feed_reader_provider(logger, outbound_http);
const services = create_services_provider(conf.services, logger, outbound_http);
// Wait for any async init steps
await oidc.ready;
await storage.ready;
await color_themes.ready;
await services.ready;
// Perform any cleanup steps before starting up
await storage.cleanup_old_sessions();
@@ -57,6 +64,9 @@ async function main() {
argon2,
session,
color_themes,
outbound_http,
rss_feed_reader,
services,
};
// Create the main web server

View File

@@ -0,0 +1,6 @@
export interface NamedLocation {
name: string;
latitude: number;
longitude: number;
}

View File

@@ -23,6 +23,9 @@ export function create_color_theme_provider(conf: ColorThemeConfig) {
get ready() {
return ready;
},
get themes() {
return structuredClone(conf);
},
get css() {
return css;
},

View File

@@ -1,13 +0,0 @@
export function deep_merge<T>(host: T, donor: T) : T {
for (const [key, value] of Object.entries(donor)) {
if (value != null && host[key] != null && typeof value === 'object' && typeof host[key] === 'object') {
deep_merge(host[key], value);
continue;
}
host[key] = value;
}
return host;
}

41
src/utilities/deep.ts Normal file
View File

@@ -0,0 +1,41 @@
export function deep_merge<T>(host: T, donor: T) : T {
for (const [key, value] of Object.entries(donor)) {
if (value != null && host[key] != null && typeof value === 'object' && typeof host[key] === 'object') {
deep_merge(host[key], value);
continue;
}
host[key] = value;
}
return host;
}
export function deep_freeze<T>(obj: T) : Readonly<T> {
for (const value of Object.values(obj)) {
if (value == null) {
continue;
}
else if (typeof value === 'object' || typeof value === 'function') {
deep_freeze(value);
}
}
return Object.freeze(obj);
}
export function deep_copy<T>(obj: T) : T {
return structuredClone(obj);
// try {
// return structuredClone(obj);
// }
// catch (error) {
// if (error instanceof ReferenceError) {
// return JSON.parse(JSON.stringify(obj));
// }
// }
}

22
src/utilities/icons.ts Normal file
View File

@@ -0,0 +1,22 @@
export const icons: Record<string, string> = Object.create(null);
const whitespace = /[\s\t\n]+/g;
const feather_icons: Record<string, string> = require('../../vendor/feather-icons/icons.json');
for (const [name, contents] of Object.entries(feather_icons)) {
icons[name] = `
<svg xmlns="http://www.w3.org/2000/svg"
class="icon ${name}"
aria-hidden="true"
viewBox="0 0 24 24"
fill="none"
stroke="currentcolor"
stroke-width="2"
stroke-linecap="round"
stroke-linejoin="round"
>${contents}</svg>
`.replace(whitespace, ' ').trim();
}
Object.freeze(icons);

View File

@@ -51,7 +51,8 @@ export function memo<T extends Func>(ttl: number, func: T, opts: MemoParams<T> =
}
function set_expire(key: string, args: Params<T>, stored: number) {
setTimeout(opts.validator ? revalidate(key, args, stored) : invalidate(key), ttl);
const timer = setTimeout(opts.validator ? revalidate(key, args, stored) : invalidate(key), ttl);
timer.unref();
}
function revalidate(key: string, args: Params<T>, stored: number) {

View File

@@ -0,0 +1,50 @@
import pino from 'pino';
import type { HttpURL } from './types';
import type { OutboundHttpProvider } from '../http/outbound';
import { XMLParser, XMLValidator, X2jOptions } from 'fast-xml-parser';
export type RSSFeedReaderProvider = ReturnType<typeof create_rss_feed_reader_provider>;
export function create_rss_feed_reader_provider(logger: pino.Logger, outbound_http: OutboundHttpProvider) {
const log = logger.child({ logger: 'rss-feed-reader' });
return {
async read_feed(feed_url: HttpURL, skip_cache = false) {
const req_headers = {
accept: 'application/rss+xml; q=1.0, text/xml; q=0.9',
};
const { status, headers, body } = await outbound_http.get(feed_url, req_headers, skip_cache);
if (status === 200) {
const content_type = headers['content-type'] || '';
if (content_type.startsWith('text/xml') || content_type.startsWith('application/rss+xml')) {
const opts: Partial<X2jOptions> = { };
const result = XMLValidator.validate(body, opts);
if (typeof result === 'object' && result.err) {
log.error(result.err, 'RSS feed XML validation error');
throw_error('received invalid XML response from server');
}
const xmlParser = new XMLParser(opts);
return xmlParser.parse(body);
}
else {
throw_error('received invalid content type from server');
}
}
else {
throw_error('received a non-200 response code');
}
},
};
function throw_error(message: string) : never {
throw new Error(`failed to fetch RSS feed content; ${message}`);
}
}

View File

@@ -17,8 +17,16 @@ export type ISOZoneOffset = 'Z' | `${'+' | '-' | ''}${number}:${number}`;
export type ISOTimestamp = `${ISODate}T${ISOTime}${ISOZoneOffset}`;
export type TimeShort = `${number}:${number}`;
export type LocalDateTime = `${ISODate} ${TimeShort}`;
export type Timezone = `${string}/${string}`;
export type Locale = `${string}-${string}`;
export type HttpURL = `http${'s' | ''}://${string}`;
export type HttpURLSchemeless = `//${string}`;
export type URN<T extends string = string> = `urn:${T}`;