work on weather services; outbound http request caching; styles updates and icons
This commit is contained in:
200
src/http/outbound.ts
Normal file
200
src/http/outbound.ts
Normal file
@@ -0,0 +1,200 @@
|
||||
|
||||
import pino from 'pino';
|
||||
import { URL } from 'url';
|
||||
import { request as http_request, OutgoingHttpHeaders, IncomingHttpHeaders } from 'http';
|
||||
import { request as https_request } from 'https';
|
||||
import { create_request_cache_provider } from './request-cache';
|
||||
import { HttpURL } from '../utilities/types';
|
||||
import { deep_copy } from '../utilities/deep';
|
||||
|
||||
let next_id = 1;
|
||||
const timeout = 5000;
|
||||
|
||||
export interface HttpResult {
|
||||
url: URL;
|
||||
// req: ClientRequest;
|
||||
// res: IncomingMessage;
|
||||
status: number;
|
||||
body: string;
|
||||
headers: IncomingHttpHeaders;
|
||||
}
|
||||
|
||||
export interface OutboundHttpConfig {
|
||||
https_only: boolean;
|
||||
}
|
||||
|
||||
export function validate_outbound_http_conf(conf: unknown) : asserts conf is OutboundHttpConfig {
|
||||
// todo: validate config
|
||||
}
|
||||
|
||||
export type OutboundHttpProvider = ReturnType<typeof create_outbound_http_provider>;
|
||||
|
||||
export function create_outbound_http_provider(conf: OutboundHttpConfig, logger: pino.Logger) {
|
||||
const http_log = logger.child({ logger: 'outbound-http' });
|
||||
const response_cache = create_request_cache_provider({ }, http_log);
|
||||
|
||||
return {
|
||||
req: http_req,
|
||||
get(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) {
|
||||
return http_req('GET', url_str, headers, null, skip_cache);
|
||||
},
|
||||
head(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) {
|
||||
return http_req('HEAD', url_str, headers, null, skip_cache);
|
||||
},
|
||||
post(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||
return http_req('POST', url_str, headers, body, skip_cache);
|
||||
},
|
||||
put(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||
return http_req('PUT', url_str, headers, body, skip_cache);
|
||||
},
|
||||
patch(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||
return http_req('PATCH', url_str, headers, body, skip_cache);
|
||||
},
|
||||
delete(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||
return http_req('DELETE', url_str, headers, body, skip_cache);
|
||||
},
|
||||
};
|
||||
|
||||
async function http_req(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||
const url = new URL(url_str);
|
||||
const make_request
|
||||
= url.protocol === 'https:' ? https_request
|
||||
: url.protocol === 'http:' ? http_request
|
||||
: null;
|
||||
|
||||
if (! make_request) {
|
||||
throw new Error('illegal protocol');
|
||||
}
|
||||
|
||||
if (url.protocol !== 'https:' && conf.https_only) {
|
||||
throw new Error('non-secure http requests not allowed (by configuration)');
|
||||
}
|
||||
|
||||
if (url.username || url.password) {
|
||||
throw new Error('urls containing user credentials not allowed');
|
||||
}
|
||||
|
||||
const req_headers = headers ? deep_copy(headers) : { };
|
||||
|
||||
const result: HttpResult = {
|
||||
url,
|
||||
// req: null,
|
||||
// res: null,
|
||||
status: null,
|
||||
body: null,
|
||||
headers: null,
|
||||
};
|
||||
|
||||
const path = url.pathname + (url.search || '');
|
||||
const port = url.port ? parseInt(url.port, 10) : (url.protocol === 'https:' ? 443 : 80);
|
||||
const log = http_log.child({
|
||||
req_id: next_id++,
|
||||
method: method,
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname,
|
||||
port: port,
|
||||
path: path,
|
||||
});
|
||||
|
||||
const cached = skip_cache
|
||||
? null
|
||||
: response_cache.check_for_cached_response(method, url_str, req_headers);
|
||||
|
||||
if (cached) {
|
||||
const revalidation = response_cache.determine_revalidation_needed(cached);
|
||||
|
||||
if (! revalidation.must_revalidate) {
|
||||
log.info('serving request from cache (no revalidation required)');
|
||||
result.status = cached.status;
|
||||
result.body = cached.body;
|
||||
result.headers = deep_copy(cached.res_headers);
|
||||
return Promise.resolve(result);
|
||||
}
|
||||
|
||||
log.info('cached response found, but requires revalidation');
|
||||
Object.assign(req_headers, revalidation.headers);
|
||||
}
|
||||
|
||||
log.info('starting outbound http request');
|
||||
|
||||
return new Promise<HttpResult>((resolve, reject) => {
|
||||
const start = Date.now();
|
||||
const req = make_request({
|
||||
method: method,
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname,
|
||||
port: port,
|
||||
path: path,
|
||||
headers: req_headers
|
||||
}, (res) => {
|
||||
// result.res = res;
|
||||
result.status = res.statusCode;
|
||||
|
||||
log.info({
|
||||
status: res.statusCode,
|
||||
duration: Date.now() - start,
|
||||
}, `headers received`);
|
||||
|
||||
switch (result.status) {
|
||||
case 304:
|
||||
log.info({ status: cached.status }, 're-using cached response (not modified)');
|
||||
result.status = cached.status;
|
||||
result.body = cached.body;
|
||||
result.headers = deep_copy(cached.res_headers);
|
||||
response_cache.freshen_cache(cached, res.headers);
|
||||
resolve(result);
|
||||
break;
|
||||
|
||||
case 500:
|
||||
case 502:
|
||||
case 503:
|
||||
case 504:
|
||||
if (cached?.cache_info?.cache_control?.stale_if_error) {
|
||||
log.info({ status: cached.status }, 're-using cached response (response returned an error status, and `stale-if-error` directive was provided)');
|
||||
result.status = cached.status;
|
||||
result.body = cached.body;
|
||||
result.headers = deep_copy(cached.res_headers);
|
||||
resolve(result);
|
||||
break;
|
||||
}
|
||||
// nobreak
|
||||
|
||||
default: {
|
||||
const start = Date.now();
|
||||
let data = '';
|
||||
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on('end', () => {
|
||||
log.info({ duration: Date.now() - start }, `response finished`);
|
||||
result.body = data;
|
||||
result.headers = res.headers;
|
||||
response_cache.add_response_to_cache(method, url_str, req_headers, result);
|
||||
resolve(result);
|
||||
});
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
// result.req = req;
|
||||
|
||||
req.setTimeout(timeout, () => {
|
||||
log.error('request timeout');
|
||||
req.destroy();
|
||||
});
|
||||
|
||||
req.on('error', (error) => {
|
||||
log.error('request error ' + error.stack);
|
||||
reject(error);
|
||||
});
|
||||
|
||||
if (body) {
|
||||
req.write(body);
|
||||
}
|
||||
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -17,8 +17,13 @@ export interface CacheControl {
|
||||
no_store?: boolean;
|
||||
no_cache?: boolean;
|
||||
max_age?: number;
|
||||
s_max_age?: number;
|
||||
must_revalidate?: boolean;
|
||||
proxy_revalidate?: boolean;
|
||||
immutable?: boolean;
|
||||
stale_while_revalidate?: boolean;
|
||||
stale_if_error?: boolean;
|
||||
must_understand?: boolean;
|
||||
}
|
||||
|
||||
export function parse_cache_headers(headers: IncomingHttpHeaders) {
|
||||
@@ -62,12 +67,29 @@ export function parse_cache_headers(headers: IncomingHttpHeaders) {
|
||||
case 'proxy-revalidate':
|
||||
result.cache_control.proxy_revalidate = true;
|
||||
break;
|
||||
case 'immutable':
|
||||
result.cache_control.immutable = true;
|
||||
break;
|
||||
case 'stale-while-revalidate':
|
||||
result.cache_control.stale_while_revalidate = true;
|
||||
break;
|
||||
case 'stale-if-error':
|
||||
result.cache_control.stale_if_error = true;
|
||||
break;
|
||||
case 'must-understand':
|
||||
result.cache_control.must_understand = true;
|
||||
break;
|
||||
default:
|
||||
if (directive.startsWith('max-age=')) {
|
||||
result.cache_control.max_age = parseInt(directive.slice(8), 10);
|
||||
break;
|
||||
}
|
||||
|
||||
if (directive.startsWith('s-max-age=')) {
|
||||
result.cache_control.s_max_age = parseInt(directive.slice(10), 10);
|
||||
break;
|
||||
}
|
||||
|
||||
// todo: log something here about unknown directive
|
||||
}
|
||||
}
|
||||
|
||||
228
src/http/request-cache.ts
Normal file
228
src/http/request-cache.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
|
||||
import { pino } from 'pino';
|
||||
import { DateTime } from 'luxon';
|
||||
import { deep_copy } from '../utilities/deep';
|
||||
import { OutgoingHttpHeaders, IncomingHttpHeaders } from 'http';
|
||||
import { parse_cache_headers, ParsedCacheHeaders } from './parse-cache-headers';
|
||||
import type { HttpURL } from '../utilities/types';
|
||||
import type { HttpResult } from './outbound';
|
||||
|
||||
export interface RequestCacheConfig {
|
||||
// todo: support for configurable limits:
|
||||
max_records?: number; // max number of cached responses
|
||||
max_length?: number; // max content-length for one response
|
||||
evict_after?: number; // period of time unused after which a response should be evicted
|
||||
}
|
||||
|
||||
export interface CachedResponse {
|
||||
cache_key: string;
|
||||
cached_time: DateTime;
|
||||
is_stale: boolean;
|
||||
url: HttpURL;
|
||||
method: string;
|
||||
req_headers: OutgoingHttpHeaders;
|
||||
status: number;
|
||||
body: string;
|
||||
res_headers: IncomingHttpHeaders;
|
||||
cache_info: ParsedCacheHeaders;
|
||||
expire_timer: NodeJS.Timeout;
|
||||
}
|
||||
|
||||
export function validate_request_cache_conf(conf: unknown) : asserts conf is RequestCacheConfig {
|
||||
// todo: validate config
|
||||
}
|
||||
|
||||
export type RequestCacheProvider = ReturnType<typeof create_request_cache_provider>;
|
||||
|
||||
export function create_request_cache_provider(conf: RequestCacheConfig, logger: pino.Logger) {
|
||||
type URLCache = Record<string, CachedResponse>;
|
||||
const cached_responses: Record<HttpURL, URLCache> = Object.create(null);
|
||||
|
||||
const cacheable_methods = new Set([ 'GET', 'HEAD' ]);
|
||||
const cacheable_status_codes = new Set([
|
||||
200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501
|
||||
]);
|
||||
|
||||
return {
|
||||
/**
|
||||
* Should be called before making a new request to check if a cached response already exists
|
||||
*/
|
||||
check_for_cached_response(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders) : Readonly<CachedResponse> {
|
||||
if (! cached_responses[url_str]) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [cache_key, cached] of Object.entries(cached_responses[url_str])) {
|
||||
if (cache_key === header_cache_key(method, cached.cache_info.vary, headers)) {
|
||||
return cached;
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Should be called when a cached response is found, to determine what if any revalidation is
|
||||
* required before re-using the response
|
||||
*/
|
||||
determine_revalidation_needed(cached: CachedResponse) {
|
||||
const { cache_control, etag, date, last_modified } = cached.cache_info;
|
||||
|
||||
const must_revalidate = Boolean(cache_control?.must_revalidate || cache_control?.no_cache);
|
||||
const headers: OutgoingHttpHeaders = { };
|
||||
|
||||
if (etag) {
|
||||
headers['if-none-match'] = etag;
|
||||
}
|
||||
|
||||
if (last_modified || date) {
|
||||
headers['if-modified-since'] = last_modified?.toHTTP() ?? date?.toHTTP();
|
||||
}
|
||||
|
||||
else {
|
||||
headers['if-modified-since'] = cached.cached_time.toHTTP();
|
||||
}
|
||||
|
||||
return {
|
||||
must_revalidate,
|
||||
headers,
|
||||
};
|
||||
},
|
||||
|
||||
/**
|
||||
* Should be called when receiving a non-`304 Not Modified` response from a request, either
|
||||
* as a novel request, or during a revalidation request
|
||||
*/
|
||||
add_response_to_cache(method: string, url_str: HttpURL, headers: OutgoingHttpHeaders, result: HttpResult) {
|
||||
if (! cacheable_methods.has(method) || ! cacheable_status_codes.has(result.status)) {
|
||||
logger.info('method/status not cacheable');
|
||||
return;
|
||||
}
|
||||
|
||||
const cache_info = parse_cache_headers(result.headers);
|
||||
|
||||
if (cache_info.cache_control?.no_store || cache_info.cache_control?.private) {
|
||||
logger.info(cache_info, 'response contains cache-control directives that prevent caching');
|
||||
return;
|
||||
}
|
||||
|
||||
const cache_key = header_cache_key(method, cache_info.vary, headers);
|
||||
unset_expire_timer(url_str, cache_key);
|
||||
|
||||
const cached: CachedResponse = {
|
||||
cache_key,
|
||||
cached_time: DateTime.now(),
|
||||
is_stale: false,
|
||||
url: url_str,
|
||||
method,
|
||||
req_headers: deep_copy(headers),
|
||||
status: result.status,
|
||||
body: result.body,
|
||||
res_headers: deep_copy(result.headers),
|
||||
cache_info,
|
||||
expire_timer: null,
|
||||
};
|
||||
|
||||
logger.info(cache_info, 'caching response');
|
||||
store_to_cache(cached, cache_info);
|
||||
},
|
||||
|
||||
/**
|
||||
* Should be called when receiving a `304 Not Modified` response during a revalidation request
|
||||
* to update the cache with more recent freshness info
|
||||
*/
|
||||
freshen_cache(cached: CachedResponse, headers: IncomingHttpHeaders) {
|
||||
const cache_info = parse_cache_headers(headers);
|
||||
unset_expire_timer(cached.url, cached.cache_key);
|
||||
store_to_cache(cached, cache_info);
|
||||
},
|
||||
};
|
||||
|
||||
function store_to_cache(cached: CachedResponse, cache_info: ParsedCacheHeaders) {
|
||||
if (! cached_responses[cached.url]) {
|
||||
cached_responses[cached.url] = Object.create(null);
|
||||
}
|
||||
|
||||
cached_responses[cached.url][cached.cache_key] = cached;
|
||||
cached.is_stale = false;
|
||||
|
||||
if (cache_info.cache_control?.max_age) {
|
||||
let ttl_ms = cache_info.cache_control.max_age * 1000;
|
||||
|
||||
if (cache_info.age) {
|
||||
ttl_ms -= cache_info.age * 1000;
|
||||
}
|
||||
|
||||
if (ttl_ms <= 0) {
|
||||
cached.is_stale = true;
|
||||
return;
|
||||
}
|
||||
|
||||
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
|
||||
}
|
||||
|
||||
else if (cache_info.expires) {
|
||||
const ttl_ms = cache_info.expires.toUnixInteger() - Date.now();
|
||||
|
||||
if (ttl_ms <= 0) {
|
||||
cached.is_stale = true;
|
||||
return;
|
||||
}
|
||||
|
||||
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
|
||||
}
|
||||
|
||||
// If no cache-control or expires header is given, default to the
|
||||
// recommended 10% of time since last-modified
|
||||
else if (cache_info.last_modified) {
|
||||
const last_modified_ago_ms = Date.now() - cache_info.last_modified.toUnixInteger();
|
||||
|
||||
if (last_modified_ago_ms <= 0) {
|
||||
cached.is_stale = true;
|
||||
return;
|
||||
}
|
||||
|
||||
const ttl_ms = (last_modified_ago_ms / 10) | 0;
|
||||
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
|
||||
}
|
||||
}
|
||||
|
||||
function unset_expire_timer(url_str: HttpURL, cache_key: string) {
|
||||
const cached = cached_responses[url_str]?.[cache_key];
|
||||
|
||||
if (cached?.expire_timer) {
|
||||
clearTimeout(cached.expire_timer);
|
||||
}
|
||||
}
|
||||
|
||||
function set_expire_timer(url_str: HttpURL, cache_key: string, ttl_ms: number) {
|
||||
const timer = setTimeout(expire, ttl_ms);
|
||||
timer.unref();
|
||||
return timer;
|
||||
|
||||
function expire() {
|
||||
if (cached_responses[url_str]?.[cache_key]) {
|
||||
cached_responses[url_str][cache_key].is_stale = true;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function header_cache_key(method: string, vary: string[], headers: OutgoingHttpHeaders) {
|
||||
const parts: string[] = [ method ];
|
||||
|
||||
if (vary?.length) {
|
||||
for (const header of vary) {
|
||||
if (Array.isArray(headers[header])) {
|
||||
const values = (headers[header] as string[]).sort();
|
||||
for (const value of values) {
|
||||
parts.push(`${header}: ${value}`);
|
||||
}
|
||||
}
|
||||
|
||||
else if (headers[header]) {
|
||||
parts.push(`${header}: ${headers[header]}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\0');
|
||||
}
|
||||
Reference in New Issue
Block a user