migrate basic http client and response cache into library repo
Some checks failed
Build and test / build-and-test (18.x) (push) Failing after 10s
Build and test / build-and-test (20.x) (push) Failing after 40s

This commit is contained in:
James Brumond 2023-08-21 13:22:23 -07:00
parent acf02696ce
commit 07138d4111
Signed by: james
GPG Key ID: E8F2FC44BAA3357A
9 changed files with 716 additions and 22 deletions

View File

@ -2,7 +2,7 @@
name: Build and publish name: Build and publish
on: on:
workflow_dispatch: { } - workflow_dispatch
# push: # push:
# branches: # branches:
# - master # - master
@ -23,8 +23,14 @@ jobs:
- name: Login to package registry - name: Login to package registry
run: | run: |
npm config set @<scope name>:registry https://gitea.jbrumond.me/api/packages/<scope name>/npm/ npm config set @js:registry https://gitea.jbrumond.me/api/packages/js/npm/
npm config set -- '//gitea.jbrumond.me/api/packages/<scope name>/npm/:_authToken' "$NPM_PUBLISH_TOKEN" npm config set -- '//gitea.jbrumond.me/api/packages/js/npm/:_authToken' "$NPM_PUBLISH_TOKEN"
- name: Install dependencies
run: npm ci
- name: Compile TypeScript
run: npm run tsc
- name: Publish package - name: Publish package
run: npm publish run: npm publish

View File

@ -0,0 +1,40 @@
name: Build and test
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build-and-test:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [18.x, 20.x]
steps:
- name: Check out the repo
uses: actions/checkout@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
# - name: Login to package registry
# run: |
# npm config set @js:registry https://gitea.jbrumond.me/api/packages/js/npm/
# npm config set -- '//gitea.jbrumond.me/api/packages/js/npm/:_authToken' "$NPM_PUBLISH_TOKEN"
- name: Install dependencies
run: npm ci
- name: Compile TypeScript
run: npm run tsc
# todo: tests
- name: Run tests
run: exit 0

14
package-lock.json generated
View File

@ -1,17 +1,27 @@
{ {
"name": "@templates/typescript-library", "name": "@js/http-client",
"version": "1.0.0", "version": "1.0.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@templates/typescript-library", "name": "@js/http-client",
"version": "1.0.0", "version": "1.0.0",
"license": "ISC", "license": "ISC",
"devDependencies": { "devDependencies": {
"@types/node": "^20.5.1",
"typescript": "^5.1.3" "typescript": "^5.1.3"
},
"engines": {
"node": ">=18"
} }
}, },
"node_modules/@types/node": {
"version": "20.5.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.1.tgz",
"integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==",
"dev": true
},
"node_modules/typescript": { "node_modules/typescript": {
"version": "5.1.6", "version": "5.1.6",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz",

View File

@ -1,7 +1,7 @@
{ {
"name": "@templates/typescript-library", "name": "@js/http-client",
"version": "1.0.0", "version": "1.0.0",
"description": "Template project for creating new TypeScript library packages", "description": "HTTP client utilities",
"main": "build/index.js", "main": "build/index.js",
"types": "build/index.d.ts", "types": "build/index.d.ts",
"scripts": { "scripts": {
@ -9,15 +9,20 @@
"clean": "rm -rf ./build" "clean": "rm -rf ./build"
}, },
"publishConfig": { "publishConfig": {
"registry": "https://gitea.jbrumond.me/api/packages/templates/npm/" "registry": "https://gitea.jbrumond.me/api/packages/js/npm/"
}, },
"repository": { "repository": {
"type": "git", "type": "git",
"url": "https://gitea.jbrumond.me/templates/typescript-library.git" "url": "https://gitea.jbrumond.me/js/http-client.git"
},
"engines": {
"node": ">=18"
}, },
"author": "James Brumond <https://jbrumond.me>", "author": "James Brumond <https://jbrumond.me>",
"license": "ISC", "license": "ISC",
"devDependencies": { "devDependencies": {
"@js/types": "^0.1.0",
"@types/node": "^20.5.1",
"typescript": "^5.1.3" "typescript": "^5.1.3"
} }
} }

View File

@ -1,30 +1,70 @@
Template project for creating new TypeScript library packages Bare-minimum HTTP client library
Features:
- **Zero** dependencies
- Built-in response cache
--- ---
## Get Started ## Install
### Pull down the code
```bash ```bash
git init # Update project npm config to refer to correct registry for the @js scope
git pull https://gitea.jbrumond.me/templates/typescript-library.git master echo '@js:registry=https://gitea.jbrumond.me/api/packages/js/npm/' >> ./.npmrc
npm install --save @js/http-client
# optional - additional supporting typescript definitions
npm install --save-dev @js/types
``` ```
### Update configuration
- In `package.json`, update any fields like `name`, `description`, `repository`, etc.
- In `.gitea/workflows/publish.yaml`, update `<scope name>` placeholders ## Usage
```ts
import { create_http_client } from '@js/http-client';
const http = create_http_client({
// Default configuration:
logger: null, // (optional) expects a pino logger or compatible
https_only: false,
cache: {
// These settings are based on the recommendations in [RFC 9111], with the
// exception of not caching POST responses (because its not very commonly
// useful and is complicated) and with the addition of invalidating caches
// on successful PATCH requests (because they are fairly common use and
// are expected to modify the referenced resource)
//
// [RFC 9111]: https://www.rfc-editor.org/rfc/rfc9111.html
cacheable_methods: [ 'GET', 'HEAD' ],
cache_invalidating_methods: [ 'POST', 'PUT', 'PATCH', 'DELETE' ],
cacheable_status_codes: [
200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501
],
},
});
const { status, headers, content } = await http.get('https://example.com', {
'accept': 'application/json'
});
if (status === 200) {
const json = JSON.parse(content);
console.log('received data', json);
}
```
## Building ## Building Locally (for development)
```bash ```bash
npm ci
npm run tsc npm run tsc
``` ```

View File

@ -1,4 +1,217 @@
export function hello() : string { import { URL } from 'url';
return 'hello'; import { request as https_request } from 'https';
import { request as http_request, OutgoingHttpHeaders, IncomingHttpHeaders } from 'http';
import { ResponseCacheConfig, create_response_cache, validate_response_cache_conf } from './response-cache';
import { noop_logger } from './noop-logger';
import type { HttpURL, Logger } from '@js/types';
export { ResponseCacheConfig, validate_response_cache_conf } from './response-cache';
let next_id = 1;
const timeout = 5000;
export interface HttpResult {
url: URL;
status: number;
headers: IncomingHttpHeaders;
content: string;
}
export interface HttpClientConfig {
logger?: Logger;
https_only?: boolean;
cache?: ResponseCacheConfig;
}
export function validate_http_conf(conf: unknown) : asserts conf is HttpClientConfig {
if (typeof conf !== 'object' || ! conf) {
throw new Error('Configuration must be an object');
}
if ('logger' in conf) {
// todo: validate
}
if ('https_only' in conf) {
if (typeof conf.https_only !== 'boolean') {
throw new Error('"https_only" must be a boolean if provided');
}
}
if ('cache' in conf) {
validate_response_cache_conf(conf.cache);
}
}
export type HttpClient = ReturnType<typeof create_http_client>;
export function create_http_client(conf: HttpClientConfig) {
const { logger = noop_logger } = conf;
const response_cache = create_response_cache(conf.cache ?? { }, logger);
return {
req: http_req,
get(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) {
return http_req('GET', url_str, headers, null, skip_cache);
},
head(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) {
return http_req('HEAD', url_str, headers, null, skip_cache);
},
post(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
return http_req('POST', url_str, headers, body, skip_cache);
},
put(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
return http_req('PUT', url_str, headers, body, skip_cache);
},
patch(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
return http_req('PATCH', url_str, headers, body, skip_cache);
},
delete(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
return http_req('DELETE', url_str, headers, body, skip_cache);
},
};
async function http_req(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
const url = new URL(url_str);
const make_request
= url.protocol === 'https:' ? https_request
: url.protocol === 'http:' ? http_request
: null;
if (! make_request) {
throw new Error('illegal protocol');
}
if (url.protocol !== 'https:' && conf.https_only) {
throw new Error('non-secure http requests not allowed (by configuration)');
}
if (url.username || url.password) {
throw new Error('urls containing user credentials not allowed');
}
const req_headers = headers ? structuredClone(headers) : { };
const result: HttpResult = {
url,
status: null,
content: null,
headers: null,
};
const path = url.pathname + (url.search || '');
const port = url.port ? parseInt(url.port, 10) : (url.protocol === 'https:' ? 443 : 80);
const log = logger.child({
req_id: next_id++,
method: method,
protocol: url.protocol,
hostname: url.hostname,
port: port,
path: path,
});
const cached = skip_cache
? null
: response_cache.check_for_cached_response(method, url_str, req_headers);
if (cached) {
const revalidation = response_cache.determine_revalidation_needed(cached);
if (! revalidation.should_revalidate) {
log.info('serving request from cache (no revalidation required)');
result.status = cached.status;
result.content = cached.body;
result.headers = structuredClone(cached.res_headers);
return Promise.resolve(result);
}
// if (! revalidation.must_revalidate) {
// // todo: lazy revalidation option
// }
log.info('cached response found, but requires revalidation');
Object.assign(req_headers, revalidation.headers);
}
log.info('starting outbound http request');
return new Promise<HttpResult>((resolve, reject) => {
const start = Date.now();
const req = make_request({
method: method,
protocol: url.protocol,
hostname: url.hostname,
port: port,
path: path,
headers: req_headers
}, (res) => {
// result.res = res;
result.status = res.statusCode;
log.info({
status: res.statusCode,
duration: Date.now() - start,
}, `headers received`);
switch (result.status) {
case 304:
log.info({ status: cached.status }, 're-using cached response (not modified)');
result.status = cached.status;
result.content = cached.body;
result.headers = structuredClone(cached.res_headers);
response_cache.freshen_cache(cached, res.headers);
resolve(result);
break;
case 500:
case 502:
case 503:
case 504:
if (cached?.cache_info?.cache_control?.stale_if_error) {
log.info({ status: cached.status }, 're-using cached response (response returned an error status, and `stale-if-error` directive was provided)');
result.status = cached.status;
result.content = cached.body;
result.headers = structuredClone(cached.res_headers);
resolve(result);
break;
}
// nobreak
default: {
const start = Date.now();
let data = '';
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
log.info({ duration: Date.now() - start }, `response finished`);
result.content = data;
result.headers = res.headers;
response_cache.add_response_to_cache(method, url_str, req_headers, result);
resolve(result);
});
};
}
});
req.setTimeout(timeout, () => {
log.error('request timeout');
req.destroy();
});
req.on('error', (error) => {
log.error('request error ' + error.stack);
reject(error);
});
if (body) {
req.write(body);
}
req.end();
});
}
} }

13
src/noop-logger.ts Normal file
View File

@ -0,0 +1,13 @@
import type { Logger } from '@js/types';
export const noop_logger: Logger = {
child() { return noop_logger; },
fatal() { },
error() { },
warn() { },
info() { },
debug() { },
trace() { },
silent() { },
};

110
src/parse-cache-headers.ts Normal file
View File

@ -0,0 +1,110 @@
import type { IncomingHttpHeaders } from 'http';
export interface ParsedCacheHeaders {
age?: number;
etag?: string;
vary?: string[];
date?: Date;
cache_control?: CacheControl;
expires?: Date;
last_modified?: Date;
}
export interface CacheControl {
public?: boolean;
private?: boolean;
no_store?: boolean;
no_cache?: boolean;
max_age?: number;
s_max_age?: number;
must_revalidate?: boolean;
proxy_revalidate?: boolean;
immutable?: boolean;
stale_while_revalidate?: boolean;
stale_if_error?: boolean;
must_understand?: boolean;
}
export function parse_cache_headers(headers: IncomingHttpHeaders) {
const result: ParsedCacheHeaders = { };
if (headers['age']) {
result.age = parseInt(headers['age'], 10);
}
if (headers['etag']) {
result.etag = headers['etag'];
}
if (headers['vary']) {
result.vary = headers['vary'].split(',').map((str) => str.trim().toLowerCase());
}
if (headers['date']) {
result.date = new Date(Date.parse(headers['date']));
}
if (headers['cache-control']) {
result.cache_control = { };
for (let directive of headers['cache-control'].split(',')) {
directive = directive.trim();
switch (directive) {
case 'public':
result.cache_control.public = true;
break;
case 'private':
result.cache_control.private = true;
break;
case 'no-store':
result.cache_control.no_store = true;
break;
case 'no-cache':
result.cache_control.no_cache = true;
break;
case 'must-revalidate':
result.cache_control.must_revalidate = true;
break;
case 'proxy-revalidate':
result.cache_control.proxy_revalidate = true;
break;
case 'immutable':
result.cache_control.immutable = true;
break;
case 'stale-while-revalidate':
result.cache_control.stale_while_revalidate = true;
break;
case 'stale-if-error':
result.cache_control.stale_if_error = true;
break;
case 'must-understand':
result.cache_control.must_understand = true;
break;
default:
if (directive.startsWith('max-age=')) {
result.cache_control.max_age = parseInt(directive.slice(8), 10);
break;
}
if (directive.startsWith('s-maxage=')) {
result.cache_control.s_max_age = parseInt(directive.slice(10), 10);
break;
}
// todo: log something here about unknown directive
}
}
}
if (headers['expires']) {
result.expires = new Date(Date.parse(headers['expires']));
}
if (headers['last-modified']) {
result.last_modified = new Date(Date.parse(headers['last-modified']));
}
return result;
}

257
src/response-cache.ts Normal file
View File

@ -0,0 +1,257 @@
import { parse_cache_headers, ParsedCacheHeaders } from './parse-cache-headers';
import type { HttpResult } from './index';
import type { HttpURL, Logger } from '@js/types';
import type { OutgoingHttpHeaders, IncomingHttpHeaders } from 'http';
export interface ResponseCacheConfig {
cacheable_methods?: string[];
cacheable_status_codes?: number[];
cache_invalidating_methods?: string[];
// todo: support for configurable limits:
max_records?: number; // max number of cached responses
max_length?: number; // max content-length for one response
evict_after?: number; // period of time unused after which a response should be evicted
}
export interface CachedResponse {
cache_key: string;
cached_time: Date;
is_stale: boolean;
url: HttpURL;
method: string;
status: number;
body: string;
res_headers: IncomingHttpHeaders;
cache_info: ParsedCacheHeaders;
expire_timer: NodeJS.Timeout;
}
export function validate_response_cache_conf(conf: unknown) : asserts conf is ResponseCacheConfig {
// todo: validate config
}
export type ResponseCache = ReturnType<typeof create_response_cache>;
export function create_response_cache(conf: ResponseCacheConfig, logger: Logger) {
type URLCache = Record<string, CachedResponse>;
const cached_responses: Record<HttpURL, URLCache> = Object.create(null);
const cacheable_methods = new Set(conf.cacheable_methods ?? [ 'GET', 'HEAD' ]);
const cache_invalidating_methods = new Set(conf.cache_invalidating_methods ?? [ 'POST', 'PUT', 'PATCH', 'DELETE' ]);
const cacheable_status_codes = new Set(conf.cacheable_status_codes ?? [
200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501
]);
return {
/**
* Should be called before making a new request to check if a cached response already exists
*/
check_for_cached_response(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders) : Readonly<CachedResponse> {
if (! cached_responses[url_str]) {
return;
}
for (const [cache_key, cached] of Object.entries(cached_responses[url_str])) {
if (cache_key === header_cache_key(method, cached.cache_info.vary, headers)) {
return cached;
}
}
},
/**
* Should be called when a cached response is found, to determine what if any revalidation is
* required before re-using the response
*/
determine_revalidation_needed(cached: CachedResponse) {
const { cache_control, etag, date, last_modified } = cached.cache_info;
const must_revalidate = Boolean(cache_control?.must_revalidate || cache_control?.no_cache);
const should_revalidate = must_revalidate || cached.is_stale;
const headers: OutgoingHttpHeaders = { };
if (etag) {
headers['if-none-match'] = etag;
}
if (last_modified || date) {
headers['if-modified-since'] = last_modified?.toUTCString() ?? date?.toUTCString();
}
else {
headers['if-modified-since'] = cached.cached_time.toUTCString();
}
return {
must_revalidate,
should_revalidate,
headers,
};
},
/**
* Should be called when receiving a non-`304 Not Modified` response from a request, either
* as a novel request, or during a revalidation request
*/
add_response_to_cache(method: string, url_str: HttpURL, headers: OutgoingHttpHeaders, result: HttpResult) {
if (! cacheable_methods.has(method) || ! cacheable_status_codes.has(result.status)) {
logger.info({ method, status: result.status }, 'method/status not cacheable');
return;
}
const cache_info = parse_cache_headers(result.headers);
if (headers.authorization) {
// todo: caching authenticated requests
// see <https://www.rfc-editor.org/rfc/rfc9111#section-3.5>
return;
}
// todo: if `vary` has changed, invalidate all stored responses
// todo: source link for ^
if (cache_info.vary.includes('*')) {
logger.info(cache_info, 'response contains "Vary: *" header that prevents caching');
return;
}
if (cache_info.cache_control?.no_store || cache_info.cache_control?.private) {
logger.info(cache_info, 'response contains "Cache-Control" directives that prevent caching');
return;
}
const cache_key = header_cache_key(method, cache_info.vary, headers);
unset_expire_timer(url_str, cache_key);
const cached: CachedResponse = {
cache_key,
cached_time: new Date(Date.now()),
is_stale: false,
url: url_str,
method,
// req_headers: structuredClone(headers),
status: result.status,
body: result.content,
res_headers: structuredClone(result.headers),
cache_info,
expire_timer: null,
};
logger.info(cache_info, 'caching response');
store_to_cache(cached, cache_info);
},
/**
* Should be called when receiving a `304 Not Modified` response during a revalidation request
* to update the cache with more recent freshness info
*/
freshen_cache(cached: CachedResponse, headers: IncomingHttpHeaders) {
const cache_info = parse_cache_headers(headers);
unset_expire_timer(cached.url, cached.cache_key);
store_to_cache(cached, cache_info);
},
/**
*
*/
invalidate_cache_for_update(method: string, url_str: HttpURL, status: number, headers: IncomingHttpHeaders) {
if (! cache_invalidating_methods.has(method)) {
return;
}
if (status < 200 || status >= 400) {
return;
}
},
};
function store_to_cache(cached: CachedResponse, cache_info: ParsedCacheHeaders) {
if (! cached_responses[cached.url]) {
cached_responses[cached.url] = Object.create(null);
}
cached_responses[cached.url][cached.cache_key] = cached;
cached.is_stale = false;
if (cache_info.cache_control?.max_age) {
let ttl_ms = cache_info.cache_control.max_age * 1000;
if (cache_info.age) {
ttl_ms -= cache_info.age * 1000;
}
if (ttl_ms <= 0) {
cached.is_stale = true;
return;
}
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
}
else if (cache_info.expires) {
const ttl_ms = cache_info.expires.getTime() - Date.now();
if (ttl_ms <= 0) {
cached.is_stale = true;
return;
}
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
}
// If no cache-control or expires header is given, default to the
// recommended 10% of time since last-modified
else if (cache_info.last_modified) {
const last_modified_ago_ms = Date.now() - cache_info.last_modified.getTime();
if (last_modified_ago_ms <= 0) {
cached.is_stale = true;
return;
}
const ttl_ms = (last_modified_ago_ms / 10) | 0;
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
}
}
function unset_expire_timer(url_str: HttpURL, cache_key: string) {
const cached = cached_responses[url_str]?.[cache_key];
if (cached?.expire_timer) {
clearTimeout(cached.expire_timer);
}
}
function set_expire_timer(url_str: HttpURL, cache_key: string, ttl_ms: number) {
const timer = setTimeout(expire, ttl_ms);
timer.unref();
return timer;
function expire() {
if (cached_responses[url_str]?.[cache_key]) {
cached_responses[url_str][cache_key].is_stale = true;
}
};
}
}
function header_cache_key(method: string, vary: string[], headers: OutgoingHttpHeaders) {
const parts: string[] = [ method ];
if (vary?.length) {
for (const header of vary) {
if (Array.isArray(headers[header])) {
const values = (headers[header] as string[]).sort();
for (const value of values) {
parts.push(`${header}: ${value}`);
}
}
else if (headers[header]) {
parts.push(`${header}: ${headers[header]}`);
}
}
}
return parts.join('\0');
}