generated from templates/typescript-library
migrate basic http client and response cache into library repo
This commit is contained in:
parent
acf02696ce
commit
07138d4111
@ -2,7 +2,7 @@
|
|||||||
name: Build and publish
|
name: Build and publish
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch: { }
|
- workflow_dispatch
|
||||||
# push:
|
# push:
|
||||||
# branches:
|
# branches:
|
||||||
# - master
|
# - master
|
||||||
@ -23,8 +23,14 @@ jobs:
|
|||||||
|
|
||||||
- name: Login to package registry
|
- name: Login to package registry
|
||||||
run: |
|
run: |
|
||||||
npm config set @<scope name>:registry https://gitea.jbrumond.me/api/packages/<scope name>/npm/
|
npm config set @js:registry https://gitea.jbrumond.me/api/packages/js/npm/
|
||||||
npm config set -- '//gitea.jbrumond.me/api/packages/<scope name>/npm/:_authToken' "$NPM_PUBLISH_TOKEN"
|
npm config set -- '//gitea.jbrumond.me/api/packages/js/npm/:_authToken' "$NPM_PUBLISH_TOKEN"
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Compile TypeScript
|
||||||
|
run: npm run tsc
|
||||||
|
|
||||||
- name: Publish package
|
- name: Publish package
|
||||||
run: npm publish
|
run: npm publish
|
40
.gitea/workflows/build-and-test.yaml
Normal file
40
.gitea/workflows/build-and-test.yaml
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
|
||||||
|
name: Build and test
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-and-test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
node-version: [18.x, 20.x]
|
||||||
|
steps:
|
||||||
|
- name: Check out the repo
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
- name: Use Node.js ${{ matrix.node-version }}
|
||||||
|
uses: actions/setup-node@v3
|
||||||
|
with:
|
||||||
|
node-version: ${{ matrix.node-version }}
|
||||||
|
|
||||||
|
# - name: Login to package registry
|
||||||
|
# run: |
|
||||||
|
# npm config set @js:registry https://gitea.jbrumond.me/api/packages/js/npm/
|
||||||
|
# npm config set -- '//gitea.jbrumond.me/api/packages/js/npm/:_authToken' "$NPM_PUBLISH_TOKEN"
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Compile TypeScript
|
||||||
|
run: npm run tsc
|
||||||
|
|
||||||
|
# todo: tests
|
||||||
|
- name: Run tests
|
||||||
|
run: exit 0
|
14
package-lock.json
generated
14
package-lock.json
generated
@ -1,17 +1,27 @@
|
|||||||
{
|
{
|
||||||
"name": "@templates/typescript-library",
|
"name": "@js/http-client",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@templates/typescript-library",
|
"name": "@js/http-client",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.5.1",
|
||||||
"typescript": "^5.1.3"
|
"typescript": "^5.1.3"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/node": {
|
||||||
|
"version": "20.5.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.1.tgz",
|
||||||
|
"integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==",
|
||||||
|
"dev": true
|
||||||
|
},
|
||||||
"node_modules/typescript": {
|
"node_modules/typescript": {
|
||||||
"version": "5.1.6",
|
"version": "5.1.6",
|
||||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz",
|
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz",
|
||||||
|
13
package.json
13
package.json
@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "@templates/typescript-library",
|
"name": "@js/http-client",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"description": "Template project for creating new TypeScript library packages",
|
"description": "HTTP client utilities",
|
||||||
"main": "build/index.js",
|
"main": "build/index.js",
|
||||||
"types": "build/index.d.ts",
|
"types": "build/index.d.ts",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
@ -9,15 +9,20 @@
|
|||||||
"clean": "rm -rf ./build"
|
"clean": "rm -rf ./build"
|
||||||
},
|
},
|
||||||
"publishConfig": {
|
"publishConfig": {
|
||||||
"registry": "https://gitea.jbrumond.me/api/packages/templates/npm/"
|
"registry": "https://gitea.jbrumond.me/api/packages/js/npm/"
|
||||||
},
|
},
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://gitea.jbrumond.me/templates/typescript-library.git"
|
"url": "https://gitea.jbrumond.me/js/http-client.git"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=18"
|
||||||
},
|
},
|
||||||
"author": "James Brumond <https://jbrumond.me>",
|
"author": "James Brumond <https://jbrumond.me>",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@js/types": "^0.1.0",
|
||||||
|
"@types/node": "^20.5.1",
|
||||||
"typescript": "^5.1.3"
|
"typescript": "^5.1.3"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
62
readme.md
62
readme.md
@ -1,30 +1,70 @@
|
|||||||
|
|
||||||
Template project for creating new TypeScript library packages
|
Bare-minimum HTTP client library
|
||||||
|
|
||||||
|
Features:
|
||||||
|
|
||||||
|
- **Zero** dependencies
|
||||||
|
- Built-in response cache
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Get Started
|
## Install
|
||||||
|
|
||||||
### Pull down the code
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git init
|
# Update project npm config to refer to correct registry for the @js scope
|
||||||
git pull https://gitea.jbrumond.me/templates/typescript-library.git master
|
echo '@js:registry=https://gitea.jbrumond.me/api/packages/js/npm/' >> ./.npmrc
|
||||||
|
|
||||||
|
npm install --save @js/http-client
|
||||||
|
|
||||||
|
# optional - additional supporting typescript definitions
|
||||||
|
npm install --save-dev @js/types
|
||||||
```
|
```
|
||||||
|
|
||||||
### Update configuration
|
|
||||||
|
|
||||||
- In `package.json`, update any fields like `name`, `description`, `repository`, etc.
|
|
||||||
- In `.gitea/workflows/publish.yaml`, update `<scope name>` placeholders
|
## Usage
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { create_http_client } from '@js/http-client';
|
||||||
|
|
||||||
|
const http = create_http_client({
|
||||||
|
// Default configuration:
|
||||||
|
logger: null, // (optional) expects a pino logger or compatible
|
||||||
|
https_only: false,
|
||||||
|
cache: {
|
||||||
|
// These settings are based on the recommendations in [RFC 9111], with the
|
||||||
|
// exception of not caching POST responses (because its not very commonly
|
||||||
|
// useful and is complicated) and with the addition of invalidating caches
|
||||||
|
// on successful PATCH requests (because they are fairly common use and
|
||||||
|
// are expected to modify the referenced resource)
|
||||||
|
//
|
||||||
|
// [RFC 9111]: https://www.rfc-editor.org/rfc/rfc9111.html
|
||||||
|
cacheable_methods: [ 'GET', 'HEAD' ],
|
||||||
|
cache_invalidating_methods: [ 'POST', 'PUT', 'PATCH', 'DELETE' ],
|
||||||
|
cacheable_status_codes: [
|
||||||
|
200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501
|
||||||
|
],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const { status, headers, content } = await http.get('https://example.com', {
|
||||||
|
'accept': 'application/json'
|
||||||
|
});
|
||||||
|
|
||||||
|
if (status === 200) {
|
||||||
|
const json = JSON.parse(content);
|
||||||
|
console.log('received data', json);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## Building
|
## Building Locally (for development)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
npm ci
|
||||||
npm run tsc
|
npm run tsc
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
217
src/index.ts
217
src/index.ts
@ -1,4 +1,217 @@
|
|||||||
|
|
||||||
export function hello() : string {
|
import { URL } from 'url';
|
||||||
return 'hello';
|
import { request as https_request } from 'https';
|
||||||
|
import { request as http_request, OutgoingHttpHeaders, IncomingHttpHeaders } from 'http';
|
||||||
|
import { ResponseCacheConfig, create_response_cache, validate_response_cache_conf } from './response-cache';
|
||||||
|
import { noop_logger } from './noop-logger';
|
||||||
|
import type { HttpURL, Logger } from '@js/types';
|
||||||
|
|
||||||
|
export { ResponseCacheConfig, validate_response_cache_conf } from './response-cache';
|
||||||
|
|
||||||
|
let next_id = 1;
|
||||||
|
const timeout = 5000;
|
||||||
|
|
||||||
|
export interface HttpResult {
|
||||||
|
url: URL;
|
||||||
|
status: number;
|
||||||
|
headers: IncomingHttpHeaders;
|
||||||
|
content: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HttpClientConfig {
|
||||||
|
logger?: Logger;
|
||||||
|
https_only?: boolean;
|
||||||
|
cache?: ResponseCacheConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validate_http_conf(conf: unknown) : asserts conf is HttpClientConfig {
|
||||||
|
if (typeof conf !== 'object' || ! conf) {
|
||||||
|
throw new Error('Configuration must be an object');
|
||||||
|
}
|
||||||
|
|
||||||
|
if ('logger' in conf) {
|
||||||
|
// todo: validate
|
||||||
|
}
|
||||||
|
|
||||||
|
if ('https_only' in conf) {
|
||||||
|
if (typeof conf.https_only !== 'boolean') {
|
||||||
|
throw new Error('"https_only" must be a boolean if provided');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ('cache' in conf) {
|
||||||
|
validate_response_cache_conf(conf.cache);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type HttpClient = ReturnType<typeof create_http_client>;
|
||||||
|
|
||||||
|
export function create_http_client(conf: HttpClientConfig) {
|
||||||
|
const { logger = noop_logger } = conf;
|
||||||
|
const response_cache = create_response_cache(conf.cache ?? { }, logger);
|
||||||
|
|
||||||
|
return {
|
||||||
|
req: http_req,
|
||||||
|
get(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) {
|
||||||
|
return http_req('GET', url_str, headers, null, skip_cache);
|
||||||
|
},
|
||||||
|
head(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) {
|
||||||
|
return http_req('HEAD', url_str, headers, null, skip_cache);
|
||||||
|
},
|
||||||
|
post(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||||
|
return http_req('POST', url_str, headers, body, skip_cache);
|
||||||
|
},
|
||||||
|
put(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||||
|
return http_req('PUT', url_str, headers, body, skip_cache);
|
||||||
|
},
|
||||||
|
patch(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||||
|
return http_req('PATCH', url_str, headers, body, skip_cache);
|
||||||
|
},
|
||||||
|
delete(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||||
|
return http_req('DELETE', url_str, headers, body, skip_cache);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
async function http_req(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) {
|
||||||
|
const url = new URL(url_str);
|
||||||
|
const make_request
|
||||||
|
= url.protocol === 'https:' ? https_request
|
||||||
|
: url.protocol === 'http:' ? http_request
|
||||||
|
: null;
|
||||||
|
|
||||||
|
if (! make_request) {
|
||||||
|
throw new Error('illegal protocol');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (url.protocol !== 'https:' && conf.https_only) {
|
||||||
|
throw new Error('non-secure http requests not allowed (by configuration)');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (url.username || url.password) {
|
||||||
|
throw new Error('urls containing user credentials not allowed');
|
||||||
|
}
|
||||||
|
|
||||||
|
const req_headers = headers ? structuredClone(headers) : { };
|
||||||
|
|
||||||
|
const result: HttpResult = {
|
||||||
|
url,
|
||||||
|
status: null,
|
||||||
|
content: null,
|
||||||
|
headers: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
const path = url.pathname + (url.search || '');
|
||||||
|
const port = url.port ? parseInt(url.port, 10) : (url.protocol === 'https:' ? 443 : 80);
|
||||||
|
const log = logger.child({
|
||||||
|
req_id: next_id++,
|
||||||
|
method: method,
|
||||||
|
protocol: url.protocol,
|
||||||
|
hostname: url.hostname,
|
||||||
|
port: port,
|
||||||
|
path: path,
|
||||||
|
});
|
||||||
|
|
||||||
|
const cached = skip_cache
|
||||||
|
? null
|
||||||
|
: response_cache.check_for_cached_response(method, url_str, req_headers);
|
||||||
|
|
||||||
|
if (cached) {
|
||||||
|
const revalidation = response_cache.determine_revalidation_needed(cached);
|
||||||
|
|
||||||
|
if (! revalidation.should_revalidate) {
|
||||||
|
log.info('serving request from cache (no revalidation required)');
|
||||||
|
result.status = cached.status;
|
||||||
|
result.content = cached.body;
|
||||||
|
result.headers = structuredClone(cached.res_headers);
|
||||||
|
return Promise.resolve(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
// if (! revalidation.must_revalidate) {
|
||||||
|
// // todo: lazy revalidation option
|
||||||
|
// }
|
||||||
|
|
||||||
|
log.info('cached response found, but requires revalidation');
|
||||||
|
Object.assign(req_headers, revalidation.headers);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info('starting outbound http request');
|
||||||
|
|
||||||
|
return new Promise<HttpResult>((resolve, reject) => {
|
||||||
|
const start = Date.now();
|
||||||
|
const req = make_request({
|
||||||
|
method: method,
|
||||||
|
protocol: url.protocol,
|
||||||
|
hostname: url.hostname,
|
||||||
|
port: port,
|
||||||
|
path: path,
|
||||||
|
headers: req_headers
|
||||||
|
}, (res) => {
|
||||||
|
// result.res = res;
|
||||||
|
result.status = res.statusCode;
|
||||||
|
|
||||||
|
log.info({
|
||||||
|
status: res.statusCode,
|
||||||
|
duration: Date.now() - start,
|
||||||
|
}, `headers received`);
|
||||||
|
|
||||||
|
switch (result.status) {
|
||||||
|
case 304:
|
||||||
|
log.info({ status: cached.status }, 're-using cached response (not modified)');
|
||||||
|
result.status = cached.status;
|
||||||
|
result.content = cached.body;
|
||||||
|
result.headers = structuredClone(cached.res_headers);
|
||||||
|
response_cache.freshen_cache(cached, res.headers);
|
||||||
|
resolve(result);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 500:
|
||||||
|
case 502:
|
||||||
|
case 503:
|
||||||
|
case 504:
|
||||||
|
if (cached?.cache_info?.cache_control?.stale_if_error) {
|
||||||
|
log.info({ status: cached.status }, 're-using cached response (response returned an error status, and `stale-if-error` directive was provided)');
|
||||||
|
result.status = cached.status;
|
||||||
|
result.content = cached.body;
|
||||||
|
result.headers = structuredClone(cached.res_headers);
|
||||||
|
resolve(result);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
// nobreak
|
||||||
|
|
||||||
|
default: {
|
||||||
|
const start = Date.now();
|
||||||
|
let data = '';
|
||||||
|
|
||||||
|
res.on('data', (chunk) => {
|
||||||
|
data += chunk;
|
||||||
|
});
|
||||||
|
|
||||||
|
res.on('end', () => {
|
||||||
|
log.info({ duration: Date.now() - start }, `response finished`);
|
||||||
|
result.content = data;
|
||||||
|
result.headers = res.headers;
|
||||||
|
response_cache.add_response_to_cache(method, url_str, req_headers, result);
|
||||||
|
resolve(result);
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
req.setTimeout(timeout, () => {
|
||||||
|
log.error('request timeout');
|
||||||
|
req.destroy();
|
||||||
|
});
|
||||||
|
|
||||||
|
req.on('error', (error) => {
|
||||||
|
log.error('request error ' + error.stack);
|
||||||
|
reject(error);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (body) {
|
||||||
|
req.write(body);
|
||||||
|
}
|
||||||
|
|
||||||
|
req.end();
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
13
src/noop-logger.ts
Normal file
13
src/noop-logger.ts
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
|
||||||
|
import type { Logger } from '@js/types';
|
||||||
|
|
||||||
|
export const noop_logger: Logger = {
|
||||||
|
child() { return noop_logger; },
|
||||||
|
fatal() { },
|
||||||
|
error() { },
|
||||||
|
warn() { },
|
||||||
|
info() { },
|
||||||
|
debug() { },
|
||||||
|
trace() { },
|
||||||
|
silent() { },
|
||||||
|
};
|
110
src/parse-cache-headers.ts
Normal file
110
src/parse-cache-headers.ts
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
|
||||||
|
import type { IncomingHttpHeaders } from 'http';
|
||||||
|
|
||||||
|
export interface ParsedCacheHeaders {
|
||||||
|
age?: number;
|
||||||
|
etag?: string;
|
||||||
|
vary?: string[];
|
||||||
|
date?: Date;
|
||||||
|
cache_control?: CacheControl;
|
||||||
|
expires?: Date;
|
||||||
|
last_modified?: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CacheControl {
|
||||||
|
public?: boolean;
|
||||||
|
private?: boolean;
|
||||||
|
no_store?: boolean;
|
||||||
|
no_cache?: boolean;
|
||||||
|
max_age?: number;
|
||||||
|
s_max_age?: number;
|
||||||
|
must_revalidate?: boolean;
|
||||||
|
proxy_revalidate?: boolean;
|
||||||
|
immutable?: boolean;
|
||||||
|
stale_while_revalidate?: boolean;
|
||||||
|
stale_if_error?: boolean;
|
||||||
|
must_understand?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parse_cache_headers(headers: IncomingHttpHeaders) {
|
||||||
|
const result: ParsedCacheHeaders = { };
|
||||||
|
|
||||||
|
if (headers['age']) {
|
||||||
|
result.age = parseInt(headers['age'], 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (headers['etag']) {
|
||||||
|
result.etag = headers['etag'];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (headers['vary']) {
|
||||||
|
result.vary = headers['vary'].split(',').map((str) => str.trim().toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (headers['date']) {
|
||||||
|
result.date = new Date(Date.parse(headers['date']));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (headers['cache-control']) {
|
||||||
|
result.cache_control = { };
|
||||||
|
|
||||||
|
for (let directive of headers['cache-control'].split(',')) {
|
||||||
|
directive = directive.trim();
|
||||||
|
|
||||||
|
switch (directive) {
|
||||||
|
case 'public':
|
||||||
|
result.cache_control.public = true;
|
||||||
|
break;
|
||||||
|
case 'private':
|
||||||
|
result.cache_control.private = true;
|
||||||
|
break;
|
||||||
|
case 'no-store':
|
||||||
|
result.cache_control.no_store = true;
|
||||||
|
break;
|
||||||
|
case 'no-cache':
|
||||||
|
result.cache_control.no_cache = true;
|
||||||
|
break;
|
||||||
|
case 'must-revalidate':
|
||||||
|
result.cache_control.must_revalidate = true;
|
||||||
|
break;
|
||||||
|
case 'proxy-revalidate':
|
||||||
|
result.cache_control.proxy_revalidate = true;
|
||||||
|
break;
|
||||||
|
case 'immutable':
|
||||||
|
result.cache_control.immutable = true;
|
||||||
|
break;
|
||||||
|
case 'stale-while-revalidate':
|
||||||
|
result.cache_control.stale_while_revalidate = true;
|
||||||
|
break;
|
||||||
|
case 'stale-if-error':
|
||||||
|
result.cache_control.stale_if_error = true;
|
||||||
|
break;
|
||||||
|
case 'must-understand':
|
||||||
|
result.cache_control.must_understand = true;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (directive.startsWith('max-age=')) {
|
||||||
|
result.cache_control.max_age = parseInt(directive.slice(8), 10);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (directive.startsWith('s-maxage=')) {
|
||||||
|
result.cache_control.s_max_age = parseInt(directive.slice(10), 10);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo: log something here about unknown directive
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (headers['expires']) {
|
||||||
|
result.expires = new Date(Date.parse(headers['expires']));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (headers['last-modified']) {
|
||||||
|
result.last_modified = new Date(Date.parse(headers['last-modified']));
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
257
src/response-cache.ts
Normal file
257
src/response-cache.ts
Normal file
@ -0,0 +1,257 @@
|
|||||||
|
|
||||||
|
import { parse_cache_headers, ParsedCacheHeaders } from './parse-cache-headers';
|
||||||
|
import type { HttpResult } from './index';
|
||||||
|
import type { HttpURL, Logger } from '@js/types';
|
||||||
|
import type { OutgoingHttpHeaders, IncomingHttpHeaders } from 'http';
|
||||||
|
|
||||||
|
export interface ResponseCacheConfig {
|
||||||
|
cacheable_methods?: string[];
|
||||||
|
cacheable_status_codes?: number[];
|
||||||
|
cache_invalidating_methods?: string[];
|
||||||
|
// todo: support for configurable limits:
|
||||||
|
max_records?: number; // max number of cached responses
|
||||||
|
max_length?: number; // max content-length for one response
|
||||||
|
evict_after?: number; // period of time unused after which a response should be evicted
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CachedResponse {
|
||||||
|
cache_key: string;
|
||||||
|
cached_time: Date;
|
||||||
|
is_stale: boolean;
|
||||||
|
url: HttpURL;
|
||||||
|
method: string;
|
||||||
|
status: number;
|
||||||
|
body: string;
|
||||||
|
res_headers: IncomingHttpHeaders;
|
||||||
|
cache_info: ParsedCacheHeaders;
|
||||||
|
expire_timer: NodeJS.Timeout;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validate_response_cache_conf(conf: unknown) : asserts conf is ResponseCacheConfig {
|
||||||
|
// todo: validate config
|
||||||
|
}
|
||||||
|
|
||||||
|
export type ResponseCache = ReturnType<typeof create_response_cache>;
|
||||||
|
|
||||||
|
export function create_response_cache(conf: ResponseCacheConfig, logger: Logger) {
|
||||||
|
type URLCache = Record<string, CachedResponse>;
|
||||||
|
const cached_responses: Record<HttpURL, URLCache> = Object.create(null);
|
||||||
|
|
||||||
|
const cacheable_methods = new Set(conf.cacheable_methods ?? [ 'GET', 'HEAD' ]);
|
||||||
|
const cache_invalidating_methods = new Set(conf.cache_invalidating_methods ?? [ 'POST', 'PUT', 'PATCH', 'DELETE' ]);
|
||||||
|
const cacheable_status_codes = new Set(conf.cacheable_status_codes ?? [
|
||||||
|
200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501
|
||||||
|
]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
/**
|
||||||
|
* Should be called before making a new request to check if a cached response already exists
|
||||||
|
*/
|
||||||
|
check_for_cached_response(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders) : Readonly<CachedResponse> {
|
||||||
|
if (! cached_responses[url_str]) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [cache_key, cached] of Object.entries(cached_responses[url_str])) {
|
||||||
|
if (cache_key === header_cache_key(method, cached.cache_info.vary, headers)) {
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Should be called when a cached response is found, to determine what if any revalidation is
|
||||||
|
* required before re-using the response
|
||||||
|
*/
|
||||||
|
determine_revalidation_needed(cached: CachedResponse) {
|
||||||
|
const { cache_control, etag, date, last_modified } = cached.cache_info;
|
||||||
|
|
||||||
|
const must_revalidate = Boolean(cache_control?.must_revalidate || cache_control?.no_cache);
|
||||||
|
const should_revalidate = must_revalidate || cached.is_stale;
|
||||||
|
const headers: OutgoingHttpHeaders = { };
|
||||||
|
|
||||||
|
if (etag) {
|
||||||
|
headers['if-none-match'] = etag;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (last_modified || date) {
|
||||||
|
headers['if-modified-since'] = last_modified?.toUTCString() ?? date?.toUTCString();
|
||||||
|
}
|
||||||
|
|
||||||
|
else {
|
||||||
|
headers['if-modified-since'] = cached.cached_time.toUTCString();
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
must_revalidate,
|
||||||
|
should_revalidate,
|
||||||
|
headers,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Should be called when receiving a non-`304 Not Modified` response from a request, either
|
||||||
|
* as a novel request, or during a revalidation request
|
||||||
|
*/
|
||||||
|
add_response_to_cache(method: string, url_str: HttpURL, headers: OutgoingHttpHeaders, result: HttpResult) {
|
||||||
|
if (! cacheable_methods.has(method) || ! cacheable_status_codes.has(result.status)) {
|
||||||
|
logger.info({ method, status: result.status }, 'method/status not cacheable');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cache_info = parse_cache_headers(result.headers);
|
||||||
|
|
||||||
|
if (headers.authorization) {
|
||||||
|
// todo: caching authenticated requests
|
||||||
|
// see <https://www.rfc-editor.org/rfc/rfc9111#section-3.5>
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo: if `vary` has changed, invalidate all stored responses
|
||||||
|
// todo: source link for ^
|
||||||
|
|
||||||
|
if (cache_info.vary.includes('*')) {
|
||||||
|
logger.info(cache_info, 'response contains "Vary: *" header that prevents caching');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cache_info.cache_control?.no_store || cache_info.cache_control?.private) {
|
||||||
|
logger.info(cache_info, 'response contains "Cache-Control" directives that prevent caching');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cache_key = header_cache_key(method, cache_info.vary, headers);
|
||||||
|
unset_expire_timer(url_str, cache_key);
|
||||||
|
|
||||||
|
const cached: CachedResponse = {
|
||||||
|
cache_key,
|
||||||
|
cached_time: new Date(Date.now()),
|
||||||
|
is_stale: false,
|
||||||
|
url: url_str,
|
||||||
|
method,
|
||||||
|
// req_headers: structuredClone(headers),
|
||||||
|
status: result.status,
|
||||||
|
body: result.content,
|
||||||
|
res_headers: structuredClone(result.headers),
|
||||||
|
cache_info,
|
||||||
|
expire_timer: null,
|
||||||
|
};
|
||||||
|
|
||||||
|
logger.info(cache_info, 'caching response');
|
||||||
|
store_to_cache(cached, cache_info);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Should be called when receiving a `304 Not Modified` response during a revalidation request
|
||||||
|
* to update the cache with more recent freshness info
|
||||||
|
*/
|
||||||
|
freshen_cache(cached: CachedResponse, headers: IncomingHttpHeaders) {
|
||||||
|
const cache_info = parse_cache_headers(headers);
|
||||||
|
unset_expire_timer(cached.url, cached.cache_key);
|
||||||
|
store_to_cache(cached, cache_info);
|
||||||
|
},
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
invalidate_cache_for_update(method: string, url_str: HttpURL, status: number, headers: IncomingHttpHeaders) {
|
||||||
|
if (! cache_invalidating_methods.has(method)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (status < 200 || status >= 400) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
function store_to_cache(cached: CachedResponse, cache_info: ParsedCacheHeaders) {
|
||||||
|
if (! cached_responses[cached.url]) {
|
||||||
|
cached_responses[cached.url] = Object.create(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
cached_responses[cached.url][cached.cache_key] = cached;
|
||||||
|
cached.is_stale = false;
|
||||||
|
|
||||||
|
if (cache_info.cache_control?.max_age) {
|
||||||
|
let ttl_ms = cache_info.cache_control.max_age * 1000;
|
||||||
|
|
||||||
|
if (cache_info.age) {
|
||||||
|
ttl_ms -= cache_info.age * 1000;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ttl_ms <= 0) {
|
||||||
|
cached.is_stale = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
else if (cache_info.expires) {
|
||||||
|
const ttl_ms = cache_info.expires.getTime() - Date.now();
|
||||||
|
|
||||||
|
if (ttl_ms <= 0) {
|
||||||
|
cached.is_stale = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no cache-control or expires header is given, default to the
|
||||||
|
// recommended 10% of time since last-modified
|
||||||
|
else if (cache_info.last_modified) {
|
||||||
|
const last_modified_ago_ms = Date.now() - cache_info.last_modified.getTime();
|
||||||
|
|
||||||
|
if (last_modified_ago_ms <= 0) {
|
||||||
|
cached.is_stale = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ttl_ms = (last_modified_ago_ms / 10) | 0;
|
||||||
|
cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function unset_expire_timer(url_str: HttpURL, cache_key: string) {
|
||||||
|
const cached = cached_responses[url_str]?.[cache_key];
|
||||||
|
|
||||||
|
if (cached?.expire_timer) {
|
||||||
|
clearTimeout(cached.expire_timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function set_expire_timer(url_str: HttpURL, cache_key: string, ttl_ms: number) {
|
||||||
|
const timer = setTimeout(expire, ttl_ms);
|
||||||
|
timer.unref();
|
||||||
|
return timer;
|
||||||
|
|
||||||
|
function expire() {
|
||||||
|
if (cached_responses[url_str]?.[cache_key]) {
|
||||||
|
cached_responses[url_str][cache_key].is_stale = true;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function header_cache_key(method: string, vary: string[], headers: OutgoingHttpHeaders) {
|
||||||
|
const parts: string[] = [ method ];
|
||||||
|
|
||||||
|
if (vary?.length) {
|
||||||
|
for (const header of vary) {
|
||||||
|
if (Array.isArray(headers[header])) {
|
||||||
|
const values = (headers[header] as string[]).sort();
|
||||||
|
for (const value of values) {
|
||||||
|
parts.push(`${header}: ${value}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
else if (headers[header]) {
|
||||||
|
parts.push(`${header}: ${headers[header]}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return parts.join('\0');
|
||||||
|
}
|
Loading…
x
Reference in New Issue
Block a user