diff --git a/.gitea/workflows/publish.yaml b/.gitea/workflows/build-and-publish.yaml similarity index 59% rename from .gitea/workflows/publish.yaml rename to .gitea/workflows/build-and-publish.yaml index d155937..d80b4c7 100644 --- a/.gitea/workflows/publish.yaml +++ b/.gitea/workflows/build-and-publish.yaml @@ -2,7 +2,7 @@ name: Build and publish on: - workflow_dispatch: { } +- workflow_dispatch # push: # branches: # - master @@ -23,8 +23,14 @@ jobs: - name: Login to package registry run: | - npm config set @:registry https://gitea.jbrumond.me/api/packages//npm/ - npm config set -- '//gitea.jbrumond.me/api/packages//npm/:_authToken' "$NPM_PUBLISH_TOKEN" + npm config set @js:registry https://gitea.jbrumond.me/api/packages/js/npm/ + npm config set -- '//gitea.jbrumond.me/api/packages/js/npm/:_authToken' "$NPM_PUBLISH_TOKEN" + + - name: Install dependencies + run: npm ci + + - name: Compile TypeScript + run: npm run tsc - name: Publish package run: npm publish diff --git a/.gitea/workflows/build-and-test.yaml b/.gitea/workflows/build-and-test.yaml new file mode 100644 index 0000000..ecda6ba --- /dev/null +++ b/.gitea/workflows/build-and-test.yaml @@ -0,0 +1,40 @@ + +name: Build and test + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + build-and-test: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: [18.x, 20.x] + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + # - name: Login to package registry + # run: | + # npm config set @js:registry https://gitea.jbrumond.me/api/packages/js/npm/ + # npm config set -- '//gitea.jbrumond.me/api/packages/js/npm/:_authToken' "$NPM_PUBLISH_TOKEN" + + - name: Install dependencies + run: npm ci + + - name: Compile TypeScript + run: npm run tsc + + # todo: tests + - name: Run tests + run: exit 0 diff --git a/package-lock.json b/package-lock.json index b8d09b5..c59a91b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,27 @@ { - "name": "@templates/typescript-library", + "name": "@js/http-client", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@templates/typescript-library", + "name": "@js/http-client", "version": "1.0.0", "license": "ISC", "devDependencies": { + "@types/node": "^20.5.1", "typescript": "^5.1.3" + }, + "engines": { + "node": ">=18" } }, + "node_modules/@types/node": { + "version": "20.5.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.1.tgz", + "integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==", + "dev": true + }, "node_modules/typescript": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", diff --git a/package.json b/package.json index ded3a6d..857767e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { - "name": "@templates/typescript-library", + "name": "@js/http-client", "version": "1.0.0", - "description": "Template project for creating new TypeScript library packages", + "description": "HTTP client utilities", "main": "build/index.js", "types": "build/index.d.ts", "scripts": { @@ -9,15 +9,20 @@ "clean": "rm -rf ./build" }, "publishConfig": { - "registry": "https://gitea.jbrumond.me/api/packages/templates/npm/" + "registry": "https://gitea.jbrumond.me/api/packages/js/npm/" }, "repository": { "type": "git", - "url": "https://gitea.jbrumond.me/templates/typescript-library.git" + "url": "https://gitea.jbrumond.me/js/http-client.git" + }, + "engines": { + "node": ">=18" }, "author": "James Brumond ", "license": "ISC", "devDependencies": { + "@js/types": "^0.1.0", + "@types/node": "^20.5.1", "typescript": "^5.1.3" } } diff --git a/readme.md b/readme.md index 7389046..13d064e 100644 --- a/readme.md +++ b/readme.md @@ -1,30 +1,70 @@ -Template project for creating new TypeScript library packages +Bare-minimum HTTP client library + +Features: + +- **Zero** dependencies +- Built-in response cache --- -## Get Started - -### Pull down the code +## Install ```bash -git init -git pull https://gitea.jbrumond.me/templates/typescript-library.git master +# Update project npm config to refer to correct registry for the @js scope +echo '@js:registry=https://gitea.jbrumond.me/api/packages/js/npm/' >> ./.npmrc + +npm install --save @js/http-client + +# optional - additional supporting typescript definitions +npm install --save-dev @js/types ``` -### Update configuration -- In `package.json`, update any fields like `name`, `description`, `repository`, etc. -- In `.gitea/workflows/publish.yaml`, update `` placeholders + +## Usage + +```ts +import { create_http_client } from '@js/http-client'; + +const http = create_http_client({ + // Default configuration: + logger: null, // (optional) expects a pino logger or compatible + https_only: false, + cache: { + // These settings are based on the recommendations in [RFC 9111], with the + // exception of not caching POST responses (because its not very commonly + // useful and is complicated) and with the addition of invalidating caches + // on successful PATCH requests (because they are fairly common use and + // are expected to modify the referenced resource) + // + // [RFC 9111]: https://www.rfc-editor.org/rfc/rfc9111.html + cacheable_methods: [ 'GET', 'HEAD' ], + cache_invalidating_methods: [ 'POST', 'PUT', 'PATCH', 'DELETE' ], + cacheable_status_codes: [ + 200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501 + ], + }, +}); + +const { status, headers, content } = await http.get('https://example.com', { + 'accept': 'application/json' +}); + +if (status === 200) { + const json = JSON.parse(content); + console.log('received data', json); +} +``` -## Building +## Building Locally (for development) ```bash +npm ci npm run tsc ``` - diff --git a/src/index.ts b/src/index.ts index 4fa933d..baf8bcd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,217 @@ -export function hello() : string { - return 'hello'; +import { URL } from 'url'; +import { request as https_request } from 'https'; +import { request as http_request, OutgoingHttpHeaders, IncomingHttpHeaders } from 'http'; +import { ResponseCacheConfig, create_response_cache, validate_response_cache_conf } from './response-cache'; +import { noop_logger } from './noop-logger'; +import type { HttpURL, Logger } from '@js/types'; + +export { ResponseCacheConfig, validate_response_cache_conf } from './response-cache'; + +let next_id = 1; +const timeout = 5000; + +export interface HttpResult { + url: URL; + status: number; + headers: IncomingHttpHeaders; + content: string; +} + +export interface HttpClientConfig { + logger?: Logger; + https_only?: boolean; + cache?: ResponseCacheConfig; +} + +export function validate_http_conf(conf: unknown) : asserts conf is HttpClientConfig { + if (typeof conf !== 'object' || ! conf) { + throw new Error('Configuration must be an object'); + } + + if ('logger' in conf) { + // todo: validate + } + + if ('https_only' in conf) { + if (typeof conf.https_only !== 'boolean') { + throw new Error('"https_only" must be a boolean if provided'); + } + } + + if ('cache' in conf) { + validate_response_cache_conf(conf.cache); + } +} + +export type HttpClient = ReturnType; + +export function create_http_client(conf: HttpClientConfig) { + const { logger = noop_logger } = conf; + const response_cache = create_response_cache(conf.cache ?? { }, logger); + + return { + req: http_req, + get(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) { + return http_req('GET', url_str, headers, null, skip_cache); + }, + head(url_str: HttpURL, headers?: OutgoingHttpHeaders, skip_cache = false) { + return http_req('HEAD', url_str, headers, null, skip_cache); + }, + post(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) { + return http_req('POST', url_str, headers, body, skip_cache); + }, + put(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) { + return http_req('PUT', url_str, headers, body, skip_cache); + }, + patch(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) { + return http_req('PATCH', url_str, headers, body, skip_cache); + }, + delete(url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) { + return http_req('DELETE', url_str, headers, body, skip_cache); + }, + }; + + async function http_req(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders, body?: string, skip_cache = false) { + const url = new URL(url_str); + const make_request + = url.protocol === 'https:' ? https_request + : url.protocol === 'http:' ? http_request + : null; + + if (! make_request) { + throw new Error('illegal protocol'); + } + + if (url.protocol !== 'https:' && conf.https_only) { + throw new Error('non-secure http requests not allowed (by configuration)'); + } + + if (url.username || url.password) { + throw new Error('urls containing user credentials not allowed'); + } + + const req_headers = headers ? structuredClone(headers) : { }; + + const result: HttpResult = { + url, + status: null, + content: null, + headers: null, + }; + + const path = url.pathname + (url.search || ''); + const port = url.port ? parseInt(url.port, 10) : (url.protocol === 'https:' ? 443 : 80); + const log = logger.child({ + req_id: next_id++, + method: method, + protocol: url.protocol, + hostname: url.hostname, + port: port, + path: path, + }); + + const cached = skip_cache + ? null + : response_cache.check_for_cached_response(method, url_str, req_headers); + + if (cached) { + const revalidation = response_cache.determine_revalidation_needed(cached); + + if (! revalidation.should_revalidate) { + log.info('serving request from cache (no revalidation required)'); + result.status = cached.status; + result.content = cached.body; + result.headers = structuredClone(cached.res_headers); + return Promise.resolve(result); + } + + // if (! revalidation.must_revalidate) { + // // todo: lazy revalidation option + // } + + log.info('cached response found, but requires revalidation'); + Object.assign(req_headers, revalidation.headers); + } + + log.info('starting outbound http request'); + + return new Promise((resolve, reject) => { + const start = Date.now(); + const req = make_request({ + method: method, + protocol: url.protocol, + hostname: url.hostname, + port: port, + path: path, + headers: req_headers + }, (res) => { + // result.res = res; + result.status = res.statusCode; + + log.info({ + status: res.statusCode, + duration: Date.now() - start, + }, `headers received`); + + switch (result.status) { + case 304: + log.info({ status: cached.status }, 're-using cached response (not modified)'); + result.status = cached.status; + result.content = cached.body; + result.headers = structuredClone(cached.res_headers); + response_cache.freshen_cache(cached, res.headers); + resolve(result); + break; + + case 500: + case 502: + case 503: + case 504: + if (cached?.cache_info?.cache_control?.stale_if_error) { + log.info({ status: cached.status }, 're-using cached response (response returned an error status, and `stale-if-error` directive was provided)'); + result.status = cached.status; + result.content = cached.body; + result.headers = structuredClone(cached.res_headers); + resolve(result); + break; + } + // nobreak + + default: { + const start = Date.now(); + let data = ''; + + res.on('data', (chunk) => { + data += chunk; + }); + + res.on('end', () => { + log.info({ duration: Date.now() - start }, `response finished`); + result.content = data; + result.headers = res.headers; + response_cache.add_response_to_cache(method, url_str, req_headers, result); + resolve(result); + }); + }; + } + }); + + req.setTimeout(timeout, () => { + log.error('request timeout'); + req.destroy(); + }); + + req.on('error', (error) => { + log.error('request error ' + error.stack); + reject(error); + }); + + if (body) { + req.write(body); + } + + req.end(); + }); + } } diff --git a/src/noop-logger.ts b/src/noop-logger.ts new file mode 100644 index 0000000..8f9413b --- /dev/null +++ b/src/noop-logger.ts @@ -0,0 +1,13 @@ + +import type { Logger } from '@js/types'; + +export const noop_logger: Logger = { + child() { return noop_logger; }, + fatal() { }, + error() { }, + warn() { }, + info() { }, + debug() { }, + trace() { }, + silent() { }, +}; diff --git a/src/parse-cache-headers.ts b/src/parse-cache-headers.ts new file mode 100644 index 0000000..7c62d6c --- /dev/null +++ b/src/parse-cache-headers.ts @@ -0,0 +1,110 @@ + +import type { IncomingHttpHeaders } from 'http'; + +export interface ParsedCacheHeaders { + age?: number; + etag?: string; + vary?: string[]; + date?: Date; + cache_control?: CacheControl; + expires?: Date; + last_modified?: Date; +} + +export interface CacheControl { + public?: boolean; + private?: boolean; + no_store?: boolean; + no_cache?: boolean; + max_age?: number; + s_max_age?: number; + must_revalidate?: boolean; + proxy_revalidate?: boolean; + immutable?: boolean; + stale_while_revalidate?: boolean; + stale_if_error?: boolean; + must_understand?: boolean; +} + +export function parse_cache_headers(headers: IncomingHttpHeaders) { + const result: ParsedCacheHeaders = { }; + + if (headers['age']) { + result.age = parseInt(headers['age'], 10); + } + + if (headers['etag']) { + result.etag = headers['etag']; + } + + if (headers['vary']) { + result.vary = headers['vary'].split(',').map((str) => str.trim().toLowerCase()); + } + + if (headers['date']) { + result.date = new Date(Date.parse(headers['date'])); + } + + if (headers['cache-control']) { + result.cache_control = { }; + + for (let directive of headers['cache-control'].split(',')) { + directive = directive.trim(); + + switch (directive) { + case 'public': + result.cache_control.public = true; + break; + case 'private': + result.cache_control.private = true; + break; + case 'no-store': + result.cache_control.no_store = true; + break; + case 'no-cache': + result.cache_control.no_cache = true; + break; + case 'must-revalidate': + result.cache_control.must_revalidate = true; + break; + case 'proxy-revalidate': + result.cache_control.proxy_revalidate = true; + break; + case 'immutable': + result.cache_control.immutable = true; + break; + case 'stale-while-revalidate': + result.cache_control.stale_while_revalidate = true; + break; + case 'stale-if-error': + result.cache_control.stale_if_error = true; + break; + case 'must-understand': + result.cache_control.must_understand = true; + break; + default: + if (directive.startsWith('max-age=')) { + result.cache_control.max_age = parseInt(directive.slice(8), 10); + break; + } + + if (directive.startsWith('s-maxage=')) { + result.cache_control.s_max_age = parseInt(directive.slice(10), 10); + break; + } + + // todo: log something here about unknown directive + } + } + } + + if (headers['expires']) { + result.expires = new Date(Date.parse(headers['expires'])); + } + + if (headers['last-modified']) { + result.last_modified = new Date(Date.parse(headers['last-modified'])); + } + + return result; +} diff --git a/src/response-cache.ts b/src/response-cache.ts new file mode 100644 index 0000000..5f85352 --- /dev/null +++ b/src/response-cache.ts @@ -0,0 +1,257 @@ + +import { parse_cache_headers, ParsedCacheHeaders } from './parse-cache-headers'; +import type { HttpResult } from './index'; +import type { HttpURL, Logger } from '@js/types'; +import type { OutgoingHttpHeaders, IncomingHttpHeaders } from 'http'; + +export interface ResponseCacheConfig { + cacheable_methods?: string[]; + cacheable_status_codes?: number[]; + cache_invalidating_methods?: string[]; + // todo: support for configurable limits: + max_records?: number; // max number of cached responses + max_length?: number; // max content-length for one response + evict_after?: number; // period of time unused after which a response should be evicted +} + +export interface CachedResponse { + cache_key: string; + cached_time: Date; + is_stale: boolean; + url: HttpURL; + method: string; + status: number; + body: string; + res_headers: IncomingHttpHeaders; + cache_info: ParsedCacheHeaders; + expire_timer: NodeJS.Timeout; +} + +export function validate_response_cache_conf(conf: unknown) : asserts conf is ResponseCacheConfig { + // todo: validate config +} + +export type ResponseCache = ReturnType; + +export function create_response_cache(conf: ResponseCacheConfig, logger: Logger) { + type URLCache = Record; + const cached_responses: Record = Object.create(null); + + const cacheable_methods = new Set(conf.cacheable_methods ?? [ 'GET', 'HEAD' ]); + const cache_invalidating_methods = new Set(conf.cache_invalidating_methods ?? [ 'POST', 'PUT', 'PATCH', 'DELETE' ]); + const cacheable_status_codes = new Set(conf.cacheable_status_codes ?? [ + 200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501 + ]); + + return { + /** + * Should be called before making a new request to check if a cached response already exists + */ + check_for_cached_response(method: string, url_str: HttpURL, headers?: OutgoingHttpHeaders) : Readonly { + if (! cached_responses[url_str]) { + return; + } + + for (const [cache_key, cached] of Object.entries(cached_responses[url_str])) { + if (cache_key === header_cache_key(method, cached.cache_info.vary, headers)) { + return cached; + } + } + }, + + /** + * Should be called when a cached response is found, to determine what if any revalidation is + * required before re-using the response + */ + determine_revalidation_needed(cached: CachedResponse) { + const { cache_control, etag, date, last_modified } = cached.cache_info; + + const must_revalidate = Boolean(cache_control?.must_revalidate || cache_control?.no_cache); + const should_revalidate = must_revalidate || cached.is_stale; + const headers: OutgoingHttpHeaders = { }; + + if (etag) { + headers['if-none-match'] = etag; + } + + if (last_modified || date) { + headers['if-modified-since'] = last_modified?.toUTCString() ?? date?.toUTCString(); + } + + else { + headers['if-modified-since'] = cached.cached_time.toUTCString(); + } + + return { + must_revalidate, + should_revalidate, + headers, + }; + }, + + /** + * Should be called when receiving a non-`304 Not Modified` response from a request, either + * as a novel request, or during a revalidation request + */ + add_response_to_cache(method: string, url_str: HttpURL, headers: OutgoingHttpHeaders, result: HttpResult) { + if (! cacheable_methods.has(method) || ! cacheable_status_codes.has(result.status)) { + logger.info({ method, status: result.status }, 'method/status not cacheable'); + return; + } + + const cache_info = parse_cache_headers(result.headers); + + if (headers.authorization) { + // todo: caching authenticated requests + // see + return; + } + + // todo: if `vary` has changed, invalidate all stored responses + // todo: source link for ^ + + if (cache_info.vary.includes('*')) { + logger.info(cache_info, 'response contains "Vary: *" header that prevents caching'); + return; + } + + if (cache_info.cache_control?.no_store || cache_info.cache_control?.private) { + logger.info(cache_info, 'response contains "Cache-Control" directives that prevent caching'); + return; + } + + const cache_key = header_cache_key(method, cache_info.vary, headers); + unset_expire_timer(url_str, cache_key); + + const cached: CachedResponse = { + cache_key, + cached_time: new Date(Date.now()), + is_stale: false, + url: url_str, + method, + // req_headers: structuredClone(headers), + status: result.status, + body: result.content, + res_headers: structuredClone(result.headers), + cache_info, + expire_timer: null, + }; + + logger.info(cache_info, 'caching response'); + store_to_cache(cached, cache_info); + }, + + /** + * Should be called when receiving a `304 Not Modified` response during a revalidation request + * to update the cache with more recent freshness info + */ + freshen_cache(cached: CachedResponse, headers: IncomingHttpHeaders) { + const cache_info = parse_cache_headers(headers); + unset_expire_timer(cached.url, cached.cache_key); + store_to_cache(cached, cache_info); + }, + + /** + * + */ + invalidate_cache_for_update(method: string, url_str: HttpURL, status: number, headers: IncomingHttpHeaders) { + if (! cache_invalidating_methods.has(method)) { + return; + } + + if (status < 200 || status >= 400) { + return; + } + }, + }; + + function store_to_cache(cached: CachedResponse, cache_info: ParsedCacheHeaders) { + if (! cached_responses[cached.url]) { + cached_responses[cached.url] = Object.create(null); + } + + cached_responses[cached.url][cached.cache_key] = cached; + cached.is_stale = false; + + if (cache_info.cache_control?.max_age) { + let ttl_ms = cache_info.cache_control.max_age * 1000; + + if (cache_info.age) { + ttl_ms -= cache_info.age * 1000; + } + + if (ttl_ms <= 0) { + cached.is_stale = true; + return; + } + + cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms); + } + + else if (cache_info.expires) { + const ttl_ms = cache_info.expires.getTime() - Date.now(); + + if (ttl_ms <= 0) { + cached.is_stale = true; + return; + } + + cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms); + } + + // If no cache-control or expires header is given, default to the + // recommended 10% of time since last-modified + else if (cache_info.last_modified) { + const last_modified_ago_ms = Date.now() - cache_info.last_modified.getTime(); + + if (last_modified_ago_ms <= 0) { + cached.is_stale = true; + return; + } + + const ttl_ms = (last_modified_ago_ms / 10) | 0; + cached.expire_timer = set_expire_timer(cached.url, cached.cache_key, ttl_ms); + } + } + + function unset_expire_timer(url_str: HttpURL, cache_key: string) { + const cached = cached_responses[url_str]?.[cache_key]; + + if (cached?.expire_timer) { + clearTimeout(cached.expire_timer); + } + } + + function set_expire_timer(url_str: HttpURL, cache_key: string, ttl_ms: number) { + const timer = setTimeout(expire, ttl_ms); + timer.unref(); + return timer; + + function expire() { + if (cached_responses[url_str]?.[cache_key]) { + cached_responses[url_str][cache_key].is_stale = true; + } + }; + } +} + +function header_cache_key(method: string, vary: string[], headers: OutgoingHttpHeaders) { + const parts: string[] = [ method ]; + + if (vary?.length) { + for (const header of vary) { + if (Array.isArray(headers[header])) { + const values = (headers[header] as string[]).sort(); + for (const value of values) { + parts.push(`${header}: ${value}`); + } + } + + else if (headers[header]) { + parts.push(`${header}: ${headers[header]}`); + } + } + } + + return parts.join('\0'); +}