minimal/dashboard
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refine outbound request caching

Browse Source
This commit is contained in:
James Brumond 2023-08-12 19:44:58 -07:00
parent 85d72b43d2
commit a89a859e24
Signed by: james
GPG Key ID: E8F2FC44BAA3357A
5 changed files with 24 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/http-web/root-page.ts
View File

@ -28,11 +28,7 @@ export function register_root_page_endpoint(http_server: FastifyInstance, conf:
longitude: -122.6714,
};
const [
{ value: forecast },
{ value: alerts },
{ value: weather }
] = await Promise.all([
const [ forecast, alerts, weather ] = await Promise.all([
services.weather_gov.get_forecast_for_location(location, 'us'),
services.weather_gov.get_alerts_for_location(location),
services.weatherapi_com.get_current_for_location(location),

6
src/http/outbound.ts
View File

@ -103,7 +103,7 @@ export function create_outbound_http_provider(conf: OutboundHttpConfig, logger:
if (cached) {
const revalidation = response_cache.determine_revalidation_needed(cached);
if (! revalidation.must_revalidate) {
if (! revalidation.should_revalidate) {
log.info('serving request from cache (no revalidation required)');
result.status = cached.status;
result.body = cached.body;
@ -111,6 +111,10 @@ export function create_outbound_http_provider(conf: OutboundHttpConfig, logger:
return Promise.resolve(result);
}
// if (! revalidation.must_revalidate) {
// // todo: lazy revalidation option
// }
log.info('cached response found, but requires revalidation');
Object.assign(req_headers, revalidation.headers);
}

4
src/http/request-cache.ts
View File

@ -7,6 +7,8 @@ import { parse_cache_headers, ParsedCacheHeaders } from './parse-cache-headers';
import type { HttpURL } from '../utilities/types';
import type { HttpResult } from './outbound';
// todo: second implementation that stores to disk / db
export interface RequestCacheConfig {
// todo: support for configurable limits:
max_records?: number; // max number of cached responses
@ -67,6 +69,7 @@ export function create_request_cache_provider(conf: RequestCacheConfig, logger:
const { cache_control, etag, date, last_modified } = cached.cache_info;
const must_revalidate = Boolean(cache_control?.must_revalidate || cache_control?.no_cache);
const should_revalidate = must_revalidate || cached.is_stale;
const headers: OutgoingHttpHeaders = { };
if (etag) {
@ -83,6 +86,7 @@ export function create_request_cache_provider(conf: RequestCacheConfig, logger:
return {
must_revalidate,
should_revalidate,
headers,
};
},

45
src/services/weather.gov/index.ts
View File

@ -1,6 +1,5 @@
import pino from 'pino';
import { memo } from '../../utilities/memo';
import { deep_freeze } from '../../utilities/deep';
import { render_forecast } from './render-forecast';
import type { NamedLocation } from '../../storage/named-location';
@ -22,15 +21,6 @@ export type WeatherGovProvider = ReturnType<typeof create_weather_gov_provider>;
export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino.Logger, outbound_http: OutboundHttpProvider) {
const log = logger.child({ logger: 'weather.gov' });
const get_points_ttl = 1000 * 60 * 60 * 24;
const get_points = memo(get_points_ttl, get_points_nocache, { });
const get_forecast_ttl = 1000 * 60 * 30;
const get_forecast = memo(get_forecast_ttl, get_forecast_nocache, { });
const get_alerts_ttl = 1000 * 30;
const get_alerts = memo(get_alerts_ttl, get_alerts_nocache, { });
return {
ready: Promise.resolve(),
@ -43,22 +33,22 @@ export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino
get_alerts: if_enabled(get_alerts),
async get_forecast_for_lat_long(latitude: number, longitude: number, units: 'us' | 'si') {
const { value: points } = await get_points(latitude, longitude);
const points = await get_points(latitude, longitude);
return get_forecast(points.gridId, points.gridX, points.gridY, units, false);
},
async get_hourly_forecast_for_lat_long(latitude: number, longitude: number, units: 'us' | 'si') {
const { value: points } = await get_points(latitude, longitude);
const points = await get_points(latitude, longitude);
return get_forecast(points.gridId, points.gridX, points.gridY, units, true);
},
async get_forecast_for_location(location: NamedLocation, units: 'us' | 'si') {
const { value: points } = await get_points(location.latitude, location.longitude);
const points = await get_points(location.latitude, location.longitude);
return get_forecast(points.gridId, points.gridX, points.gridY, units, false);
},
async get_hourly_forecast_for_location(location: NamedLocation, units: 'us' | 'si') {
const { value: points } = await get_points(location.latitude, location.longitude);
const points = await get_points(location.latitude, location.longitude);
return get_forecast(points.gridId, points.gridX, points.gridY, units, true);
},
@ -79,7 +69,7 @@ export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino
}) as T;
}
async function get_points_nocache(latitude: number, longitude: number) : Promise<Cachable<WeatherGovPoints>> {
async function get_points(latitude: number, longitude: number) : Promise<WeatherGovPoints> {
log.info({ latitude, longitude }, 'fetching points data');
let { status, headers, body } = await outbound_http.get(`https://api.weather.gov/points/${latitude},${longitude}`, {
@ -97,7 +87,7 @@ export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino
if (status === 200) {
const data = JSON.parse(body);
return cacheable(get_points_ttl, data as WeatherGovPoints);
return data as WeatherGovPoints;
}
const data = headers['content-type'].startsWith('application/problem+json')
@ -108,7 +98,7 @@ export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino
throw new Error('failed to fetch points data');
}
async function get_forecast_nocache(gridId: string, gridX: number, gridY: number, units: 'us' | 'si', hourly = false) : Promise<Cachable<WeatherGovForecast>> {
async function get_forecast(gridId: string, gridX: number, gridY: number, units: 'us' | 'si', hourly = false) : Promise<WeatherGovForecast> {
log.info({ gridId, gridX, gridY }, 'fetching forecast info');
const url = `https://api.weather.gov/gridpoints/${gridId}/${gridX},${gridY}/forecast${hourly ? '/hourly' : ''}?units=${units}`;
@ -127,7 +117,7 @@ export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino
if (status === 200) {
const data = JSON.parse(body);
return cacheable(get_forecast_ttl, data as WeatherGovForecast);
return data as WeatherGovForecast;
}
const data = headers['content-type'].startsWith('application/problem+json')
@ -138,7 +128,7 @@ export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino
throw new Error('failed to fetch forecast data');
}
async function get_alerts_nocache(latitude: number, longitude: number) : Promise<Cachable<WeatherGovAlerts>> {
async function get_alerts(latitude: number, longitude: number) : Promise<WeatherGovAlerts> {
log.info({ latitude, longitude }, 'fetching alerts info');
let { status, headers, body } = await outbound_http.get(`https://api.weather.gov/alerts?active=1&point=${latitude},${longitude}`, {
@ -156,7 +146,7 @@ export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino
if (status === 200) {
const data = JSON.parse(body);
return cacheable(get_alerts_ttl, data as WeatherGovAlerts);
return data as WeatherGovAlerts;
}
const data = headers['content-type'].startsWith('application/problem+json')
@ -167,18 +157,3 @@ export function create_weather_gov_provider(conf: WeatherGovConfig, logger: pino
throw new Error('failed to fetch alerts data');
}
}
function cacheable<T>(ttl: number, value: T) : Cachable<T> {
const now = new Date();
return deep_freeze({
fetched: now,
expires: new Date(now.getTime() + ttl),
value,
});
}
interface Cachable<T> {
readonly fetched: Date;
readonly expires: Date;
readonly value: Readonly<T>;
}

36
src/services/weatherapi.com/index.ts
View File

@ -1,13 +1,10 @@
import pino from 'pino';
import { memo } from '../../utilities/memo';
import { deep_freeze } from '../../utilities/deep';
// import { render_forecast } from './render-forecast';
import { render_current_weather } from './render-current-weather';
import { SecretValue, resolve_secret_value } from '../../conf';
import type { OutboundHttpProvider } from '../../http/outbound';
import type { WeatherAPIComCurrentWeather, WeatherAPIComLocation } from './interface';
import type { NamedLocation } from '../../storage/named-location';
import type { OutboundHttpProvider } from '../../http/outbound';
import type { WeatherAPIComCurrentWeather } from './interface';
export interface WeatherAPIComConfig {
enabled: boolean;
@ -24,16 +21,6 @@ export function create_weatherapi_com_provider(conf: WeatherAPIComConfig, logger
const log = logger.child({ logger: 'weatherapi.com' });
const api_key = resolve_secret_value(conf.api_key);
const get_current_ttl = 1000;
// const get_current_ttl = 1000 * 60 * 3;
const get_current = memo(get_current_ttl, get_current_nocache, { });
// const get_forecast_ttl = 1000 * 60 * 30;
// const get_forecast = memo(get_forecast_ttl, get_forecast_nocache, { });
// const get_alerts_ttl = 1000 * 30;
// const get_alerts = memo(get_alerts_ttl, get_alerts_nocache, { });
return {
ready: Promise.resolve(),
@ -59,7 +46,7 @@ export function create_weatherapi_com_provider(conf: WeatherAPIComConfig, logger
}) as T;
}
async function get_current_nocache(query: string) : Promise<Cachable<WeatherAPIComCurrentWeather>> {
async function get_current(query: string) : Promise<WeatherAPIComCurrentWeather> {
log.info({ query }, 'fetching current weather');
let { status, headers, body } = await outbound_http.get(`https://api.weatherapi.com/v1/current.json?key=${api_key}&q=${query}`, {
@ -68,7 +55,7 @@ export function create_weatherapi_com_provider(conf: WeatherAPIComConfig, logger
if (status === 200) {
const data = JSON.parse(body);
return cacheable(get_current_ttl, data as WeatherAPIComCurrentWeather);
return data as WeatherAPIComCurrentWeather;
}
const data = headers['content-type'].startsWith('application/json')
@ -79,18 +66,3 @@ export function create_weatherapi_com_provider(conf: WeatherAPIComConfig, logger
throw new Error('failed to fetch current weather');
}
}
function cacheable<T>(ttl: number, value: T) : Cachable<T> {
const now = new Date();
return deep_freeze({
fetched: now,
expires: new Date(now.getTime() + ttl),
value,
});
}
interface Cachable<T> {
readonly fetched: Date;
readonly expires: Date;
readonly value: Readonly<T>;
}