From abfbabf71f54b5a4023c0a5d217778a727dd97a9 Mon Sep 17 00:00:00 2001 From: James Brumond Date: Mon, 21 Aug 2023 19:45:34 -0700 Subject: [PATCH] migrate snowflake generator to new library --- .gitea/workflows/build-and-publish.yaml | 4 +- .gitea/workflows/build-and-test.yaml | 4 +- package-lock.json | 7 ++ package.json | 11 +-- readme.md | 64 +++++++++++---- src/index.ts | 103 +++++++++++++++++++++++- 6 files changed, 166 insertions(+), 27 deletions(-) diff --git a/.gitea/workflows/build-and-publish.yaml b/.gitea/workflows/build-and-publish.yaml index 07fd5c3..d80b4c7 100644 --- a/.gitea/workflows/build-and-publish.yaml +++ b/.gitea/workflows/build-and-publish.yaml @@ -23,8 +23,8 @@ jobs: - name: Login to package registry run: | - npm config set @:registry https://gitea.jbrumond.me/api/packages//npm/ - npm config set -- '//gitea.jbrumond.me/api/packages//npm/:_authToken' "$NPM_PUBLISH_TOKEN" + npm config set @js:registry https://gitea.jbrumond.me/api/packages/js/npm/ + npm config set -- '//gitea.jbrumond.me/api/packages/js/npm/:_authToken' "$NPM_PUBLISH_TOKEN" - name: Install dependencies run: npm ci diff --git a/.gitea/workflows/build-and-test.yaml b/.gitea/workflows/build-and-test.yaml index 3f97cd5..f074919 100644 --- a/.gitea/workflows/build-and-test.yaml +++ b/.gitea/workflows/build-and-test.yaml @@ -26,8 +26,8 @@ jobs: - name: Login to package registry run: | - npm config set @:registry https://gitea.jbrumond.me/api/packages//npm/ - npm config set -- '//gitea.jbrumond.me/api/packages//npm/:_authToken' "$NPM_PUBLISH_TOKEN" + npm config set @js:registry https://gitea.jbrumond.me/api/packages/js/npm/ + npm config set -- '//gitea.jbrumond.me/api/packages/js/npm/:_authToken' "$NPM_PUBLISH_TOKEN" - name: Install dependencies run: npm ci diff --git a/package-lock.json b/package-lock.json index b8d09b5..e301821 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,16 @@ "version": "1.0.0", "license": "ISC", "devDependencies": { + "@types/node": "^20.5.1", "typescript": "^5.1.3" } }, + "node_modules/@types/node": { + "version": "20.5.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.5.1.tgz", + "integrity": "sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==", + "dev": true + }, "node_modules/typescript": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", diff --git a/package.json b/package.json index ded3a6d..55ddd07 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { - "name": "@templates/typescript-library", - "version": "1.0.0", - "description": "Template project for creating new TypeScript library packages", + "name": "@js/snowflake-uid", + "version": "0.1.0", + "description": "Utility for generating Snowflake UIDs", "main": "build/index.js", "types": "build/index.d.ts", "scripts": { @@ -9,15 +9,16 @@ "clean": "rm -rf ./build" }, "publishConfig": { - "registry": "https://gitea.jbrumond.me/api/packages/templates/npm/" + "registry": "https://gitea.jbrumond.me/api/packages/js/npm/" }, "repository": { "type": "git", - "url": "https://gitea.jbrumond.me/templates/typescript-library.git" + "url": "https://gitea.jbrumond.me/js/snowflake-uid.git" }, "author": "James Brumond ", "license": "ISC", "devDependencies": { + "@types/node": "^20.5.1", "typescript": "^5.1.3" } } diff --git a/readme.md b/readme.md index 7389046..9a055a3 100644 --- a/readme.md +++ b/readme.md @@ -1,30 +1,62 @@ -Template project for creating new TypeScript library packages +Utility for generating Snowflake UIDs (see: ) --- -## Get Started +## Install -### Pull down the code - -```bash -git init -git pull https://gitea.jbrumond.me/templates/typescript-library.git master -``` - -### Update configuration - -- In `package.json`, update any fields like `name`, `description`, `repository`, etc. -- In `.gitea/workflows/publish.yaml`, update `` placeholders + -## Building +## Usage -```bash -npm run tsc +```ts +import { Snowflake, create_snowflake_uid_generator } from '@js/snowflake-uid'; + +const snowflake = create_snowflake_uid_generator({ + // See "More about instance_size" below + instance_size: 10n, + + // Epoch time for the timestamp field. Static field that cannot be + // changed once you start generating IDs. Must be a time in the past. + epoch: BigInt(Date.parse('1 Jan 2020 00:00:00')), + + // Instance ID for this generator. Must be unique across all generators + // in the same ID space. IDs can be reused once a new generator can no + // longer create IDs. These should likely be treated as a shared resource + // allocated from a pool. Must fit inside of the space allocated by the + // instance_size field. + instance: process.env.SNOWFLAKE_INSTANCE_ID, +}); + +// IDs can be generated as bigint or string values +const id1: bigint = snowflake.uid(); +const id2: Snowflake = snowflake.uid_str(); ``` +### More about `instance_size` + +A larger value enables running more concurrent instances generating IDs in the +same space (as few as 16 for 4-bits, or as many as 1024 for 10-bits). + +This number cannot be changed once you start generating IDs, so you should choose +a value that you don't think you will exceed the supported space limitations for. + +Using fewer bits for the instance creates more space for the timestamp and sequence +number fields. The supported configurations are listed below, with the `instance_size` +parameter representing the bit-size of the "instances" column: + +| instances | timestamp space | sequence space | +|--------------------------|-----------------------|-----------------------------| +| `4-bit`: 16 instances | `44-bit`: ~560 years | `15-bit`: 32768 IDs/ms/inst | +| `5-bit`: 32 instances | `44-bit`: ~560 years | `14-bit`: 16384 IDs/ms/inst | +| `6-bit`: 64 instances | `43-bit`: ~280 years | `14-bit`: 16384 IDs/ms/inst | +| `7-bit`: 128 instances | `43-bit`: ~280 years | `13-bit`: 8192 IDs/ms/inst | +| `8-bit`: 256 instances | `42-bit`: ~140 years | `13-bit`: 8192 IDs/ms/inst | +| `9-bit`: 512 instances | `42-bit`: ~140 years | `12-bit`: 4096 IDs/ms/inst | +| `10-bit`: 1024 instances | `41-bit`: ~70 years | `12-bit`: 4096 IDs/ms/inst | + diff --git a/src/index.ts b/src/index.ts index 4fa933d..5eb857c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,103 @@ -export function hello() : string { - return 'hello'; +import { randomInt } from 'crypto'; + +export type Snowflake = `${bigint}`; + +export interface SnowflakeConfig { + /** + * Number of bits allocated to the instance ID. + * + * A larger value enables running more concurrent instances generating IDs in the + * same space (as few as 16 for 4-bits, or as many as 1024 for 10-bits). + * + * This number cannot be changed once you start generating IDs, so you should choose + * a value that you don't think you will exceed the supported instance count for. + * + * Using fewer bits for the instance creates more space for the timestamp and sequence + * number fields. The supported configurations are listed below, with the `instance_size` + * parameter representing the bit-size of the "instances" column: + * + * | instances | timestamp space | sequence space | + * |--------------------------|-----------------------|-----------------------------| + * | `4-bit`: 16 instances | `44-bit`: ~560 years | `15-bit`: 32768 IDs/ms/inst | + * | `5-bit`: 32 instances | `44-bit`: ~560 years | `14-bit`: 16384 IDs/ms/inst | + * | `6-bit`: 64 instances | `43-bit`: ~280 years | `14-bit`: 16384 IDs/ms/inst | + * | `7-bit`: 128 instances | `43-bit`: ~280 years | `13-bit`: 8192 IDs/ms/inst | + * | `8-bit`: 256 instances | `42-bit`: ~140 years | `13-bit`: 8192 IDs/ms/inst | + * | `9-bit`: 512 instances | `42-bit`: ~140 years | `12-bit`: 4096 IDs/ms/inst | + * | `10-bit`: 1024 instances | `41-bit`: ~70 years | `12-bit`: 4096 IDs/ms/inst | + */ + instance_size: 4n | 5n | 6n | 7n | 8n | 9n | 10n; + + /** + * Epoch time, e.g. `1577836800000n` for 1 Jan 2020 00:00:00 + */ + epoch: bigint; + + /** + * Instance ID to use in generated IDs. This value MUST be unique across all generators + * within the ID space (otherwise, ID collisions will occur). Must fit within the space + * allocated by the `instance_size` parameter. + */ + instance: bigint; +} + +export function validate_snowflake_conf(conf: unknown) : asserts conf is SnowflakeConfig { + // todo: validate config +} + +export type SnowflakeUIDGenerator = ReturnType; + +/** + * Generates a unique 64-bit integer ID. + * + * Format based on Snowflake IDs (https://en.wikipedia.org/wiki/Snowflake_ID), + * with the following modifications / details: + * + * - Uses a configurable epoch time + * - Uses a configurable instance size (rather than the hard-coded 10-bits), and increases + * the size of the other fields if set to values smaller than 10. + */ +export function create_snowflake_uid_generator(conf: SnowflakeConfig) { + switch (conf.instance_size) { + case 4n: return generator(15n, 0x7fffn, 0xfffffffffffn); + case 5n: return generator(14n, 0x3fffn, 0xfffffffffffn); + case 6n: return generator(14n, 0x3fffn, 0x7ffffffffffn); + case 7n: return generator(13n, 0x1fffn, 0x7ffffffffffn); + case 8n: return generator(13n, 0x1fffn, 0x3ffffffffffn); + case 9n: return generator(12n, 0x0fffn, 0x3ffffffffffn); + case 10n: return generator(12n, 0x0fffn, 0x1ffffffffffn); + } + + function generator(sequence_size: bigint, sequence_mask: bigint, timestamp_mask: bigint) { + const instance = BigInt(conf.instance) << sequence_size; + const timestamp_offset = conf.instance_size + sequence_size; + + let sequence = BigInt(randomInt(Number(sequence_mask))); + + function next_sequence() { + const value = sequence++; + sequence &= sequence_mask; + return value; + } + + return { + uid() { + const sequence = next_sequence(); + const timestamp = (now_big() - conf.epoch) & timestamp_mask; + return BigInt.asUintN(64, (timestamp << timestamp_offset) | instance | sequence); + }, + uid_str() { + return this.uid().toString(10) as Snowflake; + } + }; + } +} + +export function is_snowflake_uid(value: string) : value is Snowflake { + return /^\d+$/.test(value) && value.length <= 20; +} + +function now_big() { + return BigInt(Date.now()); }