1 files changed, 69 insertions, 0 deletions
diff --git a/std/datetime/tokenizer.ts b/std/datetime/tokenizer.ts
new file mode 100644
index 000000000..05314b770
--- /dev/null
+++ b/std/datetime/tokenizer.ts
@@ -0,0 +1,69 @@
+export type Token = {
+  type: string;
+  value: string | number;
+  index: number;
+};
+
+interface ReceiverResult {
+  [name: string]: string | number;
+}
+export type CallbackResult = { type: string; value: string | number };
+type CallbackFunction = (value: unknown) => CallbackResult;
+
+export type TestResult = { value: unknown; length: number } | undefined;
+export type TestFunction = (
+  string: string,
+) => TestResult | undefined;
+
+export interface Rule {
+  test: TestFunction;
+  fn: CallbackFunction;
+}
+
+export class Tokenizer {
+  rules: Rule[];
+
+  constructor(rules: Rule[] = []) {
+    this.rules = rules;
+  }
+
+  addRule(test: TestFunction, fn: CallbackFunction): Tokenizer {
+    this.rules.push({ test, fn });
+    return this;
+  }
+
+  tokenize(
+    string: string,
+    receiver = (token: Token): ReceiverResult => token,
+  ): ReceiverResult[] {
+    function* generator(rules: Rule[]): IterableIterator<ReceiverResult> {
+      let index = 0;
+      for (const rule of rules) {
+        const result = rule.test(string);
+        if (result) {
+          const { value, length } = result;
+          index += length;
+          string = string.slice(length);
+          const token = { ...rule.fn(value), index };
+          yield receiver(token);
+          yield* generator(rules);
+        }
+      }
+    }
+    const tokenGenerator = generator(this.rules);
+
+    const tokens: ReceiverResult[] = [];
+
+    for (const token of tokenGenerator) {
+      tokens.push(token);
+    }
+
+    if (string.length) {
+      throw new Error(
+        `parser error: string not fully parsed! ${string.slice(0, 25)}`,
+      );
+    }
+
+    return tokens;
+  }
+}