Save

src/functions/definitions.ts

@@ -22,3 +22,40 @@ export interface JsonParseOptions {
     pretty: boolean;
 }
 
+export enum FlushStrategy {
+    sampling,
+    rolling,
+    sliding,
+}
+
+export type AccumulatorOptions<T, R, S> = S extends FlushStrategy.sampling
+    ? SamplingFlushOptions<T, R>
+    : S extends FlushStrategy.sliding
+    ? SlidingFlushOptions<T, R>
+    : S extends FlushStrategy.rolling
+    ? RollingFlushOptions<T, R>
+    : never;
+
+export interface RollingFlushOptions<T, R> {
+    windowLength: number;
+    afterFlush?: (flushed: Array<T>) => Array<R>;
+}
+
+export interface SlidingFlushOptions<T, R> {
+    windowLength: number;
+    afterFlush?: (flushed: Array<T>) => Array<R>;
+}
+
+export interface SlidingFlushResult<T> {
+    first: T;
+}
+
+export interface SamplingFlushOptions<T, R> {
+    condition: (event: T, buffer: Array<T>) => boolean;
+    flushMapper?: (flushed: Array<T>) => Array<R>;
+}
+
+export interface SamplingFlushResult<T> {
+    flushed: boolean;
+    flush?: Array<T> | null;
+}

src/functions/functions.spec.ts

@@ -3,6 +3,7 @@ import test from "ava";
 import { expect } from "chai";
 import { performance } from "perf_hooks";
 import { Readable } from "stream";
+import { FlushStrategy } from "./definitions";
 import {
     fromArray,
     map,
@@ -24,6 +25,7 @@ import {
     unbatch,
     rate,
     parallelMap,
+    accumulator,
 } from ".";
 import { sleep } from "../helpers";
 
@@ -1401,3 +1403,38 @@ test.cb("parallel() parallel mapping", t => {
     source.push("f");
     source.push(null);
 });
+
+test.cb.only("accumulator() buffering strategy", t => {
+    interface TestObject {
+        ts: number;
+        key: string;
+    }
+    const source = new Readable({ objectMode: true });
+    const expectedElements = [
+        { ts: 0, key: "a" },
+        { ts: 1, key: "b" },
+        { ts: 2, key: "c" },
+        { ts: 2, key: "d" },
+        { ts: 3, key: "e" },
+    ];
+    source
+        .pipe(
+            accumulator(FlushStrategy.sampling, {
+                condition: (event: TestObject) => event.ts > 2,
+            }),
+        )
+        .on("data", (flush: TestObject[]) => {
+            console.log("FLUSH", flush);
+            flush.forEach(item => expectedElements.includes(item));
+        })
+        .on("error", e => {
+            console.log("Got error: ", e);
+            t.end();
+        })
+        .on("end", () => {
+            console.log("end");
+            t.end();
+        });
+    source.push(expectedElements);
+    source.push(null);
+});

src/functions/functions.ts

@@ -4,6 +4,10 @@ import { ChildProcess } from "child_process";
 import { StringDecoder } from "string_decoder";
 
 import {
+    FlushStrategy,
+    AccumulatorOptions,
+    SamplingFlushOptions,
+    SamplingFlushResult,
     TransformOptions,
     ThroughOptions,
     WithEncoding,
@@ -602,3 +606,64 @@ export function parallelMap<T, R>(
         },
     });
 }
+
+function samplingFlush<T, R>(
+    event: T,
+    options: SamplingFlushOptions<T, R>,
+    buffer: Array<T>,
+): SamplingFlushResult<T> {
+    let flush = null;
+    if (options.condition(event, buffer)) {
+        flush = buffer.slice(0);
+        buffer.length = 0;
+    }
+    buffer.push(event);
+    return { flushed: true, flush };
+}
+
+function executeSamplingStrategy<T, R>(
+    events: T[],
+    options: SamplingFlushOptions<T, R>,
+    buffer: Array<T>,
+    stream: Transform,
+): void {
+    events.forEach(event => {
+        const sample = samplingFlush(event, options, buffer);
+        if (sample.flushed && sample.flush && options.flushMapper) {
+            stream.push(options.flushMapper(sample.flush));
+        } else if (sample.flushed && sample.flush) {
+            stream.push(sample.flush);
+        }
+    });
+}
+
+export function accumulator<T, R, S extends FlushStrategy>(
+    flushStrategy: S,
+    options: AccumulatorOptions<T, R, S>,
+) {
+    const buffer: Array<T> = [];
+    return new Transform({
+        objectMode: true,
+        async transform(data, encoding, callback) {
+            callback();
+            switch (flushStrategy) {
+                case FlushStrategy.sampling: {
+                    executeSamplingStrategy(
+                        data,
+                        options as SamplingFlushOptions<T, R>,
+                        buffer,
+                        this,
+                    );
+                    break;
+                }
+                case FlushStrategy.sliding: {
+                    break;
+                }
+            }
+        },
+        flush(callback) {
+            this.push(buffer);
+            callback();
+        },
+    });
+}

src/functions/index.ts

@@ -3,6 +3,8 @@ import { ChildProcess } from "child_process";
 import * as baseFunctions from "./functions";
 
 import {
+    AccumulatorOptions,
+    FlushStrategy,
     ThroughOptions,
     TransformOptions,
     WithEncoding,
@@ -205,7 +207,10 @@ export function last<T>(readable: Readable): Promise<T | null> {
  * @param batchSize Size of the batches, defaults to 1000.
  * @param maxBatchAge? Max lifetime of a batch, defaults to 500
  */
-export function batch(batchSize: number, maxBatchAge?: number): NodeJS.ReadWriteStream {
+export function batch(
+    batchSize: number,
+    maxBatchAge?: number,
+): NodeJS.ReadWriteStream {
     return baseFunctions.batch(batchSize, maxBatchAge);
 }
 
@@ -222,7 +227,10 @@ export function unbatch(): NodeJS.ReadWriteStream {
  * @param targetRate? Desired rate in ms
  * @param period? Period to sleep for when rate is above or equal to targetRate
  */
-export function rate(targetRate?: number, period?: number): NodeJS.ReadWriteStream {
+export function rate(
+    targetRate?: number,
+    period?: number,
+): NodeJS.ReadWriteStream {
     return baseFunctions.rate(targetRate, period);
 }
 
@@ -237,5 +245,13 @@ export function parallelMap<T, R>(
     parallel?: number,
     sleepTime?: number,
 ) {
+    console.log("hi");
     return baseFunctions.parallelMap(mapper, parallel, sleepTime);
 }
+
+export function accumulator<T, R, S extends FlushStrategy>(
+    flushStrategy: S,
+    options: AccumulatorOptions<T, R, S>,
+) {
+    return baseFunctions.accumulator(flushStrategy, options);
+}