This commit is contained in:
Jerry Kurian
2019-09-26 09:23:09 -04:00
parent 70edee51c4
commit a11aa10d16
12 changed files with 36 additions and 56 deletions

View File

@@ -1,7 +1,13 @@
import { Transform, TransformOptions } from "stream";
import { AccumulatorByIteratee, FlushStrategy } from "./baseDefinitions";
import { batch } from ".";
export enum FlushStrategy {
rolling = "rolling",
sliding = "sliding",
}
export type AccumulatorByIteratee<T> = (event: T, bufferChunk: T) => boolean;
function _accumulator<T>(
accumulateBy: (data: T, buffer: T[], stream: Transform) => void,
shouldFlush: boolean = true,

View File

@@ -1,12 +1,3 @@
export interface ThroughOptions {
objectMode?: boolean;
}
export interface TransformOptions {
readableObjectMode?: boolean;
writableObjectMode?: boolean;
}
export interface WithEncoding {
encoding: string;
}
@@ -21,9 +12,3 @@ export type JsonValue = JsonPrimitive | JsonPrimitive[];
export interface JsonParseOptions {
pretty: boolean;
}
export enum FlushStrategy {
rolling = "rolling",
sliding = "sliding",
}
export type AccumulatorByIteratee<T> = (event: T, bufferChunk: T) => boolean;

View File

@@ -1,12 +1,10 @@
import { Transform } from "stream";
import { TransformOptions } from "./baseDefinitions";
import { Transform, TransformOptions } from "stream";
export function batch(
batchSize: number = 1000,
maxBatchAge: number = 500,
options: TransformOptions = {
readableObjectMode: true,
writableObjectMode: true,
objectMode: true,
},
): Transform {
let buffer: any[] = [];

View File

@@ -1,9 +1,6 @@
import { Transform } from "stream";
import { ThroughOptions } from "./baseDefinitions";
import { Transform, TransformOptions } from "stream";
export function collect(
options: ThroughOptions = { objectMode: false },
): Transform {
export function collect(options: TransformOptions = {}): Transform {
const collected: any[] = [];
return new Transform({
...options,

View File

@@ -1,13 +1,11 @@
import { Transform } from "stream";
import { TransformOptions } from "./baseDefinitions";
import { Transform, TransformOptions } from "stream";
export function flatMap<T, R>(
mapper:
| ((chunk: T, encoding: string) => R[])
| ((chunk: T, encoding: string) => Promise<R[]>),
options: TransformOptions = {
readableObjectMode: true,
writableObjectMode: true,
objectMode: true,
},
): Transform {
return new Transform({

View File

@@ -1,12 +1,8 @@
import { Transform } from "stream";
import { TransformOptions } from "./baseDefinitions";
import { Transform, TransformOptions } from "stream";
export function map<T, R>(
mapper: (chunk: T, encoding: string) => R,
options: TransformOptions = {
readableObjectMode: true,
writableObjectMode: true,
},
options: TransformOptions = { objectMode: true },
): Transform {
return new Transform({
...options,

View File

@@ -1,14 +1,12 @@
import { Transform } from "stream";
import { Transform, TransformOptions } from "stream";
import { sleep } from "../helpers";
import { TransformOptions } from "./baseDefinitions";
export function parallelMap<T, R>(
mapper: (data: T) => R,
parallel: number = 10,
sleepTime: number = 5,
options: TransformOptions = {
readableObjectMode: true,
writableObjectMode: true,
objectMode: true,
},
) {
let inflight = 0;

View File

@@ -1,14 +1,12 @@
import { Transform } from "stream";
import { Transform, TransformOptions } from "stream";
import { performance } from "perf_hooks";
import { sleep } from "../helpers";
import { TransformOptions } from "./baseDefinitions";
export function rate(
targetRate: number = 50,
period: number = 1,
options: TransformOptions = {
readableObjectMode: true,
writableObjectMode: true,
objectMode: true,
},
): Transform {
const deltaMS = ((1 / targetRate) * 1000) / period; // Skip a full period

View File

@@ -1,5 +1,4 @@
import { Transform } from "stream";
import { TransformOptions } from "./baseDefinitions";
import { Transform, TransformOptions } from "stream";
export function reduce<T, R>(
iteratee:
@@ -7,14 +6,12 @@ export function reduce<T, R>(
| ((previousValue: R, chunk: T, encoding: string) => Promise<R>),
initialValue: R,
options: TransformOptions = {
readableObjectMode: true,
writableObjectMode: true,
objectMode: true,
},
) {
let value = initialValue;
return new Transform({
readableObjectMode: options.readableObjectMode,
writableObjectMode: options.writableObjectMode,
...options,
async transform(chunk: T, encoding, callback) {
value = await iteratee(value, chunk, encoding);
callback();

View File

@@ -1,10 +1,8 @@
import { Transform } from "stream";
import { TransformOptions } from "./baseDefinitions";
import { Transform, TransformOptions } from "stream";
export function unbatch(
options: TransformOptions = {
readableObjectMode: true,
writableObjectMode: true,
objectMode: true,
},
) {
return new Transform({