Clean up
This commit is contained in:
@@ -1,7 +1,13 @@
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
import { AccumulatorByIteratee, FlushStrategy } from "./baseDefinitions";
|
||||
import { batch } from ".";
|
||||
|
||||
export enum FlushStrategy {
|
||||
rolling = "rolling",
|
||||
sliding = "sliding",
|
||||
}
|
||||
|
||||
export type AccumulatorByIteratee<T> = (event: T, bufferChunk: T) => boolean;
|
||||
|
||||
function _accumulator<T>(
|
||||
accumulateBy: (data: T, buffer: T[], stream: Transform) => void,
|
||||
shouldFlush: boolean = true,
|
||||
|
||||
@@ -1,12 +1,3 @@
|
||||
export interface ThroughOptions {
|
||||
objectMode?: boolean;
|
||||
}
|
||||
|
||||
export interface TransformOptions {
|
||||
readableObjectMode?: boolean;
|
||||
writableObjectMode?: boolean;
|
||||
}
|
||||
|
||||
export interface WithEncoding {
|
||||
encoding: string;
|
||||
}
|
||||
@@ -21,9 +12,3 @@ export type JsonValue = JsonPrimitive | JsonPrimitive[];
|
||||
export interface JsonParseOptions {
|
||||
pretty: boolean;
|
||||
}
|
||||
export enum FlushStrategy {
|
||||
rolling = "rolling",
|
||||
sliding = "sliding",
|
||||
}
|
||||
|
||||
export type AccumulatorByIteratee<T> = (event: T, bufferChunk: T) => boolean;
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import { Transform } from "stream";
|
||||
import { TransformOptions } from "./baseDefinitions";
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
|
||||
export function batch(
|
||||
batchSize: number = 1000,
|
||||
maxBatchAge: number = 500,
|
||||
options: TransformOptions = {
|
||||
readableObjectMode: true,
|
||||
writableObjectMode: true,
|
||||
objectMode: true,
|
||||
},
|
||||
): Transform {
|
||||
let buffer: any[] = [];
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
import { Transform } from "stream";
|
||||
import { ThroughOptions } from "./baseDefinitions";
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
|
||||
export function collect(
|
||||
options: ThroughOptions = { objectMode: false },
|
||||
): Transform {
|
||||
export function collect(options: TransformOptions = {}): Transform {
|
||||
const collected: any[] = [];
|
||||
return new Transform({
|
||||
...options,
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
import { Transform } from "stream";
|
||||
import { TransformOptions } from "./baseDefinitions";
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
|
||||
export function flatMap<T, R>(
|
||||
mapper:
|
||||
| ((chunk: T, encoding: string) => R[])
|
||||
| ((chunk: T, encoding: string) => Promise<R[]>),
|
||||
options: TransformOptions = {
|
||||
readableObjectMode: true,
|
||||
writableObjectMode: true,
|
||||
objectMode: true,
|
||||
},
|
||||
): Transform {
|
||||
return new Transform({
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
import { Transform } from "stream";
|
||||
import { TransformOptions } from "./baseDefinitions";
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
|
||||
export function map<T, R>(
|
||||
mapper: (chunk: T, encoding: string) => R,
|
||||
options: TransformOptions = {
|
||||
readableObjectMode: true,
|
||||
writableObjectMode: true,
|
||||
},
|
||||
options: TransformOptions = { objectMode: true },
|
||||
): Transform {
|
||||
return new Transform({
|
||||
...options,
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
import { Transform } from "stream";
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
import { sleep } from "../helpers";
|
||||
import { TransformOptions } from "./baseDefinitions";
|
||||
|
||||
export function parallelMap<T, R>(
|
||||
mapper: (data: T) => R,
|
||||
parallel: number = 10,
|
||||
sleepTime: number = 5,
|
||||
options: TransformOptions = {
|
||||
readableObjectMode: true,
|
||||
writableObjectMode: true,
|
||||
objectMode: true,
|
||||
},
|
||||
) {
|
||||
let inflight = 0;
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
import { Transform } from "stream";
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
import { performance } from "perf_hooks";
|
||||
import { sleep } from "../helpers";
|
||||
import { TransformOptions } from "./baseDefinitions";
|
||||
|
||||
export function rate(
|
||||
targetRate: number = 50,
|
||||
period: number = 1,
|
||||
options: TransformOptions = {
|
||||
readableObjectMode: true,
|
||||
writableObjectMode: true,
|
||||
objectMode: true,
|
||||
},
|
||||
): Transform {
|
||||
const deltaMS = ((1 / targetRate) * 1000) / period; // Skip a full period
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { Transform } from "stream";
|
||||
import { TransformOptions } from "./baseDefinitions";
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
|
||||
export function reduce<T, R>(
|
||||
iteratee:
|
||||
@@ -7,14 +6,12 @@ export function reduce<T, R>(
|
||||
| ((previousValue: R, chunk: T, encoding: string) => Promise<R>),
|
||||
initialValue: R,
|
||||
options: TransformOptions = {
|
||||
readableObjectMode: true,
|
||||
writableObjectMode: true,
|
||||
objectMode: true,
|
||||
},
|
||||
) {
|
||||
let value = initialValue;
|
||||
return new Transform({
|
||||
readableObjectMode: options.readableObjectMode,
|
||||
writableObjectMode: options.writableObjectMode,
|
||||
...options,
|
||||
async transform(chunk: T, encoding, callback) {
|
||||
value = await iteratee(value, chunk, encoding);
|
||||
callback();
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
import { Transform } from "stream";
|
||||
import { TransformOptions } from "./baseDefinitions";
|
||||
import { Transform, TransformOptions } from "stream";
|
||||
|
||||
export function unbatch(
|
||||
options: TransformOptions = {
|
||||
readableObjectMode: true,
|
||||
writableObjectMode: true,
|
||||
objectMode: true,
|
||||
},
|
||||
) {
|
||||
return new Transform({
|
||||
|
||||
Reference in New Issue
Block a user