EdgeAIG's picture
download
raw
6.67 kB
/**
* Encodes and decodes newline-delimited JSON streams in Effect channels.
*
* NDJSON stores one complete JSON value on each line. This module has helpers
* for byte streams, string streams, and schema-checked records, so streaming
* code can read or write one JSON record at a time.
*
* @since 4.0.0
*/
import * as Arr from "../../Array.js";
import * as Channel from "../../Channel.js";
import * as ChannelSchema from "../../ChannelSchema.js";
import * as Data from "../../Data.js";
import * as Effect from "../../Effect.js";
import { dual, identity } from "../../Function.js";
const NdjsonErrorTypeId = "~effect/encoding/Ndjson/NdjsonError";
const encoder = /*#__PURE__*/new TextEncoder();
/**
* Error raised when NDJSON encoding or decoding fails.
*
* **Details**
*
* The `kind` field identifies whether the failure happened while packing or
* unpacking, and `cause` preserves the original error.
*
* @category errors
* @since 4.0.0
*/
export class NdjsonError extends /*#__PURE__*/Data.TaggedError("NdjsonError") {
/**
* Marks this value as an NDJSON encoding or decoding error for runtime guards.
*
* @since 4.0.0
*/
[NdjsonErrorTypeId] = NdjsonErrorTypeId;
/**
* Uses the failed NDJSON operation as the public message.
*
* @since 4.0.0
*/
get message() {
return this.kind;
}
}
/**
* Creates a channel that encodes chunks of values as NDJSON strings.
*
* **Details**
*
* Each input item is `JSON.stringify`-encoded, separated by newlines, and the
* output chunk ends with a trailing newline.
*
* @category constructors
* @since 4.0.0
*/
export const encodeString = () => Channel.fromTransform((upstream, _scope) => Effect.succeed(Effect.flatMap(upstream, input => {
try {
return Effect.succeed(Arr.of(input.map(item => JSON.stringify(item)).join("\n") + "\n"));
} catch (cause) {
return Effect.fail(new NdjsonError({
kind: "Pack",
cause
}));
}
})));
/**
* Creates a channel that encodes chunks of values as UTF-8 NDJSON bytes.
*
* @category constructors
* @since 4.0.0
*/
export const encode = () => Channel.map(encodeString(), Arr.map(_ => encoder.encode(_)));
/**
* Creates an NDJSON byte encoder channel for values of a schema.
*
* **Details**
*
* Values are first encoded with the schema and then written as UTF-8
* newline-delimited JSON.
*
* @category constructors
* @since 4.0.0
*/
export const encodeSchema = schema => () => Channel.pipeTo(ChannelSchema.encode(schema)(), encode());
/**
* Creates an NDJSON string encoder channel for values of a schema.
*
* **Details**
*
* Values are first encoded with the schema and then written as newline-delimited
* JSON strings.
*
* @category constructors
* @since 4.0.0
*/
export const encodeSchemaString = schema => () => Channel.pipeTo(ChannelSchema.encode(schema)(), encodeString());
/**
* Creates a channel that parses NDJSON string chunks into values.
*
* **When to use**
*
* Use when NDJSON input arrives as string chunks and each complete line should
* be parsed into a JSON value.
*
* **Details**
*
* Lines may span input chunks.
*
* **Gotchas**
*
* Set `ignoreEmptyLines` to skip blank lines before calling `JSON.parse`;
* otherwise blank lines are parsed and fail as invalid JSON.
*
* @category constructors
* @since 4.0.0
*/
export const decodeString = options => {
const lines = Channel.splitLines().pipe(options?.ignoreEmptyLines === true ? Channel.filterArray(line => line.length > 0) : identity);
return Channel.mapEffect(lines, chunk => {
try {
return Effect.succeed(Arr.map(chunk, line => JSON.parse(line)));
} catch (cause) {
return Effect.fail(new NdjsonError({
kind: "Unpack",
cause
}));
}
});
};
/**
* Creates a channel that decodes UTF-8 byte chunks and parses them as NDJSON.
*
* **Details**
*
* Lines may span input chunks, and `ignoreEmptyLines` controls whether blank
* lines are skipped before JSON parsing.
*
* @category constructors
* @since 4.0.0
*/
export const decode = options => {
return Channel.pipeTo(Channel.decodeText(), decodeString(options));
};
/**
* Creates an NDJSON byte decoder channel for values of a schema.
*
* **Details**
*
* The channel decodes UTF-8 bytes, parses each NDJSON line, and then decodes
* each parsed value with the schema.
*
* @category constructors
* @since 4.0.0
*/
export const decodeSchema = schema => options => Channel.pipeTo(decode(options), ChannelSchema.decodeUnknown(schema)());
/**
* Creates an NDJSON string decoder channel for values of a schema.
*
* **Details**
*
* The channel parses each line as JSON and then decodes each parsed value with
* the schema.
*
* @category constructors
* @since 4.0.0
*/
export const decodeSchemaString = schema => options => Channel.pipeTo(decodeString(options), ChannelSchema.decodeUnknown(schema)());
/**
* Wraps a bidirectional byte channel with NDJSON encoding and decoding.
*
* **Details**
*
* Outgoing values are written as UTF-8 NDJSON bytes, and incoming bytes are
* parsed as NDJSON values.
*
* @category combinators
* @since 4.0.0
*/
export const duplex = /*#__PURE__*/dual(args => Channel.isChannel(args[0]), (self, options) => Channel.pipeTo(Channel.pipeTo(encode(), self), decode(options)));
/**
* Wraps a bidirectional string channel with NDJSON encoding and decoding.
*
* **Details**
*
* Outgoing values are written as NDJSON strings, and incoming strings are parsed
* as NDJSON values.
*
* @category combinators
* @since 4.0.0
*/
export const duplexString = /*#__PURE__*/dual(args => Channel.isChannel(args[0]), (self, options) => Channel.pipeTo(Channel.pipeTo(encodeString(), self), decodeString(options)));
/**
* Wraps a bidirectional byte channel with schema-aware NDJSON encoding and
* decoding.
*
* **Details**
*
* Values sent to the wrapped channel are encoded with `inputSchema`; bytes
* received from it are parsed as NDJSON and decoded with `outputSchema`.
*
* @category combinators
* @since 4.0.0
*/
export const duplexSchema = /*#__PURE__*/dual(2, (self, options) => ChannelSchema.duplexUnknown(duplex(self, options), options));
/**
* Wraps a bidirectional string channel with schema-aware NDJSON encoding and
* decoding.
*
* **Details**
*
* Values sent to the wrapped channel are encoded with `inputSchema`; strings
* received from it are parsed as NDJSON and decoded with `outputSchema`.
*
* @category combinators
* @since 4.0.0
*/
export const duplexSchemaString = /*#__PURE__*/dual(2, (self, options) => ChannelSchema.duplexUnknown(duplexString(self, options), options));
//# sourceMappingURL=Ndjson.js.map

Xet Storage Details

Size:
6.67 kB
·
Xet hash:
6c41c539396e6cedd16f73a21b59533dea1f33d2abb61d831a2a28e2acd7cdac

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.