File size: 1,917 Bytes
024d9ea
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
"""The FastPLMs feature store: one storage format for every embedding this workspace keeps.

A feature is one value per sequence, addressed by the SHA-256 of the sequence and by a key that
names the model, its revision, the sparse autoencoder, the layer, the pooling, the dtype, and the
residue limit. `store` holds the directory format and `layouts` the row layouts: dense vectors,
compressed-sparse pooled rows, ragged hidden states, and ragged top-k residue codes. `reader`
serves rows by random access to loops that read a batch at a time, `writing` commits a window of
rows as one segment, and `conversion` moves a cache another format holds into a store and proves the
rows survived.
"""

from .async_writer import AsyncFeatureWriter, PackedBatch
from .conversion import (
    ConversionMismatch,
    ConversionReceipt,
    conversion_fingerprint,
    convert_rows,
    describe_file,
)
from .layouts import (
    CSR,
    DENSE,
    LAYOUT_NAMES,
    RAGGED,
    RAGGED_TOPK,
    SparseRow,
    TopKRow,
    dtype_name,
    value_dtype,
)
from .reader import CsrRows, FeatureReader
from .store import (
    FORMAT,
    FeatureStore,
    RowAddress,
    SegmentReceipt,
    SegmentWriter,
    StoredFeature,
    features_in,
    open_feature,
    partition_sequences,
    sequence_digest,
)
from .writing import write_rows


__all__ = [
    "CSR",
    "DENSE",
    "FORMAT",
    "LAYOUT_NAMES",
    "RAGGED",
    "RAGGED_TOPK",
    "AsyncFeatureWriter",
    "ConversionMismatch",
    "ConversionReceipt",
    "CsrRows",
    "FeatureReader",
    "FeatureStore",
    "PackedBatch",
    "RowAddress",
    "SegmentReceipt",
    "SegmentWriter",
    "SparseRow",
    "StoredFeature",
    "TopKRow",
    "conversion_fingerprint",
    "convert_rows",
    "describe_file",
    "dtype_name",
    "features_in",
    "open_feature",
    "partition_sequences",
    "sequence_digest",
    "value_dtype",
    "write_rows",
]