binoc_sdk/
types.rs

1use std::collections::BTreeMap;
2
3use serde::{Deserialize, Serialize};
4
5use crate::ir::DiffNode;
6
7// ── Artifact types ──────────────────────────────────────────────────
8
9/// Which side of a comparison an artifact describes.
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
11#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
12pub enum ArtifactSubject {
13    #[serde(rename = "left")]
14    Left,
15    #[serde(rename = "right")]
16    Right,
17    #[serde(rename = "pair")]
18    Pair,
19}
20
21/// Identifies an artifact's data format as a structured tuple of
22/// (package, name, version).
23///
24/// - **`package`** — the package that owns and defines this format,
25///   resolvable through the language's normal package system
26///   (e.g. `"binoc"`, `"binoc-csv"`, `"acme-parquet"`).
27/// - **`name`** — the format name within that package
28///   (e.g. `"tabular"`, `"relational-schema"`).
29/// - **`version`** — a single integer. Bump only for breaking schema
30///   changes. Adding optional fields to an existing version is fine
31///   and does not require a bump (JSON/serde naturally ignore unknown
32///   fields and default missing ones).
33#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
34#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
35pub struct ArtifactFormat {
36    pub package: String,
37    pub name: String,
38    pub version: u32,
39}
40
41impl ArtifactFormat {
42    pub fn new(package: impl Into<String>, name: impl Into<String>, version: u32) -> Self {
43        Self {
44            package: package.into(),
45            name: name.into(),
46            version,
47        }
48    }
49}
50
51impl std::fmt::Display for ArtifactFormat {
52    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
53        write!(f, "{}.{}.v{}", self.package, self.name, self.version)
54    }
55}
56
57/// Descriptor for a published artifact attached to a node.
58///
59/// Artifacts are the unified mechanism for both private reuse and
60/// cross-plugin composition. Parse rules publish artifacts; downstream rules
61/// consume them by format.
62#[derive(Debug, Clone, Serialize, Deserialize)]
63#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
64pub struct ArtifactDescriptor {
65    pub format: ArtifactFormat,
66    pub subject: ArtifactSubject,
67    pub producer: String,
68    /// Opaque handle managed by the SDK's DataAccess implementation.
69    /// Plugins should not create or interpret this value directly.
70    pub handle: String,
71}
72
73// ── Standard artifact formats ───────────────────────────────────────
74
75/// Standard format for tabular data artifacts.
76///
77/// Any parser for a tabular source format (CSV, TSV, Excel, Parquet, ...)
78/// should publish artifacts with this format so that generic tabular writers,
79/// compaction rules, and extractors can consume them without
80/// knowing the source format.
81pub fn tabular_v1() -> ArtifactFormat {
82    ArtifactFormat::new("binoc", "tabular", 1)
83}
84
85/// Standard format for a generic, format-neutral value tree.
86///
87/// Produced by parsers for tree-structured formats (JSON, JSONL of mixed shape,
88/// YAML, TOML, ...) and consumed by the structured-document writer. This is the
89/// fallback for any structured source that is not a consistently-shaped record
90/// collection. See the typed-record ADR.
91pub fn structured_document_v1() -> ArtifactFormat {
92    ArtifactFormat::new("binoc", "structured_document", 1)
93}
94
95/// Standard format for tier-3 *parser metadata* — facts a parser extracted about
96/// a node that are not the node's primary data payload: source-format identity
97/// and version, file-level properties, cross-table dictionaries, creator/tooling
98/// provenance. Rides as a second artifact on the parsed node (alongside a
99/// `tabular_v1` leaf, or on a multi-table container that has no table of its
100/// own). Consumed by format, like any artifact; carrying it is useful even with
101/// no current consumer (see the tiered-artifact-metadata ADR).
102pub fn parser_metadata_v1() -> ArtifactFormat {
103    ArtifactFormat::new("binoc", "parser_metadata", 1)
104}
105
106// ── Cell value model ────────────────────────────────────────────────
107
108static NULL_VALUE: Value = Value::Null;
109
110/// A single tabular cell value.
111///
112/// Scalars (`Null`/`Bool`/`Number`/`String`) diff by content. `Nested` holds a
113/// canonicalized object/array (object keys sorted recursively) and participates
114/// in diffs by equality only — a changed nested cell is reported as a cell edit,
115/// but binoc does not recurse into it (see the typed-record ADR). `String` cells
116/// are the all-untyped case used by CSV and other typeless sources.
117#[derive(Debug, Clone, PartialEq, Eq)]
118pub enum Value {
119    Null,
120    Bool(bool),
121    Number(serde_json::Number),
122    String(String),
123    Nested(Box<serde_json::Value>),
124}
125
126impl Value {
127    /// Build a cell value from arbitrary JSON, canonicalizing nested containers
128    /// so that equality is order-independent.
129    pub fn from_json(value: serde_json::Value) -> Self {
130        match value {
131            serde_json::Value::Null => Value::Null,
132            serde_json::Value::Bool(b) => Value::Bool(b),
133            serde_json::Value::Number(n) => Value::Number(n),
134            serde_json::Value::String(s) => Value::String(s),
135            other => Value::Nested(Box::new(canonicalize_json(other))),
136        }
137    }
138
139    /// The JSON representation of this cell, used when building edit params.
140    pub fn to_json(&self) -> serde_json::Value {
141        match self {
142            Value::Null => serde_json::Value::Null,
143            Value::Bool(b) => serde_json::Value::Bool(*b),
144            Value::Number(n) => serde_json::Value::Number(n.clone()),
145            Value::String(s) => serde_json::Value::String(s.clone()),
146            Value::Nested(v) => (**v).clone(),
147        }
148    }
149
150    /// A flat textual rendering for tokenization, CSV serialization, and scoring.
151    pub fn as_text(&self) -> std::borrow::Cow<'_, str> {
152        match self {
153            Value::Null => std::borrow::Cow::Borrowed(""),
154            Value::Bool(true) => std::borrow::Cow::Borrowed("true"),
155            Value::Bool(false) => std::borrow::Cow::Borrowed("false"),
156            Value::Number(n) => std::borrow::Cow::Owned(n.to_string()),
157            Value::String(s) => std::borrow::Cow::Borrowed(s.as_str()),
158            Value::Nested(v) => std::borrow::Cow::Owned(v.to_string()),
159        }
160    }
161
162    /// True when the value carries no content for keying/identity purposes
163    /// (null, or an empty/whitespace string).
164    pub fn is_blank(&self) -> bool {
165        match self {
166            Value::Null => true,
167            Value::String(s) => s.trim().is_empty(),
168            _ => false,
169        }
170    }
171
172    /// Feed a stable, type-tagged byte signature into a hasher (row alignment).
173    pub fn hash_into(&self, hasher: &mut blake3::Hasher) {
174        match self {
175            Value::Null => {
176                hasher.update(&[0]);
177            }
178            Value::Bool(b) => {
179                hasher.update(&[1, *b as u8]);
180            }
181            Value::Number(n) => {
182                hasher.update(&[2]);
183                hasher.update(n.to_string().as_bytes());
184            }
185            Value::String(s) => {
186                hasher.update(&[3]);
187                hasher.update(&(s.len() as u64).to_le_bytes());
188                hasher.update(s.as_bytes());
189            }
190            Value::Nested(v) => {
191                hasher.update(&[4]);
192                hasher.update(v.to_string().as_bytes());
193            }
194        }
195    }
196}
197
198impl Serialize for Value {
199    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
200        self.to_json().serialize(serializer)
201    }
202}
203
204impl<'de> Deserialize<'de> for Value {
205    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
206        Ok(Value::from_json(serde_json::Value::deserialize(
207            deserializer,
208        )?))
209    }
210}
211
212/// Recursively sort object keys so that nested-value equality is order-independent.
213fn canonicalize_json(value: serde_json::Value) -> serde_json::Value {
214    match value {
215        serde_json::Value::Array(items) => {
216            serde_json::Value::Array(items.into_iter().map(canonicalize_json).collect())
217        }
218        serde_json::Value::Object(map) => {
219            let sorted: BTreeMap<String, serde_json::Value> = map
220                .into_iter()
221                .map(|(k, v)| (k, canonicalize_json(v)))
222                .collect();
223            serde_json::Value::Object(sorted.into_iter().collect())
224        }
225        other => other,
226    }
227}
228
229// ── Format-neutral data types ───────────────────────────────────────
230
231/// Format-neutral tabular data: an ordered list of records with a shared column
232/// schema. Produced by CSV, JSON record arrays, JSONL, Excel, Parquet, DB, and
233/// other tabular parsers; consumed by tabular writers, compaction rules, and
234/// extractors.
235///
236/// This is the codec type for the [`tabular_v1`] artifact format.
237/// Serialize with `serde_json::to_vec`, deserialize with `serde_json::from_slice`.
238///
239/// The shape spectrum (rectangular?, named columns?, typed cells?) is *derived*
240/// from the data via [`TabularData::is_rectangular`],
241/// [`TabularData::has_named_columns`], and the cell `Value` variants — rules gate
242/// their behavior on those facts rather than on artifact subtypes.
243#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
244pub struct TabularData {
245    /// Column names. For headerless sources these are synthesized positional
246    /// labels and `has_header` is `false`.
247    pub headers: Vec<String>,
248    pub rows: Vec<Vec<Value>>,
249    /// Whether the source supplied real column names (CSV header, object keys).
250    #[serde(default = "default_true")]
251    pub has_header: bool,
252    /// Declared identity column names, in order. Empty when the source declares
253    /// no key; drives keyed row alignment when present.
254    #[serde(default, skip_serializing_if = "Vec::is_empty")]
255    pub key: Vec<String>,
256    /// Optional source-declared type per column (parallel to `headers`), when the
257    /// source format carries one (DB, Parquet, Stata). Empty means "none".
258    #[serde(default, skip_serializing_if = "Vec::is_empty")]
259    pub column_types: Vec<Option<String>>,
260    /// Optional per-column metadata bag (parallel to `headers`), when the source
261    /// format carries column-scoped facts a generic tabular consumer would not
262    /// otherwise see — labels, display formats, value-label set names, units.
263    /// Each entry is an open object (or `Null` for a column with no metadata).
264    /// Empty means "none". This is tier 1 of the tiered-metadata design (see the
265    /// tiered-artifact-metadata ADR): facts keyed to a *column*.
266    #[serde(default, skip_serializing_if = "Vec::is_empty")]
267    pub column_metadata: Vec<serde_json::Value>,
268    /// Optional table-scoped metadata bag — facts about *this table as a whole*
269    /// that are not per-column and not per-file (a single-table file folds its
270    /// source-format facts here; a table inside a multi-table container carries
271    /// only its own facts, e.g. dataset name/label). `Null` means "none". This
272    /// is tier 2 of the tiered-metadata design.
273    #[serde(default, skip_serializing_if = "serde_json::Value::is_null")]
274    pub table_metadata: serde_json::Value,
275}
276
277fn default_true() -> bool {
278    true
279}
280
281impl TabularData {
282    /// Construct from all-string cells (CSV and other untyped sources). Cells are
283    /// wrapped in [`Value::String`]; the result is byte-identical in behavior to
284    /// the legacy all-string tabular model.
285    pub fn from_string_rows(headers: Vec<String>, rows: Vec<Vec<String>>) -> Self {
286        Self {
287            headers,
288            rows: rows
289                .into_iter()
290                .map(|row| row.into_iter().map(Value::String).collect())
291                .collect(),
292            has_header: true,
293            key: Vec::new(),
294            column_types: Vec::new(),
295            column_metadata: Vec::new(),
296            table_metadata: serde_json::Value::Null,
297        }
298    }
299
300    /// Construct from typed rows with a named header.
301    pub fn new(headers: Vec<String>, rows: Vec<Vec<Value>>) -> Self {
302        Self {
303            headers,
304            rows,
305            has_header: true,
306            key: Vec::new(),
307            column_types: Vec::new(),
308            column_metadata: Vec::new(),
309            table_metadata: serde_json::Value::Null,
310        }
311    }
312
313    /// Attach tier-1 per-column metadata (parallel to `headers`). Builder-style
314    /// so producers can enrich a table without restating every field.
315    pub fn with_column_metadata(mut self, column_metadata: Vec<serde_json::Value>) -> Self {
316        self.column_metadata = column_metadata;
317        self
318    }
319
320    /// Attach tier-2 table-scoped metadata.
321    pub fn with_table_metadata(mut self, table_metadata: serde_json::Value) -> Self {
322        self.table_metadata = table_metadata;
323        self
324    }
325
326    pub fn column_index(&self, name: &str) -> Option<usize> {
327        self.headers.iter().position(|h| h == name)
328    }
329
330    pub fn column_values(&self, name: &str) -> Option<Vec<&Value>> {
331        let idx = self.column_index(name)?;
332        Some(
333            self.rows
334                .iter()
335                .map(|r| r.get(idx).unwrap_or(&NULL_VALUE))
336                .collect(),
337        )
338    }
339
340    /// Every row has arity equal to the column count.
341    pub fn is_rectangular(&self) -> bool {
342        let width = self.headers.len();
343        self.rows.iter().all(|row| row.len() == width)
344    }
345
346    /// The source supplied real, usable column names.
347    pub fn has_named_columns(&self) -> bool {
348        self.has_header && !self.headers.is_empty()
349    }
350
351    /// Columns can be identified across rows and snapshots — the precondition for
352    /// cell-grain and column-grain edits. Otherwise the writer degrades to
353    /// row-grain output.
354    pub fn stable_columns(&self) -> bool {
355        self.has_named_columns() || self.is_rectangular()
356    }
357
358    pub fn to_csv(&self) -> String {
359        let mut out = self.headers.join(",");
360        out.push('\n');
361        for row in &self.rows {
362            let cells: Vec<String> = row.iter().map(|v| v.as_text().into_owned()).collect();
363            out.push_str(&cells.join(","));
364            out.push('\n');
365        }
366        out
367    }
368}
369
370/// Generic format-neutral value tree. Codec type for [`structured_document_v1`].
371///
372/// All source formats transcode their content into a single `serde_json::Value`
373/// tree; `format` records the origin ("json", "yaml", "toml", ...) and `source`
374/// is an open bag of serialization facts (key order, indentation, BOM, trailing
375/// newline) that consumers ignore when unknown.
376#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
377pub struct StructuredDocument {
378    pub value: serde_json::Value,
379    pub format: String,
380    #[serde(default, skip_serializing_if = "serde_json::Value::is_null")]
381    pub source: serde_json::Value,
382}
383
384/// Codec type for [`parser_metadata_v1`] — tier-3 parser metadata.
385///
386/// `format` is the producer's source-format identity (e.g. `"stata_dta"`,
387/// `"sas7bdat"`, `"sas_xport"`), so a consumer can interpret `value` without
388/// guessing. `value` is an open bag of parser-level facts; consumers diff it
389/// generically and ignore keys they do not recognize. Deliberately flat: this
390/// is "a matching subtype for a record artifact" today, and may grow typed
391/// structure in a future version rather than via artifact-format inheritance.
392#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
393pub struct ParserMetadata {
394    pub format: String,
395    pub value: serde_json::Value,
396}
397
398impl ParserMetadata {
399    pub fn new(format: impl Into<String>, value: serde_json::Value) -> Self {
400        Self {
401            format: format.into(),
402            value,
403        }
404    }
405}
406
407// ── Dataset semantics config ───────────────────────────────────────
408
409/// SDK-owned dataset semantics section shared by plugins.
410///
411/// Hosts pass this through unchanged; plugins deserialize the parts they
412/// understand. The schema is intentionally conservative in v1.
413#[derive(Debug, Clone, Default, Serialize, Deserialize)]
414pub struct DatasetSemanticsV1 {
415    #[serde(default)]
416    pub defaults: DatasetDefaults,
417    #[serde(default)]
418    pub paths: Vec<PathConfigEntry>,
419    #[serde(default)]
420    pub files: FileIdentityConfig,
421    #[serde(default)]
422    pub correspondence: CorrespondenceConfig,
423    #[serde(default)]
424    pub reduced_precision: ReducedPrecisionConfig,
425}
426
427#[derive(Debug, Clone, Serialize, Deserialize)]
428pub struct ReducedPrecisionConfig {
429    /// String sentinels that represent a suppressed published value after
430    /// reduced-precision post-processing. The empty string covers both blank
431    /// cells and `null`, preserving the historical blank/null sentinel.
432    #[serde(default = "default_suppression_sentinels")]
433    pub suppression_sentinels: Vec<String>,
434}
435
436impl Default for ReducedPrecisionConfig {
437    fn default() -> Self {
438        Self {
439            suppression_sentinels: default_suppression_sentinels(),
440        }
441    }
442}
443
444fn default_suppression_sentinels() -> Vec<String> {
445    vec!["*".into(), "(D)".into(), "(S)".into(), "".into()]
446}
447
448#[derive(Debug, Clone, Default, Serialize, Deserialize)]
449pub struct DatasetDefaults {
450    #[serde(default)]
451    pub row_identity: RowIdentity,
452}
453
454#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
455#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
456pub struct NodeIdentity {
457    #[serde(default, skip_serializing_if = "String::is_empty")]
458    pub key_attribute: String,
459}
460
461#[derive(Debug, Clone, Default, Serialize)]
462pub struct PathConfigEntry {
463    #[serde(default, rename = "match")]
464    pub match_: String,
465    #[serde(default, skip_serializing_if = "Option::is_none")]
466    pub content_type: Option<String>,
467    #[serde(default, skip_serializing_if = "Option::is_none")]
468    pub rule: Option<String>,
469    #[serde(default, skip_serializing_if = "Option::is_none")]
470    pub shape: Option<TabularShapeConfig>,
471    #[serde(default, skip_serializing_if = "Option::is_none")]
472    pub dialect: Option<CsvDialectConfig>,
473    #[serde(default, skip_serializing_if = "Option::is_none")]
474    pub records_path: Option<String>,
475    #[serde(default, skip_serializing_if = "Option::is_none")]
476    pub row_identity: Option<RowIdentityPatch>,
477    #[serde(default, skip_serializing_if = "Option::is_none")]
478    pub node_identity: Option<NodeIdentity>,
479    #[serde(skip)]
480    pub unknown_fields: Vec<String>,
481}
482
483impl<'de> Deserialize<'de> for PathConfigEntry {
484    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
485    where
486        D: serde::Deserializer<'de>,
487    {
488        #[derive(Default, Deserialize)]
489        struct RawPathConfigEntry {
490            #[serde(default, rename = "match")]
491            match_: String,
492            #[serde(default)]
493            content_type: Option<String>,
494            #[serde(default)]
495            rule: Option<String>,
496            #[serde(default)]
497            dialect: Option<CsvDialectConfig>,
498            #[serde(default)]
499            shape: Option<TabularShapeConfig>,
500            #[serde(default)]
501            records_path: Option<String>,
502            #[serde(default)]
503            row_identity: Option<RowIdentityPatch>,
504            #[serde(default)]
505            node_identity: Option<NodeIdentity>,
506            #[serde(flatten)]
507            extra: BTreeMap<String, serde_json::Value>,
508        }
509
510        let raw = RawPathConfigEntry::deserialize(deserializer)?;
511        Ok(Self {
512            match_: raw.match_,
513            content_type: raw.content_type,
514            rule: raw.rule,
515            dialect: raw.dialect,
516            shape: raw.shape,
517            records_path: raw.records_path,
518            row_identity: raw.row_identity,
519            node_identity: raw.node_identity,
520            unknown_fields: raw.extra.into_keys().collect(),
521        })
522    }
523}
524
525#[derive(Debug, Clone, Default, Serialize, Deserialize)]
526pub struct CorrespondenceConfig {
527    #[serde(default, skip_serializing_if = "Option::is_none")]
528    pub expand_renamed_unchanged_collections: Option<bool>,
529    /// Byte threshold above which stdlib tabular rules skip in-memory
530    /// `tabular_v1` materialization and use the bounded streaming keyed-writer
531    /// path instead. `None` uses the stdlib default.
532    #[serde(default, skip_serializing_if = "Option::is_none")]
533    pub large_tabular_threshold_bytes: Option<u64>,
534    /// Maximum decompressed size of a single gzip stream, in bytes. `None` uses
535    /// the stdlib default. Raise this for legitimately large `.gz` payloads;
536    /// the cap exists only as a decompression-bomb bound, so any value over a
537    /// bundle's real size is safe.
538    #[serde(default, skip_serializing_if = "Option::is_none")]
539    pub max_gzip_bytes: Option<u64>,
540    /// Maximum decompressed size of a single archive entry (one member of a
541    /// `.zip`/`.tar`/`.tgz`), in bytes. `None` uses the stdlib default.
542    #[serde(default, skip_serializing_if = "Option::is_none")]
543    pub max_archive_entry_bytes: Option<u64>,
544    /// Maximum total decompressed size of a whole archive (sum over all
545    /// extracted entries), in bytes. `None` uses the stdlib default. This is the
546    /// cap a real multi-GB government bundle is most likely to hit.
547    #[serde(default, skip_serializing_if = "Option::is_none")]
548    pub max_archive_total_bytes: Option<u64>,
549}
550
551#[derive(Debug, Clone, Default, Serialize, Deserialize)]
552pub struct FileIdentityConfig {
553    #[serde(default)]
554    pub correspondences: Vec<FileCorrespondenceRule>,
555}
556
557#[derive(Debug, Clone, Serialize, Deserialize)]
558pub struct FileCorrespondenceRule {
559    pub name: String,
560    #[serde(default)]
561    pub left: FileSelector,
562    #[serde(default)]
563    pub right: FileSelector,
564    pub key: String,
565    #[serde(default, skip_serializing_if = "Option::is_none")]
566    pub logical_path: Option<String>,
567    #[serde(default)]
568    pub cardinality: Cardinality,
569    #[serde(default)]
570    pub on_null_key: IdentityFailurePolicy,
571    #[serde(default)]
572    pub on_duplicate_key: IdentityFailurePolicy,
573    #[serde(default)]
574    pub report_path_change: bool,
575}
576
577#[derive(Debug, Clone, Default, Serialize, Deserialize)]
578pub struct FileSelector {
579    #[serde(default, skip_serializing_if = "Option::is_none")]
580    pub path: Option<String>,
581    #[serde(default, skip_serializing_if = "Option::is_none")]
582    pub path_regex: Option<String>,
583}
584
585#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
586#[serde(rename_all = "kebab-case")]
587pub enum Cardinality {
588    #[default]
589    OneToOne,
590}
591
592#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
593#[serde(rename_all = "snake_case")]
594pub enum IdentityFailurePolicy {
595    #[default]
596    Diagnostic,
597    Error,
598    Ignore,
599}
600
601#[derive(Debug, Clone, Serialize, Deserialize)]
602#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
603pub struct TabularParseConfig {
604    #[serde(default = "default_header")]
605    pub header: bool,
606    #[serde(default, skip_serializing_if = "Option::is_none")]
607    pub delimiter: Option<String>,
608    #[serde(default, skip_serializing_if = "Option::is_none")]
609    pub dialect: Option<CsvDialectConfig>,
610    #[serde(default, skip_serializing_if = "Option::is_none")]
611    pub header_line: Option<usize>,
612    #[serde(default, skip_serializing_if = "Option::is_none")]
613    pub skip_lines: Option<usize>,
614    /// JSON record collection path for document formats that need to expose a
615    /// nested array as the tabular record stream.
616    #[serde(default, skip_serializing_if = "Option::is_none")]
617    pub records_path: Option<String>,
618}
619
620impl Default for TabularParseConfig {
621    fn default() -> Self {
622        Self {
623            header: true,
624            delimiter: None,
625            dialect: None,
626            header_line: None,
627            skip_lines: None,
628            records_path: None,
629        }
630    }
631}
632
633fn default_header() -> bool {
634    true
635}
636
637#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
638#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
639pub struct CsvDialectConfig {
640    #[serde(default, skip_serializing_if = "Option::is_none")]
641    pub delimiter: Option<String>,
642    #[serde(default, skip_serializing_if = "Option::is_none")]
643    pub quote: Option<String>,
644    #[serde(default, skip_serializing_if = "Option::is_none")]
645    pub escape: Option<String>,
646    #[serde(default, skip_serializing_if = "Option::is_none")]
647    pub bom: Option<bool>,
648    #[serde(default, skip_serializing_if = "Option::is_none")]
649    pub newline: Option<String>,
650}
651
652#[derive(Debug, Clone, Default, Serialize, Deserialize)]
653#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
654pub struct TabularShapeConfig {
655    #[serde(default, skip_serializing_if = "Option::is_none")]
656    pub has_header: Option<bool>,
657    #[serde(default, skip_serializing_if = "Option::is_none")]
658    pub header_line: Option<usize>,
659    #[serde(default, skip_serializing_if = "Option::is_none")]
660    pub skip_lines: Option<usize>,
661}
662
663impl TabularShapeConfig {
664    pub fn apply_to_parse_config(&self, parse: &mut TabularParseConfig) {
665        if let Some(has_header) = self.has_header {
666            parse.header = has_header;
667        }
668        if let Some(header_line) = self.header_line {
669            parse.header_line = Some(header_line);
670        }
671        if let Some(skip_lines) = self.skip_lines {
672            parse.skip_lines = Some(skip_lines);
673        }
674    }
675}
676
677#[derive(Debug, Clone, Default, Serialize, Deserialize)]
678pub struct RowIdentity {
679    #[serde(default)]
680    pub columns: Vec<String>,
681    #[serde(default)]
682    pub by_position: Vec<usize>,
683    #[serde(default)]
684    pub cardinality: Cardinality,
685    #[serde(default)]
686    pub on_null_key: IdentityFailurePolicy,
687    #[serde(default)]
688    pub on_duplicate_key: IdentityFailurePolicy,
689}
690
691/// Presence-preserving row-identity overrides for a selected path.
692///
693/// Runtime identities and dataset defaults use [`RowIdentity`]. Entry-level
694/// configuration uses this patch type so an explicitly configured default
695/// enum value is distinguishable from an omitted field. `columns` and
696/// `by_position` are alternate key selectors; when both are present,
697/// `columns` takes precedence.
698#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
699pub struct RowIdentityPatch {
700    #[serde(default, skip_serializing_if = "Option::is_none")]
701    pub columns: Option<Vec<String>>,
702    #[serde(default, skip_serializing_if = "Option::is_none")]
703    pub by_position: Option<Vec<usize>>,
704    #[serde(default, skip_serializing_if = "Option::is_none")]
705    pub cardinality: Option<Cardinality>,
706    #[serde(default, skip_serializing_if = "Option::is_none")]
707    pub on_null_key: Option<IdentityFailurePolicy>,
708    #[serde(default, skip_serializing_if = "Option::is_none")]
709    pub on_duplicate_key: Option<IdentityFailurePolicy>,
710}
711
712impl RowIdentityPatch {
713    /// Apply this entry-level override to a concrete inherited identity.
714    pub fn apply_to(&self, identity: &mut RowIdentity) {
715        if let Some(columns) = &self.columns {
716            identity.columns = columns.clone();
717            identity.by_position.clear();
718        } else if let Some(by_position) = &self.by_position {
719            identity.by_position = by_position.clone();
720            identity.columns.clear();
721        }
722        if let Some(cardinality) = self.cardinality {
723            identity.cardinality = cardinality;
724        }
725        if let Some(policy) = self.on_null_key {
726            identity.on_null_key = policy;
727        }
728        if let Some(policy) = self.on_duplicate_key {
729            identity.on_duplicate_key = policy;
730        }
731    }
732
733    pub fn has_key_selector(&self) -> bool {
734        self.columns
735            .as_ref()
736            .is_some_and(|columns| !columns.is_empty())
737            || self
738                .by_position
739                .as_ref()
740                .is_some_and(|positions| !positions.is_empty())
741    }
742}
743
744/// A pair of tabular data (left/right sides of a comparison).
745#[derive(Debug, Clone, Serialize, Deserialize)]
746pub struct TabularDataPair {
747    pub left: Option<TabularData>,
748    pub right: Option<TabularData>,
749}
750
751impl TabularDataPair {
752    /// Build a `TabularDataPair` from [`tabular_v1`] artifacts on a node.
753    ///
754    /// Returns `None` if neither left nor right artifact is present.
755    /// This is the standard way for rules and extractors to obtain
756    /// tabular data without knowing the source format.
757    pub fn from_artifacts(
758        node: &crate::ir::DiffNode,
759        data: &dyn crate::traits::DataAccess,
760    ) -> Option<Self> {
761        let fmt = tabular_v1();
762        let left = node
763            .artifacts
764            .iter()
765            .find(|a| a.format == fmt && a.subject == ArtifactSubject::Left)
766            .and_then(|desc| data.get_artifact(desc).ok()?)
767            .and_then(|bytes| serde_json::from_slice(&bytes).ok());
768        let right = node
769            .artifacts
770            .iter()
771            .find(|a| a.format == fmt && a.subject == ArtifactSubject::Right)
772            .and_then(|desc| data.get_artifact(desc).ok()?)
773            .and_then(|bytes| serde_json::from_slice(&bytes).ok());
774        if left.is_none() && right.is_none() {
775            return None;
776        }
777        Some(Self { left, right })
778    }
779}
780
781// ── Tabular extraction ──────────────────────────────────────────────
782
783/// Shared extraction logic for tabular data.
784///
785/// Given a `TabularDataPair` and an aspect name, produces the
786/// corresponding `ExtractResult`. This is format-neutral — any
787/// writer or compatibility plugin that works with tabular artifacts can
788/// delegate extraction here.
789pub fn tabular_extract(
790    pair: &TabularDataPair,
791    _node: &DiffNode,
792    aspect: &str,
793) -> Option<ExtractResult> {
794    match aspect {
795        "rows_added" => {
796            let right = pair.right.as_ref()?;
797            let left_len = pair.left.as_ref().map_or(0, |l| l.rows.len());
798            if left_len >= right.rows.len() {
799                return Some(ExtractResult::Text("No rows added.\n".into()));
800            }
801            let added = TabularData::new(right.headers.clone(), right.rows[left_len..].to_vec());
802            Some(ExtractResult::Text(added.to_csv()))
803        }
804        "rows_removed" => {
805            let left = pair.left.as_ref()?;
806            let right_len = pair.right.as_ref().map_or(0, |r| r.rows.len());
807            if right_len >= left.rows.len() {
808                return Some(ExtractResult::Text("No rows removed.\n".into()));
809            }
810            let removed = TabularData::new(left.headers.clone(), left.rows[right_len..].to_vec());
811            Some(ExtractResult::Text(removed.to_csv()))
812        }
813        "cells_changed" => {
814            let left = pair.left.as_ref()?;
815            let right = pair.right.as_ref()?;
816            let common_cols = tabular_columns_in_common(left, right);
817            let min_rows = left.rows.len().min(right.rows.len());
818
819            let mut out = String::from("row,column,old_value,new_value\n");
820            for i in 0..min_rows {
821                for col in &common_cols {
822                    let li = left.column_index(col)?;
823                    let ri = right.column_index(col)?;
824                    let lv = left.rows[i].get(li).unwrap_or(&NULL_VALUE);
825                    let rv = right.rows[i].get(ri).unwrap_or(&NULL_VALUE);
826                    if lv != rv {
827                        out.push_str(&format!("{i},{col},{},{}\n", lv.as_text(), rv.as_text()));
828                    }
829                }
830            }
831            Some(ExtractResult::Text(out))
832        }
833        "columns_added" => {
834            let left = pair.left.as_ref()?;
835            let right = pair.right.as_ref()?;
836            let left_set: std::collections::BTreeSet<&str> =
837                left.headers.iter().map(|s| s.as_str()).collect();
838            let added: Vec<&str> = right
839                .headers
840                .iter()
841                .filter(|h| !left_set.contains(h.as_str()))
842                .map(|h| h.as_str())
843                .collect();
844            if added.is_empty() {
845                return Some(ExtractResult::Text("No columns added.\n".into()));
846            }
847            let mut out = String::new();
848            for col in &added {
849                out.push_str(&format!("{col}\n"));
850                if let Some(vals) = right.column_values(col) {
851                    for val in vals {
852                        out.push_str(&format!("  {}\n", val.as_text()));
853                    }
854                }
855            }
856            Some(ExtractResult::Text(out))
857        }
858        "columns_removed" => {
859            let left = pair.left.as_ref()?;
860            let right = pair.right.as_ref()?;
861            let right_set: std::collections::BTreeSet<&str> =
862                right.headers.iter().map(|s| s.as_str()).collect();
863            let removed: Vec<&str> = left
864                .headers
865                .iter()
866                .filter(|h| !right_set.contains(h.as_str()))
867                .map(|h| h.as_str())
868                .collect();
869            if removed.is_empty() {
870                return Some(ExtractResult::Text("No columns removed.\n".into()));
871            }
872            let mut out = String::new();
873            for col in &removed {
874                out.push_str(&format!("{col}\n"));
875                if let Some(vals) = left.column_values(col) {
876                    for val in vals {
877                        out.push_str(&format!("  {}\n", val.as_text()));
878                    }
879                }
880            }
881            Some(ExtractResult::Text(out))
882        }
883        "content" | "full" => {
884            let mut out = String::new();
885            if let Some(left) = &pair.left {
886                out.push_str("--- left\n");
887                out.push_str(&left.to_csv());
888            }
889            if let Some(right) = &pair.right {
890                out.push_str("+++ right\n");
891                out.push_str(&right.to_csv());
892            }
893            Some(ExtractResult::Text(out))
894        }
895        _ => None,
896    }
897}
898
899fn tabular_columns_in_common(left: &TabularData, right: &TabularData) -> Vec<String> {
900    let left_set: std::collections::BTreeSet<&str> =
901        left.headers.iter().map(|s| s.as_str()).collect();
902    right
903        .headers
904        .iter()
905        .filter(|h| left_set.contains(h.as_str()))
906        .cloned()
907        .collect()
908}
909
910// ── Item types ──────────────────────────────────────────────────────
911
912/// Metadata-only view of one side of a comparison. Carries logical identity
913/// and content metadata but NOT a filesystem path — data access goes through
914/// `DataAccess`.
915///
916/// # Metadata invariants
917///
918/// `content_hash`, `size`, and `media_type` are **opportunistic hints**.
919/// Producers (expand rules like directory/zip, or data backends)
920/// populate them when doing so is cheap — typically as a byproduct of work
921/// they were already performing. Consumers **must not assume presence**, but
922/// **may trust presence**: when a field is set, the value accurately reflects
923/// the current bytes. Use [`ItemRef::resolve_hash`] / [`ItemRef::resolve_size`]
924/// to obtain a value with a transparent fall-back read.
925///
926/// This keeps fast paths (directory-only listings, short-circuit identical
927/// detection) cheap while letting consumers that need a value — most notably
928/// the move detector, which correlates leaves across container boundaries —
929/// hydrate on demand.
930#[derive(Debug, Clone, Serialize, Deserialize)]
931#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
932pub struct ItemRef {
933    /// User-meaningful location within a snapshot. `/>` marks a
934    /// decompose boundary; a literal segment beginning with `>` is escaped
935    /// as `\>`.
936    pub logical_path: String,
937    pub is_dir: bool,
938    #[serde(default, skip_serializing_if = "Option::is_none")]
939    pub content_hash: Option<String>,
940    #[serde(default, skip_serializing_if = "Option::is_none")]
941    pub size: Option<u64>,
942    #[serde(default, skip_serializing_if = "Option::is_none")]
943    pub media_type: Option<String>,
944    /// Optional projection metadata supplied by rule packs while they still
945    /// know the vocabulary. Core carries this through but does not interpret
946    /// file names, media types, or plugin-specific tags.
947    #[serde(default, skip_serializing_if = "crate::projection_hint_is_default")]
948    pub projection_hint: crate::ProjectionHint,
949    /// Optional stdlib-resolved tabular parse hints carried from dataset config
950    /// to whichever tabular parser eventually claims this item.
951    #[serde(default, skip_serializing_if = "Option::is_none")]
952    pub tabular_parse: Option<TabularParseConfig>,
953    /// Opaque identifier used by DataAccess implementations to locate data.
954    /// Plugin authors should not create or interpret this value directly.
955    #[serde(default)]
956    pub handle: String,
957}
958
959impl ItemRef {
960    pub fn extension(&self) -> Option<String> {
961        std::path::Path::new(&self.logical_path)
962            .extension()
963            .map(|e| format!(".{}", e.to_string_lossy().to_lowercase()))
964    }
965
966    /// Return the item's BLAKE3 content hash, computing it from bytes if
967    /// not already cached on this `ItemRef`. Never valid for directories.
968    pub fn resolve_hash(&self, data: &dyn crate::DataAccess) -> crate::BinocResult<String> {
969        if let Some(hash) = &self.content_hash {
970            return Ok(hash.clone());
971        }
972        let mut reader = data.open_read(self)?;
973        let mut hasher = blake3::Hasher::new();
974        std::io::copy(&mut reader, &mut hasher)?;
975        Ok(hasher.finalize().to_hex().to_string())
976    }
977
978    /// Return the item's byte length, reading from the backend if not already
979    /// cached on this `ItemRef`. Never valid for directories.
980    pub fn resolve_size(&self, data: &dyn crate::DataAccess) -> crate::BinocResult<u64> {
981        if let Some(size) = self.size {
982            return Ok(size);
983        }
984        let bytes = data.read_bytes(self)?;
985        Ok(bytes.len() as u64)
986    }
987}
988
989/// A pair of items to compare. Either side may be None (add/remove).
990#[derive(Debug, Clone, Serialize, Deserialize)]
991#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
992pub struct ItemPair {
993    pub left: Option<ItemRef>,
994    pub right: Option<ItemRef>,
995}
996
997impl ItemPair {
998    pub fn both(left: ItemRef, right: ItemRef) -> Self {
999        Self {
1000            left: Some(left),
1001            right: Some(right),
1002        }
1003    }
1004
1005    pub fn added(right: ItemRef) -> Self {
1006        Self {
1007            left: None,
1008            right: Some(right),
1009        }
1010    }
1011
1012    pub fn removed(left: ItemRef) -> Self {
1013        Self {
1014            left: Some(left),
1015            right: None,
1016        }
1017    }
1018
1019    pub fn logical_path(&self) -> &str {
1020        self.right
1021            .as_ref()
1022            .or(self.left.as_ref())
1023            .map(|i| i.logical_path.as_str())
1024            .unwrap_or("")
1025    }
1026
1027    pub fn extension(&self) -> Option<String> {
1028        self.right
1029            .as_ref()
1030            .or(self.left.as_ref())
1031            .and_then(|i| i.extension())
1032    }
1033
1034    pub fn media_type(&self) -> Option<&str> {
1035        self.right
1036            .as_ref()
1037            .or(self.left.as_ref())
1038            .and_then(|i| i.media_type.as_deref())
1039    }
1040
1041    pub fn is_dir(&self) -> bool {
1042        self.right.as_ref().is_some_and(|i| i.is_dir)
1043            || self.left.as_ref().is_some_and(|i| i.is_dir)
1044    }
1045
1046    pub fn matching_content_hash(&self) -> Option<&str> {
1047        match (&self.left, &self.right) {
1048            (Some(l), Some(r)) => match (&l.content_hash, &r.content_hash) {
1049                (Some(hl), Some(hr)) if hl == hr => Some(hl.as_str()),
1050                _ => None,
1051            },
1052            _ => None,
1053        }
1054    }
1055}
1056
1057/// Result of an extract (on-demand detail retrieval) operation.
1058pub enum ExtractResult {
1059    Text(String),
1060    Binary(Vec<u8>),
1061}
1062
1063#[cfg(test)]
1064mod tests {
1065    use super::*;
1066
1067    fn bare_item(logical: &str, is_dir: bool) -> ItemRef {
1068        ItemRef {
1069            logical_path: logical.into(),
1070            is_dir,
1071            content_hash: None,
1072            size: None,
1073            media_type: None,
1074            projection_hint: Default::default(),
1075            tabular_parse: None,
1076            handle: String::new(),
1077        }
1078    }
1079
1080    #[test]
1081    fn item_ref_extension() {
1082        let item = bare_item("data.csv", false);
1083        assert_eq!(item.extension(), Some(".csv".into()));
1084    }
1085
1086    #[test]
1087    fn item_ref_extension_none() {
1088        let item = bare_item("Makefile", false);
1089        assert_eq!(item.extension(), None);
1090    }
1091
1092    #[test]
1093    fn item_pair_logical_path_prefers_right() {
1094        let left = bare_item("left.txt", false);
1095        let right = bare_item("right.txt", false);
1096        let pair = ItemPair::both(left, right);
1097        assert_eq!(pair.logical_path(), "right.txt");
1098    }
1099
1100    #[test]
1101    fn item_pair_logical_path_falls_back_to_left() {
1102        let left = bare_item("only.txt", false);
1103        let pair = ItemPair::removed(left);
1104        assert_eq!(pair.logical_path(), "only.txt");
1105    }
1106
1107    #[test]
1108    fn item_pair_is_dir() {
1109        let dir = bare_item("sub", true);
1110        let pair = ItemPair::added(dir);
1111        assert!(pair.is_dir());
1112    }
1113
1114    #[test]
1115    fn item_pair_matching_hash() {
1116        let mut left = bare_item("f", false);
1117        left.content_hash = Some("abc".into());
1118        let mut right = bare_item("f", false);
1119        right.content_hash = Some("abc".into());
1120        let pair = ItemPair::both(left, right);
1121        assert_eq!(pair.matching_content_hash(), Some("abc"));
1122    }
1123
1124    #[test]
1125    fn row_identity_patch_deserialization_preserves_explicit_default_policy() {
1126        let semantics: DatasetSemanticsV1 = serde_json::from_value(serde_json::json!({
1127            "paths": [{
1128                "match": "data.csv",
1129                "row_identity": {
1130                    "columns": ["id"],
1131                    "on_null_key": "diagnostic"
1132                }
1133            }]
1134        }))
1135        .expect("dataset semantics");
1136
1137        let patch = semantics.paths[0]
1138            .row_identity
1139            .as_ref()
1140            .expect("row identity patch");
1141        assert_eq!(
1142            patch.columns.as_deref(),
1143            Some([String::from("id")].as_slice())
1144        );
1145        assert_eq!(patch.on_null_key, Some(IdentityFailurePolicy::Diagnostic));
1146        assert_eq!(patch.on_duplicate_key, None);
1147
1148        let serialized = serde_json::to_value(&semantics.paths[0]).expect("serialize path entry");
1149        assert_eq!(serialized["row_identity"]["on_null_key"], "diagnostic");
1150        assert!(serialized["row_identity"].get("on_duplicate_key").is_none());
1151    }
1152
1153    #[test]
1154    fn row_identity_patch_replaces_inherited_key_selector() {
1155        let mut identity = RowIdentity {
1156            columns: vec!["id".into()],
1157            on_null_key: IdentityFailurePolicy::Error,
1158            ..RowIdentity::default()
1159        };
1160        RowIdentityPatch {
1161            by_position: Some(vec![2]),
1162            on_null_key: Some(IdentityFailurePolicy::Diagnostic),
1163            ..RowIdentityPatch::default()
1164        }
1165        .apply_to(&mut identity);
1166
1167        assert!(identity.columns.is_empty());
1168        assert_eq!(identity.by_position, vec![2]);
1169        assert_eq!(identity.on_null_key, IdentityFailurePolicy::Diagnostic);
1170
1171        RowIdentityPatch {
1172            columns: Some(vec!["email".into()]),
1173            ..RowIdentityPatch::default()
1174        }
1175        .apply_to(&mut identity);
1176        assert_eq!(identity.columns, vec!["email"]);
1177        assert!(identity.by_position.is_empty());
1178    }
1179}
binoc_sdk/types.rs

binoc_sdk/
types.rs
