shapefile_wasm/
schema.rs

1//! Derives a dBase table schema from GeoJSON properties.
2//!
3//! dBase is far stricter than GeoJSON: field names are capped at 11 bytes, every
4//! column has one fixed type, and every record must supply a value for every
5//! column. On top of that, `dbase` crops any value that overruns its declared
6//! width — silently. So we make one pass over the data to learn the exact widths
7//! needed, rather than guessing and corrupting the tail of long values.
8
9use std::collections::HashMap;
10
11use serde_json::{Map, Value};
12
13use crate::error::{Result, ShapefileError};
14use crate::input::Feature;
15
16/// dBase caps field names at 11 bytes.
17const MAX_NAME_BYTES: usize = 11;
18/// Widest character field dBase can describe.
19pub const MAX_CHARACTER_WIDTH: usize = 254;
20/// More decimal places than an f64 can meaningfully round-trip.
21const MAX_DECIMALS: usize = 15;
22
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum FieldKind {
25    Character { width: u8 },
26    Numeric { width: u8, decimals: u8 },
27    Logical,
28}
29
30impl FieldKind {
31    pub fn label(&self) -> &'static str {
32        match self {
33            FieldKind::Character { .. } => "character",
34            FieldKind::Numeric { .. } => "numeric",
35            FieldKind::Logical => "logical",
36        }
37    }
38}
39
40/// One resolved column: where it came from, what it is called in the .dbf, and
41/// how it is stored.
42#[derive(Debug, Clone)]
43pub struct Field {
44    /// The GeoJSON property name.
45    pub source: String,
46    /// The (sanitised, truncated, de-duplicated) dBase field name.
47    pub name: String,
48    pub kind: FieldKind,
49}
50
51/// Everything we learned about one property while scanning the input.
52#[derive(Default)]
53struct Observation {
54    saw_string: bool,
55    saw_number: bool,
56    saw_bool: bool,
57    saw_nested: bool,
58    /// Longest UTF-8 rendering, used to size character fields.
59    max_text_bytes: usize,
60    /// Most fraction digits needed by any observed number.
61    max_decimals: usize,
62    numbers: Vec<f64>,
63}
64
65impl Observation {
66    fn record(&mut self, value: &Value) {
67        match value {
68            Value::Null => {}
69            Value::String(text) => {
70                self.saw_string = true;
71                self.max_text_bytes = self.max_text_bytes.max(text.len());
72            }
73            Value::Bool(_) => {
74                self.saw_bool = true;
75                self.max_text_bytes = self.max_text_bytes.max(5); // "false"
76            }
77            Value::Number(number) => {
78                self.saw_number = true;
79                if let Some(as_f64) = number.as_f64() {
80                    self.max_decimals = self.max_decimals.max(fraction_digits(as_f64));
81                    self.numbers.push(as_f64);
82                    self.max_text_bytes = self.max_text_bytes.max(shortest_text(as_f64).len());
83                } else {
84                    // Out of f64 range; it can only survive as text.
85                    self.saw_nested = true;
86                    self.max_text_bytes = self.max_text_bytes.max(number.to_string().len());
87                }
88            }
89            other => {
90                self.saw_nested = true;
91                self.max_text_bytes = self.max_text_bytes.max(other.to_string().len());
92            }
93        }
94    }
95
96    fn resolve(&self, max_character_width: usize) -> FieldKind {
97        let distinct_scalars = usize::from(self.saw_string)
98            + usize::from(self.saw_number)
99            + usize::from(self.saw_bool);
100
101        if self.saw_nested || distinct_scalars > 1 {
102            return character(self.max_text_bytes, max_character_width);
103        }
104
105        if self.saw_bool {
106            return FieldKind::Logical;
107        }
108
109        if self.saw_number {
110            // Find the widest precision that still fits inside a dBase field.
111            // Anything that cannot fit at all falls back to text rather than
112            // being cropped into a different number.
113            let mut decimals = self.max_decimals.min(MAX_DECIMALS);
114            loop {
115                let width = self
116                    .numbers
117                    .iter()
118                    .map(|value| format!("{value:.decimals$}").len())
119                    .max()
120                    .unwrap_or(1);
121                if width <= MAX_CHARACTER_WIDTH {
122                    return FieldKind::Numeric {
123                        width: width.max(1) as u8,
124                        decimals: decimals as u8,
125                    };
126                }
127                if decimals == 0 {
128                    return character(self.max_text_bytes, max_character_width);
129                }
130                decimals -= 1;
131            }
132        }
133
134        // Only nulls were ever seen; a narrow text column keeps the file valid.
135        character(self.max_text_bytes.max(1), max_character_width)
136    }
137}
138
139fn character(observed: usize, cap: usize) -> FieldKind {
140    let width = observed.clamp(1, cap.clamp(1, MAX_CHARACTER_WIDTH));
141    FieldKind::Character { width: width as u8 }
142}
143
144/// Rust's `Display` for f64 is the shortest representation that round-trips, and
145/// never uses exponent notation — exactly what a .dbf wants.
146fn shortest_text(value: f64) -> String {
147    if value.is_finite() {
148        format!("{value}")
149    } else {
150        String::new()
151    }
152}
153
154fn fraction_digits(value: f64) -> usize {
155    let text = shortest_text(value);
156    match text.split_once('.') {
157        Some((_, fraction)) => fraction.len(),
158        None => 0,
159    }
160}
161
162/// Reduces a GeoJSON property name to something dBase will accept, keeping the
163/// result unique within the table.
164fn sanitize_name(source: &str, taken: &mut HashMap<String, usize>) -> Result<String> {
165    let mut cleaned: String = source
166        .chars()
167        .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
168        .collect();
169
170    // dBase readers expect a name to start with a letter.
171    if !cleaned
172        .chars()
173        .next()
174        .is_some_and(|c| c.is_ascii_alphabetic())
175    {
176        cleaned.insert(0, 'F');
177    }
178
179    cleaned.truncate(MAX_NAME_BYTES);
180    let cleaned = cleaned.trim_end_matches('_').to_string();
181    let base = if cleaned.is_empty() {
182        "FIELD".to_string()
183    } else {
184        cleaned
185    };
186
187    let next = taken.entry(base.clone()).or_insert(0);
188    *next += 1;
189    if *next == 1 {
190        return Ok(base);
191    }
192
193    // Collision: make room for a numeric suffix inside the 11-byte budget.
194    for attempt in *next..(*next + 1000) {
195        let suffix = attempt.to_string();
196        let keep = MAX_NAME_BYTES.saturating_sub(suffix.len() + 1);
197        let mut candidate = base.clone();
198        candidate.truncate(keep);
199        let candidate = format!("{candidate}_{suffix}");
200        if !taken.contains_key(&candidate) {
201            taken.insert(candidate.clone(), 1);
202            return Ok(candidate);
203        }
204    }
205
206    Err(ShapefileError::FieldName(source.to_string()))
207}
208
209/// Marker for the column we invent when the input carries no properties at all.
210const SYNTHETIC_FID: &str = "\u{0}fid";
211
212/// The full set of columns, in the order they first appeared in the input.
213pub struct Schema {
214    pub fields: Vec<Field>,
215    /// True when the only column is the one we synthesised.
216    pub synthetic: bool,
217}
218
219impl Schema {
220    pub fn infer(features: &[Feature], max_character_width: usize) -> Result<Self> {
221        let mut order: Vec<String> = Vec::new();
222        let mut observations: HashMap<String, Observation> = HashMap::new();
223
224        for feature in features {
225            for (key, value) in &feature.properties {
226                let entry = observations.entry(key.clone()).or_insert_with(|| {
227                    order.push(key.clone());
228                    Observation::default()
229                });
230                entry.record(value);
231            }
232        }
233
234        let mut taken = HashMap::new();
235        let mut fields = Vec::with_capacity(order.len());
236        for source in order {
237            let kind = observations[&source].resolve(max_character_width);
238            let name = sanitize_name(&source, &mut taken)?;
239            fields.push(Field { source, name, kind });
240        }
241
242        // A .dbf with zero columns is technically writable but many GIS readers
243        // reject it, so give attribute-less input a sequential id instead.
244        if fields.is_empty() {
245            fields.push(Field {
246                source: SYNTHETIC_FID.to_string(),
247                name: "FID".to_string(),
248                kind: FieldKind::Numeric {
249                    width: 11,
250                    decimals: 0,
251                },
252            });
253            return Ok(Self {
254                fields,
255                synthetic: true,
256            });
257        }
258
259        Ok(Self {
260            fields,
261            synthetic: false,
262        })
263    }
264
265    pub fn builder(&self) -> Result<dbase::TableWriterBuilder> {
266        // A .dbf carries no encoding of its own; we write UTF-8 and advertise it
267        // in the companion .cpg file.
268        let mut builder = dbase::TableWriterBuilder::with_encoding(dbase::UnicodeLossy);
269
270        for field in &self.fields {
271            let name = dbase::FieldName::try_from(field.name.as_str())
272                .map_err(|_| ShapefileError::FieldName(field.source.clone()))?;
273            builder = match field.kind {
274                FieldKind::Character { width } => builder.add_character_field(name, width),
275                FieldKind::Numeric { width, decimals } => {
276                    builder.add_numeric_field(name, width, decimals)
277                }
278                FieldKind::Logical => builder.add_logical_field(name),
279            };
280        }
281
282        Ok(builder)
283    }
284
285    /// Builds a record that supplies a value for *every* column — `dbase` treats
286    /// a missing key as a hard error rather than a null.
287    pub fn record(&self, properties: &Map<String, Value>, index: usize) -> dbase::Record {
288        let mut record = dbase::Record::default();
289
290        for field in &self.fields {
291            if field.source == SYNTHETIC_FID {
292                record.insert(
293                    field.name.clone(),
294                    dbase::FieldValue::Numeric(Some(index as f64)),
295                );
296                continue;
297            }
298            let value = properties.get(&field.source).unwrap_or(&Value::Null);
299            record.insert(field.name.clone(), to_field_value(value, field.kind));
300        }
301
302        record
303    }
304}
305
306fn to_field_value(value: &Value, kind: FieldKind) -> dbase::FieldValue {
307    match kind {
308        FieldKind::Logical => dbase::FieldValue::Logical(value.as_bool()),
309        FieldKind::Numeric { .. } => dbase::FieldValue::Numeric(match value {
310            Value::Number(number) => number.as_f64().filter(|n| n.is_finite()),
311            _ => None,
312        }),
313        FieldKind::Character { width } => {
314            let text = match value {
315                Value::Null => return dbase::FieldValue::Character(None),
316                Value::String(text) => text.clone(),
317                Value::Number(number) => number
318                    .as_f64()
319                    .map(shortest_text)
320                    .unwrap_or_else(|| number.to_string()),
321                Value::Bool(flag) => flag.to_string(),
322                other => other.to_string(),
323            };
324            dbase::FieldValue::Character(Some(truncate_utf8(text, width as usize)))
325        }
326    }
327}
328
329/// `dbase` crops overlong values at the byte level, which would split a
330/// multi-byte character and produce invalid UTF-8. Cut on a char boundary first.
331fn truncate_utf8(mut text: String, max_bytes: usize) -> String {
332    if text.len() <= max_bytes {
333        return text;
334    }
335    let mut end = max_bytes;
336    while end > 0 && !text.is_char_boundary(end) {
337        end -= 1;
338    }
339    text.truncate(end);
340    text
341}
342
343#[cfg(test)]
344mod tests {
345    use super::*;
346
347    #[test]
348    fn field_names_are_truncated_and_deduplicated() {
349        let mut taken = HashMap::new();
350        assert_eq!(
351            sanitize_name("population", &mut taken).unwrap(),
352            "population"
353        );
354        // "population_density" truncates to the same 11 bytes, so it gains a suffix.
355        assert_eq!(
356            sanitize_name("population_density", &mut taken).unwrap(),
357            "populatio_2"
358        );
359        assert_eq!(
360            sanitize_name("population", &mut taken).unwrap(),
361            "populatio_3"
362        );
363    }
364
365    #[test]
366    fn names_starting_with_a_digit_get_a_prefix() {
367        let mut taken = HashMap::new();
368        assert_eq!(sanitize_name("2020_pop", &mut taken).unwrap(), "F2020_pop");
369    }
370
371    #[test]
372    fn numeric_width_covers_the_widest_value() {
373        let mut observation = Observation::default();
374        observation.record(&serde_json::json!(1.5));
375        observation.record(&serde_json::json!(-12345.25));
376        match observation.resolve(MAX_CHARACTER_WIDTH) {
377            FieldKind::Numeric { width, decimals } => {
378                assert_eq!(decimals, 2);
379                // "-12345.25" is 9 bytes.
380                assert_eq!(width, 9);
381            }
382            other => panic!("expected numeric, got {other:?}"),
383        }
384    }
385
386    #[test]
387    fn mixed_types_fall_back_to_text() {
388        let mut observation = Observation::default();
389        observation.record(&serde_json::json!("a"));
390        observation.record(&serde_json::json!(1));
391        assert!(matches!(
392            observation.resolve(MAX_CHARACTER_WIDTH),
393            FieldKind::Character { .. }
394        ));
395    }
396
397    #[test]
398    fn truncation_respects_char_boundaries() {
399        assert_eq!(truncate_utf8("héllo".to_string(), 2), "h");
400    }
401}
shapefile_wasm/schema.rs

shapefile_wasm/
schema.rs
