xml/
util.rs

1use std::fmt;
2use std::io::{self, Read};
3use std::str::{self, FromStr};
4
5#[derive(Debug)]
6pub(crate) enum CharReadError {
7    UnexpectedEof,
8    Utf8(str::Utf8Error),
9    Io(io::Error),
10}
11
12impl From<str::Utf8Error> for CharReadError {
13    #[cold]
14    fn from(e: str::Utf8Error) -> Self {
15        Self::Utf8(e)
16    }
17}
18
19impl From<io::Error> for CharReadError {
20    #[cold]
21    fn from(e: io::Error) -> Self {
22        Self::Io(e)
23    }
24}
25
26impl fmt::Display for CharReadError {
27    #[cold]
28    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
29        use self::CharReadError::{Io, UnexpectedEof, Utf8};
30        match *self {
31            UnexpectedEof => write!(f, "unexpected end of stream"),
32            Utf8(ref e) => write!(f, "UTF-8 decoding error: {e}"),
33            Io(ref e) => write!(f, "I/O error: {e}"),
34        }
35    }
36}
37
38/// Character encoding used for parsing
39#[derive(Debug, Copy, Clone, Eq, PartialEq)]
40#[non_exhaustive]
41pub enum Encoding {
42    /// Explicitly UTF-8 only
43    Utf8,
44    /// UTF-8 fallback, but can be any 8-bit encoding
45    Default,
46    /// ISO-8859-1
47    Latin1,
48    /// US-ASCII
49    Ascii,
50    /// Big-Endian
51    Utf16Be,
52    /// Little-Endian
53    Utf16Le,
54    /// Unknown endianness yet, will be sniffed
55    Utf16,
56    /// Not determined yet, may be sniffed to be anything
57    Unknown,
58}
59
60// Rustc inlines eq_ignore_ascii_case and creates kilobytes of code!
61#[inline(never)]
62fn icmp(lower: &str, varcase: &str) -> bool {
63    lower.bytes().zip(varcase.bytes()).all(|(l, v)| l == v.to_ascii_lowercase())
64}
65
66impl FromStr for Encoding {
67    type Err = &'static str;
68
69    fn from_str(val: &str) -> Result<Self, Self::Err> {
70        if ["utf-8", "utf8"].into_iter().any(move |label| icmp(label, val)) {
71            Ok(Self::Utf8)
72        } else if ["iso-8859-1", "latin1"].into_iter().any(move |label| icmp(label, val)) {
73            Ok(Self::Latin1)
74        } else if ["utf-16", "utf16"].into_iter().any(move |label| icmp(label, val)) {
75            Ok(Self::Utf16)
76        } else if ["ascii", "us-ascii"].into_iter().any(move |label| icmp(label, val)) {
77            Ok(Self::Ascii)
78        } else {
79            Err("unknown encoding name")
80        }
81    }
82}
83
84impl fmt::Display for Encoding {
85    #[cold]
86    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
87        f.write_str(match self {
88            Self::Utf8 |
89            Self::Default => "UTF-8",
90            Self::Latin1 => "ISO-8859-1",
91            Self::Ascii => "US-ASCII",
92            Self::Utf16Be |
93            Self::Utf16Le |
94            Self::Utf16 => "UTF-16",
95            Self::Unknown => "(unknown)",
96        })
97    }
98}
99
100#[derive(Clone)]
101pub(crate) struct CharReader {
102    pub encoding: Encoding,
103}
104
105impl CharReader {
106    pub fn new(encoding: Encoding) -> Self {
107        Self {
108            encoding,
109        }
110    }
111
112    #[inline]
113    pub fn next_char_from<R: Read>(&mut self, source: &mut R) -> Result<Option<char>, CharReadError> {
114        let mut bytes = source.bytes();
115        const MAX_CODEPOINT_LEN: usize = 4;
116
117        let mut buf = [0u8; MAX_CODEPOINT_LEN];
118        let mut pos = 0;
119        while pos < MAX_CODEPOINT_LEN {
120            let next = match bytes.next() {
121                Some(Ok(b)) => b,
122                Some(Err(e)) => return Err(e.into()),
123                None if pos == 0 => return Ok(None),
124                None => return Err(CharReadError::UnexpectedEof),
125            };
126
127            match self.encoding {
128                Encoding::Utf8 | Encoding::Default => {
129                    // fast path for ASCII subset
130                    if pos == 0 && next.is_ascii() {
131                        return Ok(Some(next.into()));
132                    }
133
134                    buf[pos] = next;
135                    pos += 1;
136
137                    match str::from_utf8(&buf[..pos]) {
138                        Ok(s) => return Ok(s.chars().next()), // always Some(..)
139                        Err(_) if pos < MAX_CODEPOINT_LEN => continue,
140                        Err(e) => return Err(e.into()),
141                    }
142                },
143                Encoding::Latin1 => {
144                    return Ok(Some(next.into()));
145                },
146                Encoding::Ascii => {
147                    return if next.is_ascii() {
148                        Ok(Some(next.into()))
149                    } else {
150                        Err(CharReadError::Io(io::Error::new(io::ErrorKind::InvalidData, "char is not ASCII")))
151                    };
152                },
153                Encoding::Unknown | Encoding::Utf16 => {
154                    buf[pos] = next;
155                    pos += 1;
156                    if let Some(value) = self.sniff_bom(&buf[..pos], &mut pos) {
157                        return value;
158                    }
159                },
160                Encoding::Utf16Be => {
161                    buf[pos] = next;
162                    pos += 1;
163                    if pos == 2 {
164                        if let Some(Ok(c)) = char::decode_utf16([u16::from_be_bytes(buf[..2].try_into().unwrap())]).next() {
165                            return Ok(Some(c));
166                        }
167                    } else if pos == 4 {
168                        return Self::surrogate([u16::from_be_bytes(buf[..2].try_into().unwrap()), u16::from_be_bytes(buf[2..4].try_into().unwrap())]);
169                    }
170                },
171                Encoding::Utf16Le => {
172                    buf[pos] = next;
173                    pos += 1;
174                    if pos == 2 {
175                        if let Some(Ok(c)) = char::decode_utf16([u16::from_le_bytes(buf[..2].try_into().unwrap())]).next() {
176                            return Ok(Some(c));
177                        }
178                    } else if pos == 4 {
179                        return Self::surrogate([u16::from_le_bytes(buf[..2].try_into().unwrap()), u16::from_le_bytes(buf[2..4].try_into().unwrap())]);
180                    }
181                },
182            }
183        }
184        Err(CharReadError::Io(io::ErrorKind::InvalidData.into()))
185    }
186
187    #[cold]
188    fn sniff_bom(&mut self, buf: &[u8], pos: &mut usize) -> Option<Result<Option<char>, CharReadError>> {
189        // sniff BOM
190        if buf.len() <= 3 && [0xEF, 0xBB, 0xBF].starts_with(buf) {
191            if buf.len() == 3 && self.encoding != Encoding::Utf16 {
192                *pos = 0;
193                self.encoding = Encoding::Utf8;
194            }
195        } else if buf.len() <= 2 && [0xFE, 0xFF].starts_with(buf) {
196            if buf.len() == 2 {
197                *pos = 0;
198                self.encoding = Encoding::Utf16Be;
199            }
200        } else if buf.len() <= 2 && [0xFF, 0xFE].starts_with(buf) {
201            if buf.len() == 2 {
202                *pos = 0;
203                self.encoding = Encoding::Utf16Le;
204            }
205        } else if buf.len() == 1 && self.encoding == Encoding::Utf16 {
206            // sniff ASCII char in UTF-16
207            self.encoding = if buf[0] == 0 { Encoding::Utf16Be } else { Encoding::Utf16Le };
208        } else {
209            // UTF-8 is the default, but XML decl can change it to other 8-bit encoding
210            self.encoding = Encoding::Default;
211            if buf.len() == 1 && buf[0].is_ascii() {
212                return Some(Ok(Some(buf[0].into())));
213            }
214        }
215        None
216    }
217
218    fn surrogate(buf: [u16; 2]) -> Result<Option<char>, CharReadError> {
219        char::decode_utf16(buf).next().transpose()
220            .map_err(|e| CharReadError::Io(io::Error::new(io::ErrorKind::InvalidData, e)))
221    }
222}
223
224#[cfg(test)]
225mod tests {
226    use super::{CharReadError, CharReader, Encoding};
227
228    #[test]
229    fn test_next_char_from() {
230        use std::io;
231
232        let mut bytes: &[u8] = b"correct";    // correct ASCII
233        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), Some('c'));
234
235        let mut bytes: &[u8] = b"\xEF\xBB\xBF\xE2\x80\xA2!";  // BOM
236        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), Some('•'));
237
238        let mut bytes: &[u8] = b"\xEF\xBB\xBFx123";  // BOM
239        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), Some('x'));
240
241        let mut bytes: &[u8] = b"\xEF\xBB\xBF";  // Nothing after BOM
242        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), None);
243
244        let mut bytes: &[u8] = b"\xEF\xBB";  // Nothing after BO
245        assert!(matches!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes), Err(CharReadError::UnexpectedEof)));
246
247        let mut bytes: &[u8] = b"\xEF\xBB\x42";  // Nothing after BO
248        assert!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).is_err());
249
250        let mut bytes: &[u8] = b"\xFE\xFF\x00\x42";  // UTF-16
251        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), Some('B'));
252
253        let mut bytes: &[u8] = b"\xFF\xFE\x42\x00";  // UTF-16
254        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), Some('B'));
255
256        let mut bytes: &[u8] = b"\xFF\xFE";  // UTF-16
257        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), None);
258
259        let mut bytes: &[u8] = b"\xFF\xFE\x00";  // UTF-16
260        assert!(matches!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes), Err(CharReadError::UnexpectedEof)));
261
262        let mut bytes: &[u8] = "правильно".as_bytes();  // correct BMP
263        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), Some('п'));
264
265        let mut bytes: &[u8] = "правильно".as_bytes();
266        assert_eq!(CharReader::new(Encoding::Utf16Be).next_char_from(&mut bytes).unwrap(), Some('킿'));
267
268        let mut bytes: &[u8] = "правильно".as_bytes();
269        assert_eq!(CharReader::new(Encoding::Utf16Le).next_char_from(&mut bytes).unwrap(), Some('뿐'));
270
271        let mut bytes: &[u8] = b"\xD8\xD8\x80";
272        assert!(CharReader::new(Encoding::Utf16).next_char_from(&mut bytes).is_err());
273
274        let mut bytes: &[u8] = b"\x00\x42";
275        assert_eq!(CharReader::new(Encoding::Utf16).next_char_from(&mut bytes).unwrap(), Some('B'));
276
277        let mut bytes: &[u8] = b"\x42\x00";
278        assert_eq!(CharReader::new(Encoding::Utf16).next_char_from(&mut bytes).unwrap(), Some('B'));
279
280        let mut bytes: &[u8] = &[0xEF, 0xBB, 0xBF, 0xFF, 0xFF];
281        assert!(CharReader::new(Encoding::Utf16).next_char_from(&mut bytes).is_err());
282
283        let mut bytes: &[u8] = b"\x00";
284        assert!(CharReader::new(Encoding::Utf16Be).next_char_from(&mut bytes).is_err());
285
286        let mut bytes: &[u8] = "😊".as_bytes();          // correct non-BMP
287        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), Some('😊'));
288
289        let mut bytes: &[u8] = b"";                     // empty
290        assert_eq!(CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap(), None);
291
292        let mut bytes: &[u8] = b"\xf0\x9f\x98";         // incomplete code point
293        match CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap_err() {
294            CharReadError::UnexpectedEof => {},
295            e => panic!("Unexpected result: {e:?}")
296        }
297
298        let mut bytes: &[u8] = b"\xff\x9f\x98\x32";     // invalid code point
299        match CharReader::new(Encoding::Unknown).next_char_from(&mut bytes).unwrap_err() {
300            CharReadError::Utf8(_) => {},
301            e => panic!("Unexpected result: {e:?}"),
302        }
303
304        // error during read
305        struct ErrorReader;
306        impl io::Read for ErrorReader {
307            fn read(&mut self, _: &mut [u8]) -> io::Result<usize> {
308                Err(io::Error::new(io::ErrorKind::Other, "test error"))
309            }
310        }
311
312        let mut r = ErrorReader;
313        match CharReader::new(Encoding::Unknown).next_char_from(&mut r).unwrap_err() {
314            CharReadError::Io(ref e) if e.kind() == io::ErrorKind::Other &&
315                                               e.to_string().contains("test error") => {},
316            e => panic!("Unexpected result: {e:?}")
317        }
318    }
319
320    #[test]
321    fn test_latin1_transcoding() {
322        let mut bytes: &[u8] = &[0xE9, 0x20, 0xFC]; // é, space, ü
323        let mut ch = CharReader::new(Encoding::Latin1);
324        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), Some('é'));
325        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), Some(' '));
326        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), Some('ü'));
327        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), None);
328    }
329
330    #[test]
331    fn test_ascii_encoding() {
332        let mut bytes: &[u8] = b"ok";
333        let mut ch = CharReader::new(Encoding::Ascii);
334        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), Some('o'));
335        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), Some('k'));
336        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), None);
337
338        let mut bytes: &[u8] = &[0x80];
339        let mut ch = CharReader::new(Encoding::Ascii);
340        assert!(ch.next_char_from(&mut bytes).is_err());
341    }
342
343    #[test]
344    fn test_encoding_switch() {
345        let data: &[u8] = &[b'<', b'?', 0xE9]; // <? then Latin1 é
346        let mut bytes = data;
347        let mut ch = CharReader::new(Encoding::Default);
348
349        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), Some('<'));
350        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), Some('?'));
351
352        ch.encoding = Encoding::Latin1;
353
354        assert_eq!(ch.next_char_from(&mut bytes).unwrap(), Some('é'));
355    }
356}
xml/util.rs

xml/
util.rs
