1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
#![forbid(unsafe_code)]
use crate::types::*;
use smol::prelude::*;
#[derive(Debug)]
pub struct Decode {
pub c: char,
pub skipped_bytes: usize,
pub found_bytes: usize,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum UTF8ByteType {
Single,
Introducer(u8),
Continuation,
Invalid,
}
pub fn get_utf8_byte_type(b: u8) -> UTF8ByteType {
if b & 0x80 == 0 {
UTF8ByteType::Single
} else if b & 0xC0 == 0x80 {
UTF8ByteType::Continuation
} else if b & 0xE0 == 0xC0 {
UTF8ByteType::Introducer(2)
} else if b & 0xF0 == 0xE0 {
UTF8ByteType::Introducer(3)
} else if b & 0xF8 == 0xF0 {
UTF8ByteType::Introducer(4)
} else {
UTF8ByteType::Invalid
}
}
pub async fn read_utf8_char(input: &mut (impl AsyncRead + Unpin))
-> Result<Decode>
{
let mut buf = vec![0; 4];
let mut unread_byte: Option<u8> = None;
let mut skipped_bytes = 0;
loop {
if let Some(byte) = unread_byte {
buf[0] = byte;
unread_byte = None;
} else {
input.read_exact(&mut buf[0 .. 1]).await?;
}
let found_bytes = match get_utf8_byte_type(buf[0]) {
UTF8ByteType::Single => {
1
},
UTF8ByteType::Introducer(2) => {
input.read_exact(&mut buf[1 .. 2]).await?;
if get_utf8_byte_type(buf[1]) != UTF8ByteType::Continuation {
unread_byte = Some(buf[1]);
skipped_bytes += 1;
continue;
}
2
},
UTF8ByteType::Introducer(3) => {
input.read_exact(&mut buf[1 .. 2]).await?;
if get_utf8_byte_type(buf[1]) != UTF8ByteType::Continuation {
unread_byte = Some(buf[1]);
skipped_bytes += 1;
continue;
}
input.read_exact(&mut buf[2 .. 3]).await?;
if get_utf8_byte_type(buf[2]) != UTF8ByteType::Continuation {
unread_byte = Some(buf[2]);
skipped_bytes += 2;
continue;
}
3
},
UTF8ByteType::Introducer(4) => {
input.read_exact(&mut buf[1 .. 2]).await?;
if get_utf8_byte_type(buf[1]) != UTF8ByteType::Continuation {
unread_byte = Some(buf[1]);
skipped_bytes += 1;
continue;
}
input.read_exact(&mut buf[2 .. 3]).await?;
if get_utf8_byte_type(buf[2]) != UTF8ByteType::Continuation {
unread_byte = Some(buf[2]);
skipped_bytes += 2;
continue;
}
input.read_exact(&mut buf[3 .. 4]).await?;
if get_utf8_byte_type(buf[3]) != UTF8ByteType::Continuation {
unread_byte = Some(buf[3]);
skipped_bytes += 3;
continue;
}
4
},
/* If it's not the start of a valid character, ignore it. */
_ => {
skipped_bytes += 1;
continue;
}
};
if let Ok(string) = std::str::from_utf8(&buf)
&& let Some(c) = string.chars().next()
{
return Ok(Decode { c, skipped_bytes, found_bytes });
}
}
}
|