The Annoyance Filter.pdf - Fourmilab

72 UTF-8 UNICODE DECODER ANNOYANCE-FILTER §84
84. Decode the next logical character. We return −1 when the end of the encoded line is encountered.
〈 Class implementations 11 〉 +≡
int UTF 8 Unicode MBCSdecoder ::getNextDecodedChar (void)
{
int c1 = getNextEncodedByte ( );
if (c1 < 0) {
return c1 ; /∗ End of input stream ∗/
}
string ::size type nbytes = 0;
unsigned int result ;
if (c1 ≤ # 7F) { /∗ Fast track special case for ASCII 7 bit codes ∗/
result = c1 ;
nbytes = 1;
}
else {
unsigned char chn = c1 ;
/∗ N.b. You can dramatically speed up the determination of how many bytes follow the
first byte code by looking it up in a 256 byte table of lengths (with duplicate values as
needed due to value bits in the low order positions. Once the length is determined, you can
use a table look-up to obtain the mask for the first byte rather than developing the mask
with a shift. The code which assembles the rest of the value could also be unrolled into
individual cases to avoid loop overhead. Of course none of this is worth the bother unless
you’re going to be doing this a lot. ∗/
while ((chn & # 80) ≠ 0) {
nbytes ++;
chn ≪= 1;
}
if (nbytes > 6) {
ostringstream os ;
os ≪ name ( ) ≪ "_MBCSdecoder:␣Invalid␣first␣byte␣" ≪ "0x" ≪
setiosflags (ios ::uppercase ) ≪ hex ≪ c1 ≪ "␣in␣UTF−8␣encoded␣string";
reportDecoderDiagnostic(os );
return −1;
}
result = c1 & ( # FF ≫ (nbytes + 1)); /∗ Extract bits from first byte ∗/
for (string ::size type i = 1; i < nbytes ; i++) {
c1 = getNextEncodedByte ( );
if (c1 < 0) {
ostringstream os ;
os ≪ name ( ) ≪ "_MBCSdecoder:␣Premature␣end␣of␣line␣in␣UTF−8␣character.";
reportDecoderDiagnostic(os );
return −1;
}
if ((c1 & # C0) ≠ # 80) {
ostringstream os ;
os ≪ name ( ) ≪ "_MBCSdecoder:␣Bad␣byte␣1−−n␣signature␣in␣UTF−8␣encoded␣sequ\
ence.";
reportDecoderDiagnostic(os );
}
result = (result ≪ 6) | (c1 & # 3F);