The Annoyance Filter.pdf - Fourmilab

148 TOKEN PARSER ANNOYANCE-FILTER §174
174. The nextToken method stores the next token from the input source into its dictionary word
argument and returns true if a token was found or false if the end of the input source was encountered
whilst scanning for the next token.
#define ChIx (c) (static cast〈unsigned int〉((c)) & # FF)
〈 Class implementations 11 〉 +≡
bool tokenParser ::nextToken (dictionaryWord &d)
{
string token ;
while (¬atEnd ) {
〈 Check for assembled phrases in queue and return next if so 175 〉;
token = "";
string ::size type necount = 0;
if (source ⃗ isByteStream ( )) {
〈 Parse plausible tokens from byte stream 178 〉;
} /∗ Ignore non-token characters until start of next token ∗/
while ((clp < cl .length ( )) ∧ (inHTMLcomment ∨ (¬(td ⃗ isTokenMember (ChIx (cl [clp])))))) {
〈 Check for HTML comments and ignore them 176 〉;
〈 Check for within HTML content 177 〉;
clp ++;
} /∗ If end of line encountered before token start, advance to next line ∗/
if (clp ≥ cl .length ( )) {
nextLine ( );
continue;
} /∗ Check for characters we don’t accept as the start of a token ∗/
if (td ⃗ isTokenNotAtEnd (ChIx (cl [clp]))) {
clp ++;
continue;
} /∗ First character of token recognised; store and scan balance ∗/
if (td ⃗ isTokenNotExclusively (ChIx (cl [clp]))) {
necount ++;
}
token += cl [clp ++];
while ((clp < cl .length ( ))) {
if ((¬inHTMLcomment ) ∧ (td ⃗ isTokenMember (ChIx (cl [clp])))) {
if (td ⃗ isTokenNotExclusively (ChIx (cl [clp]))) {
necount ++;
}
token += cl [clp ++];
}
else {
〈 Check for HTML comments and ignore them 176 〉;
if (inHTMLcomment ) {
clp ++;
continue;
}
break;
}
} /∗ Prune characters we don’t accept at the end of a token ∗/
while ((token .length ( ) > 0) ∧ td ⃗ isTokenNotAtEnd (ChIx (token [token .length ( ) − 1]))) {
token .erase (token .length ( ) − 1);
} /∗ Verify that the token meets our minimum and maximum length constraints ∗/