The Annoyance Filter.pdf - Fourmilab

142 TOKEN DEFINITION ANNOYANCE-FILTER §170
170. Token definition.
A tokenDefinition object provides the means by which the tokenParser (below) distinguishes tokens
in a stream of text. Tokens are defined by three arrays, each indexed by ISO character codes between 0
and 255. The first, isToken , is true for characters which comprise tokens. The second, notExclusively ,
is true for characters which may appear in tokens, but only in the company of other characters. The
third, notAtEnd is true for characters which may appear within a token, but not at the start or the end
of one.
〈 Class definitions 10 〉 +≡
class tokenDefinition {
protected:
static const int numTokenChars = 256;
bool isToken [numTokenChars ], notExclusively [numTokenChars ], notAtEnd [numTokenChars ];
unsigned int minTokenLength , maxTokenLength ;
public:
tokenDefinition( )
{
clear ( );
}
void clear (void)
{
for (int i = 0; i < numTokenChars ; i++) {
isToken [i] = notExclusively [i] = notAtEnd [i] = false ;
}
setLengthLimits (1, 65535);
}
void setLengthLimits (unsigned int lmin = 0, unsigned int lmax = 0)
{
if (lmin > 0) {
minTokenLength = lmin ;
}
if (lmax > 0) {
maxTokenLength = lmax ;
}
}
unsigned int getLengthMin (void) const
{
return minTokenLength ;
}
unsigned int getLengthMax (void) const
{
return maxTokenLength ;
}
bool isTokenMember (const int c) const
{
assert(c ≥ 0 ∧ c < numTokenChars );
return isToken [c];
}
bool isTokenNotExclusively (const int c) const
{
assert(c ≥ 0 ∧ c < numTokenChars );