The Annoyance Filter.pdf - Fourmilab

182 MAIN PROGRAM ANNOYANCE-FILTER §227
227. The addFolder procedure reads a mail folder and adds the tokens it contains to the master
dictionary dict with the specified category . The global messageCount for the given category is updated
to reflect the number of messages added from the folder.
〈 Utility functions 219 〉 +≡
static void addFolder (const char ∗fname , dictionaryWord ::mailCategory cat )
{
if (verbose ) {
cerr ≪ "Adding␣" ≪ (bsdFolder ? "BSD␣" : "") ≪ "folder␣" ≪ fname ≪ "␣as␣" ≪
dictionaryWord ::categoryName (cat ) ≪ ":" ≪ endl ;
}
}
mailFolder mf (fname , cat );
mf .setBSDmode (bsdFolder );
bsdFolder = false ; /∗ Reset BSD folder semantics ∗/
tokenParser tp;
tp.setSource (mf );
tp.setTokenDefinition (isoToken , asciiToken );
tp.setTokenLengthLimits (maxTokenLength , minTokenLength , streamMaxTokenLength ,
streamMinTokenLength );
if (pDiagFilename .length ( ) > 0) {
tp.setSaveMessage (true );
}
dictionaryWord dw ;
unsigned int ntokens = 0;
while (tp.nextToken (dw )) {
dict .add (dw , mf .getCategory ( ));
ntokens ++;
〈 Prune unique words from dictionary if autoPrune threshold is exceeded 228 〉;
}
messageCount [mf .getCategory ( )] += mf .getMessageCount ( );
if (verbose ) {
cerr ≪ "␣␣Added␣" ≪ mf .getMessageCount ( ) ≪ "␣messages,␣" ≪ ntokens ≪
"␣tokens␣in␣" ≪ mf .getLineCount ( ) ≪ "␣lines." ≪ endl ;
cerr ≪ "␣␣Dictionary␣contains␣" ≪ dict .size ( ) ≪ "␣unique␣tokens." ≪ endl ;
cerr ≪ "␣␣Dictionary␣size␣" ≪ dict .estimateMemoryRequirement ( ) ≪ "␣bytes." ≪ endl ;
}