Main Page | Class Hierarchy | Alphabetical List | Class List | File List | Class Members | File Members | Related Pages

TokenEx.d

Go to the documentation of this file.
00001 /*******************************************************************************
00002 
00003         @file TokenEx.d
00004         
00005         Copyright (C) 2004 Kris Bell
00006         
00007         This software is provided 'as-is', without any express or implied
00008         warranty. In no event will the authors be held liable for damages
00009         of any kind arising from the use of this software.
00010         
00011         Permission is hereby granted to anyone to use this software for any 
00012         purpose, including commercial applications, and to alter it and/or 
00013         redistribute it freely, subject to the following restrictions:
00014         
00015         1. The origin of this software must not be misrepresented; you must 
00016            not claim that you wrote the original software. If you use this 
00017            software in a product, an acknowledgment within documentation of 
00018            said product would be appreciated but is not required.
00019 
00020         2. Altered source versions must be plainly marked as such, and must 
00021            not be misrepresented as being the original software.
00022 
00023         3. This notice may not be removed or altered from any distribution
00024            of the source.
00025 
00026 
00027                         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
00028 
00029 
00030         @version        Initial version, Oct 2004    
00031         @author         Kris
00032 
00033 
00034 *******************************************************************************/
00035 
00036 module mango.io.TokenEx;
00037 
00038 private import  mango.io.Token,
00039                 mango.io.Tokenizer;
00040 
00041 private import  mango.io.model.IReader,
00042                 mango.io.model.IConduit;
00043 
00044 
00045 /*******************************************************************************
00046 
00047         A style of Token that's bound to a Tokenizer. This can be a handy 
00048         means of cleaning up client code, and limiting the scope of how
00049         a token is used by recieving methods.
00050 
00051         Contrast this example with that shown in the Token class:
00052 
00053         @code
00054         // open a file for reading
00055         FileConduit fc = new FileConduit ("test.txt");
00056 
00057         // create a buffer for reading the file
00058         IBuffer buffer = fc.createBuffer();
00059 
00060         // bind a line-tokenizer to our input token
00061         BoundToken line = new BoundToken (Tokenizers.line);
00062 
00063         // read file a line at a time. Method next() returns false when no more 
00064         // delimiters are found. Note there may be an unterminated line at eof
00065         while (line.next(buffer) || line.getLength())
00066                Stdout.put(line).cr();
00067         @endcode
00068 
00069         One might also consider a CompositeToken or HybridToken.
00070 
00071 *******************************************************************************/
00072 
00073 class BoundToken : Token
00074 { 
00075         private ITokenizer tk;
00076 
00077         /***********************************************************************
00078         
00079         ***********************************************************************/
00080 
00081         this (ITokenizer tk)
00082         {
00083                 this.tk = tk;
00084         }
00085 
00086         /***********************************************************************
00087         
00088                 Return the associated tokenizer
00089 
00090         ***********************************************************************/
00091 
00092         ITokenizer getTokenizer ()
00093         {     
00094                 return tk;
00095         }
00096 
00097         /***********************************************************************
00098         
00099                 Extract the next token from the provided buffer.
00100 
00101                 Returns true if a token was isolated, false if no more 
00102                 tokens were found. Note that one last token may still
00103                 be present when this return false; this may happen if
00104                 (for example) the last delimiter is missing before an
00105                 EOF condition is seen. Check token.getLength() when
00106                 this method returns false.
00107                 
00108                 For example:
00109 
00110                 @code
00111                         while (token.next() || token.getLength())
00112                                // do something
00113 
00114                 @endcode               
00115 
00116         ***********************************************************************/
00117 
00118         bool next (IBuffer buf)
00119         {
00120                 return tk.next (buf, this);
00121         }
00122 }
00123 
00124 
00125 /*******************************************************************************
00126 
00127         ReaderToken adapts a BoundToken such that it can be used directly
00128         with any IReader implementation. We just add the IReadable methods
00129         to the basic BoundToken.
00130 
00131         Here's a contrived example of how to use ReaderToken:
00132 
00133         @code
00134         // create a small buffer on the heap
00135         Buffer buf = new Buffer (256);
00136 
00137         // write items with a comma between each
00138         TextWriter tw = new TextWriter (buf, ",");
00139 
00140         // write some stuff to the buffer
00141         tw << "now is the time for all good men" << 3.14159;
00142 
00143         // bind a couple of tokens to a comma tokenizer
00144         ReaderToken text = new ReaderToken (Tokenizers.comma);
00145         ReaderToken number = new ReaderToken (Tokenizers.comma);
00146         
00147         // create any old reader since we only use it for handling tokens
00148         Reader r = new Reader (buf);
00149 
00150         // populate both tokens via reader 
00151         r >> text >> number;
00152 
00153         // print them to the console
00154         Stdout << text << ':' << number << Stdout.newline;
00155         @endcode
00156 
00157 *******************************************************************************/
00158 
00159 class ReaderToken : BoundToken, IReadable
00160 { 
00161         /***********************************************************************
00162         
00163                 Construct a ReaderToken using the provided Tokenizer.
00164 
00165         ***********************************************************************/
00166 
00167         this (ITokenizer tk)
00168         {
00169                 super (tk);
00170         }
00171 
00172         /***********************************************************************
00173         
00174                 Read the next delimited element into this token.
00175 
00176         ***********************************************************************/
00177 
00178         void read (IReader r)
00179         {
00180                 tk.next (r.getBuffer(), this);
00181         }
00182 }
00183 
00184 
00185 /*******************************************************************************
00186 
00187         Another subclass of BoundToken that combines both a Tokenizer and
00188         an input buffer. This is simply a convenience wrapper than takes
00189         care of details that would otherwise clutter the client code.
00190 
00191         Compare this to usage of a basic Token:
00192 
00193         @code
00194         // open a file for reading
00195         FileConduit fc = new FileConduit ("test.txt");
00196 
00197         // create a Token and bind it to both the file and a line-tokenizer
00198         CompositeToken line = new CompositeToken (Tokenizers.line, fc);
00199 
00200         // read file a line at a time. Method get() returns false when no more 
00201         // tokens are found. 
00202         while (line.get)
00203                Stdout.put(line).cr();
00204         @endcode
00205 
00206         You might also consider a HybridToken for further processing of
00207         token content.
00208 
00209 *******************************************************************************/
00210 
00211 class CompositeToken : BoundToken
00212 {       
00213         private IBuffer buffer;
00214 
00215         /***********************************************************************
00216         
00217                 Set this token to use the provided Tokenizer, and bind it 
00218                 to the given buffer.
00219 
00220         ***********************************************************************/
00221 
00222         this (ITokenizer tk, IBuffer buffer)
00223         {
00224                 super (tk);
00225                 this.buffer = buffer;
00226         }
00227 
00228         /***********************************************************************
00229         
00230                 Set this token to use the provided Tokenizer, and bind it 
00231                 to the buffer associated with the given conduit.
00232 
00233         ***********************************************************************/
00234 
00235         this (ITokenizer tk, IConduit conduit)
00236         {
00237                 this (tk, conduit.createBuffer());
00238         }
00239 
00240         /***********************************************************************
00241         
00242                 Return the associated buffer
00243 
00244         ***********************************************************************/
00245 
00246         IBuffer getBuffer ()
00247         {     
00248                 return buffer;
00249         }
00250 
00251         /***********************************************************************
00252 
00253                 Extract the next token. 
00254 
00255                 Returns true if a token was isolated, false if no more 
00256                 tokens were found. Note that one last token may still
00257                 be present when this return false; this may happen if
00258                 (for example) the last delimiter is missing before an
00259                 Eof condition is seen. Check token.getLength() when
00260                 this method returns false.
00261                 
00262                 For example:
00263 
00264                 @code
00265                         while (token.next || token.getLength)
00266                                // do something
00267 
00268                 @endcode               
00269 
00270         ***********************************************************************/
00271 
00272         bool next ()
00273         {
00274                 return tk.next (buffer, this);
00275         }
00276 
00277         /***********************************************************************
00278 
00279                 Extract the next token, taking Eof into consideration.
00280                 If next() returns false, then this function will still
00281                 return true as long as there's some content available.
00282 
00283                 For example:
00284 
00285                 @code
00286                         while (token.get)
00287                                // do something
00288 
00289                 @endcode               
00290 
00291         ***********************************************************************/
00292 
00293         bool get ()
00294         {
00295                 return next() || getLength();
00296         }
00297 }
00298 
00299 
00300 /*******************************************************************************
00301 
00302         A subclass of CompositeToken that combines a Tokenizer, an input buffer,
00303         and the means to bind its content to a subordinate Reader or Token. 
00304         This is another convenience wrapper than takes care of details that
00305         would otherwise complicate client code.
00306 
00307         Compare this to usage of a CompositeToken:
00308 
00309         @code
00310         // open a file for reading
00311         FileConduit fc = new FileConduit ("test.txt");
00312 
00313         // create a Token and bind it to both the file and a line-tokenizer
00314         HybridToken line = new HybridToken (Tokenizers.line, fc);
00315 
00316         // now create a reader upon the token
00317         Reader reader = new Reader (line.getHost);
00318 
00319         // read file a line at a time. Method get() returns false when no more 
00320         // tokens are found. 
00321         while (line.get)
00322               {
00323               int x, y;
00324                 
00325               // reader is now bound to the content of the current line
00326               reader.get(x).get(y);
00327 
00328               Stdout.put(x).put(y).cr();
00329               }
00330         @endcode
00331 
00332         You can use the same mechanism to bind subordinate Tokens:
00333 
00334         @code
00335         // open a file for reading
00336         FileConduit fc = new FileConduit ("test.txt");
00337 
00338         // create a Token and bind it to both the file and a line-tokenizer
00339         HybridToken line = new HybridToken (Tokenizers.line, fc);
00340 
00341         // now create a subordinate Token that splits on whitespace
00342         CompositeToken word = new CompositeToken (Tokenizers.space, line.getHost);
00343 
00344         // read file a line at a time. Method get() returns false when no more 
00345         // tokens are found. 
00346         while (line.get)
00347                // extract space delimited tokens from each line
00348                while (word.get)
00349                       Stdout.put(word).cr();
00350         @endcode
00351 
00352 
00353 *******************************************************************************/
00354 
00355 class HybridToken : CompositeToken
00356 {       
00357         private IBuffer host;
00358 
00359         /***********************************************************************
00360         
00361                 Set this token to use the provided Tokenizer, and bind it 
00362                 to the given buffer.
00363 
00364         ***********************************************************************/
00365 
00366         this (ITokenizer tk, IBuffer buffer)
00367         {
00368                 super (tk, buffer);
00369 
00370                 // create the hosting IBuffer
00371                 host = buffer.create();
00372         }
00373 
00374         /***********************************************************************
00375         
00376                 Set this token to use the provided Tokenizer, and bind it 
00377                 to the buffer associated with the given conduit.
00378 
00379         ***********************************************************************/
00380 
00381         this (ITokenizer tk, IConduit conduit)
00382         {
00383                 this (tk, conduit.createBuffer());
00384         }
00385 
00386         /***********************************************************************
00387         
00388                 Return the associated host buffer. The host should be used
00389                 for purposes of binding a subordinate Token or Reader onto
00390                 the content of this token. Each call to next() will update
00391                 this content appropriately, which is also reflected within 
00392                 said host buffer.
00393 
00394                 That is, token.toString() == token.getHost.toString().
00395 
00396         ***********************************************************************/
00397 
00398         IBuffer getHost ()
00399         {     
00400                 return host;
00401         }
00402 
00403         /***********************************************************************
00404 
00405                 Extract the next token. 
00406 
00407                 Returns true if a token was isolated, false if no more 
00408                 tokens were found. Note that one last token may still
00409                 be present when this return false; this may happen if
00410                 (for example) the last delimiter is missing before an
00411                 Eof condition is seen. Check token.getLength() when
00412                 this method returns false.
00413                 
00414                 For example:
00415 
00416                 @code
00417                         while (token.next || token.getLength)
00418                                // do something
00419 
00420                 @endcode               
00421 
00422         ***********************************************************************/
00423 
00424         bool next ()
00425         {
00426                 // get the next token
00427                 bool ret = super.next ();
00428 
00429                 // set host content
00430                 host.setValidContent (toString());
00431 
00432                 return ret;
00433         }
00434 }

Generated on Sun Nov 7 19:06:53 2004 for Mango by doxygen 1.3.6