00001 /******************************************************************************* 00002 00003 @file TokenEx.d 00004 00005 Copyright (C) 2004 Kris Bell 00006 00007 This software is provided 'as-is', without any express or implied 00008 warranty. In no event will the authors be held liable for damages 00009 of any kind arising from the use of this software. 00010 00011 Permission is hereby granted to anyone to use this software for any 00012 purpose, including commercial applications, and to alter it and/or 00013 redistribute it freely, subject to the following restrictions: 00014 00015 1. The origin of this software must not be misrepresented; you must 00016 not claim that you wrote the original software. If you use this 00017 software in a product, an acknowledgment within documentation of 00018 said product would be appreciated but is not required. 00019 00020 2. Altered source versions must be plainly marked as such, and must 00021 not be misrepresented as being the original software. 00022 00023 3. This notice may not be removed or altered from any distribution 00024 of the source. 00025 00026 00027 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 00028 00029 00030 @version Initial version, Oct 2004 00031 @author Kris 00032 00033 00034 *******************************************************************************/ 00035 00036 module mango.io.TokenEx; 00037 00038 private import mango.io.Token, 00039 mango.io.Tokenizer; 00040 00041 private import mango.io.model.IReader, 00042 mango.io.model.IConduit; 00043 00044 00045 /******************************************************************************* 00046 00047 A style of Token that's bound to a Tokenizer. This can be a handy 00048 means of cleaning up client code, and limiting the scope of how 00049 a token is used by recieving methods. 00050 00051 Contrast this example with that shown in the Token class: 00052 00053 @code 00054 // open a file for reading 00055 FileConduit fc = new FileConduit ("test.txt"); 00056 00057 // create a buffer for reading the file 00058 IBuffer buffer = fc.createBuffer(); 00059 00060 // bind a line-tokenizer to our input token 00061 BoundToken line = new BoundToken (Tokenizers.line); 00062 00063 // read file a line at a time. Method next() returns false when no more 00064 // delimiters are found. Note there may be an unterminated line at eof 00065 while (line.next(buffer) || line.getLength()) 00066 Stdout.put(line).cr(); 00067 @endcode 00068 00069 One might also consider a CompositeToken or HybridToken. 00070 00071 *******************************************************************************/ 00072 00073 class BoundToken : Token 00074 { 00075 private ITokenizer tk; 00076 00077 /*********************************************************************** 00078 00079 ***********************************************************************/ 00080 00081 this (ITokenizer tk) 00082 { 00083 this.tk = tk; 00084 } 00085 00086 /*********************************************************************** 00087 00088 Return the associated tokenizer 00089 00090 ***********************************************************************/ 00091 00092 ITokenizer getTokenizer () 00093 { 00094 return tk; 00095 } 00096 00097 /*********************************************************************** 00098 00099 Extract the next token from the provided buffer. 00100 00101 Returns true if a token was isolated, false if no more 00102 tokens were found. Note that one last token may still 00103 be present when this return false; this may happen if 00104 (for example) the last delimiter is missing before an 00105 EOF condition is seen. Check token.getLength() when 00106 this method returns false. 00107 00108 For example: 00109 00110 @code 00111 while (token.next() || token.getLength()) 00112 // do something 00113 00114 @endcode 00115 00116 ***********************************************************************/ 00117 00118 bool next (IBuffer buf) 00119 { 00120 return tk.next (buf, this); 00121 } 00122 } 00123 00124 00125 /******************************************************************************* 00126 00127 ReaderToken adapts a BoundToken such that it can be used directly 00128 with any IReader implementation. We just add the IReadable methods 00129 to the basic BoundToken. 00130 00131 Here's a contrived example of how to use ReaderToken: 00132 00133 @code 00134 // create a small buffer on the heap 00135 Buffer buf = new Buffer (256); 00136 00137 // write items with a comma between each 00138 TextWriter tw = new TextWriter (buf, ","); 00139 00140 // write some stuff to the buffer 00141 tw << "now is the time for all good men" << 3.14159; 00142 00143 // bind a couple of tokens to a comma tokenizer 00144 ReaderToken text = new ReaderToken (Tokenizers.comma); 00145 ReaderToken number = new ReaderToken (Tokenizers.comma); 00146 00147 // create any old reader since we only use it for handling tokens 00148 Reader r = new Reader (buf); 00149 00150 // populate both tokens via reader 00151 r >> text >> number; 00152 00153 // print them to the console 00154 Stdout << text << ':' << number << Stdout.newline; 00155 @endcode 00156 00157 *******************************************************************************/ 00158 00159 class ReaderToken : BoundToken, IReadable 00160 { 00161 /*********************************************************************** 00162 00163 Construct a ReaderToken using the provided Tokenizer. 00164 00165 ***********************************************************************/ 00166 00167 this (ITokenizer tk) 00168 { 00169 super (tk); 00170 } 00171 00172 /*********************************************************************** 00173 00174 Read the next delimited element into this token. 00175 00176 ***********************************************************************/ 00177 00178 void read (IReader r) 00179 { 00180 tk.next (r.getBuffer(), this); 00181 } 00182 } 00183 00184 00185 /******************************************************************************* 00186 00187 Another subclass of BoundToken that combines both a Tokenizer and 00188 an input buffer. This is simply a convenience wrapper than takes 00189 care of details that would otherwise clutter the client code. 00190 00191 Compare this to usage of a basic Token: 00192 00193 @code 00194 // open a file for reading 00195 FileConduit fc = new FileConduit ("test.txt"); 00196 00197 // create a Token and bind it to both the file and a line-tokenizer 00198 CompositeToken line = new CompositeToken (Tokenizers.line, fc); 00199 00200 // read file a line at a time. Method get() returns false when no more 00201 // tokens are found. 00202 while (line.get) 00203 Stdout.put(line).cr(); 00204 @endcode 00205 00206 You might also consider a HybridToken for further processing of 00207 token content. 00208 00209 *******************************************************************************/ 00210 00211 class CompositeToken : BoundToken 00212 { 00213 private IBuffer buffer; 00214 00215 /*********************************************************************** 00216 00217 Set this token to use the provided Tokenizer, and bind it 00218 to the given buffer. 00219 00220 ***********************************************************************/ 00221 00222 this (ITokenizer tk, IBuffer buffer) 00223 { 00224 super (tk); 00225 this.buffer = buffer; 00226 } 00227 00228 /*********************************************************************** 00229 00230 Set this token to use the provided Tokenizer, and bind it 00231 to the buffer associated with the given conduit. 00232 00233 ***********************************************************************/ 00234 00235 this (ITokenizer tk, IConduit conduit) 00236 { 00237 this (tk, conduit.createBuffer()); 00238 } 00239 00240 /*********************************************************************** 00241 00242 Return the associated buffer 00243 00244 ***********************************************************************/ 00245 00246 IBuffer getBuffer () 00247 { 00248 return buffer; 00249 } 00250 00251 /*********************************************************************** 00252 00253 Extract the next token. 00254 00255 Returns true if a token was isolated, false if no more 00256 tokens were found. Note that one last token may still 00257 be present when this return false; this may happen if 00258 (for example) the last delimiter is missing before an 00259 Eof condition is seen. Check token.getLength() when 00260 this method returns false. 00261 00262 For example: 00263 00264 @code 00265 while (token.next || token.getLength) 00266 // do something 00267 00268 @endcode 00269 00270 ***********************************************************************/ 00271 00272 bool next () 00273 { 00274 return tk.next (buffer, this); 00275 } 00276 00277 /*********************************************************************** 00278 00279 Extract the next token, taking Eof into consideration. 00280 If next() returns false, then this function will still 00281 return true as long as there's some content available. 00282 00283 For example: 00284 00285 @code 00286 while (token.get) 00287 // do something 00288 00289 @endcode 00290 00291 ***********************************************************************/ 00292 00293 bool get () 00294 { 00295 return next() || getLength(); 00296 } 00297 } 00298 00299 00300 /******************************************************************************* 00301 00302 A subclass of CompositeToken that combines a Tokenizer, an input buffer, 00303 and the means to bind its content to a subordinate Reader or Token. 00304 This is another convenience wrapper than takes care of details that 00305 would otherwise complicate client code. 00306 00307 Compare this to usage of a CompositeToken: 00308 00309 @code 00310 // open a file for reading 00311 FileConduit fc = new FileConduit ("test.txt"); 00312 00313 // create a Token and bind it to both the file and a line-tokenizer 00314 HybridToken line = new HybridToken (Tokenizers.line, fc); 00315 00316 // now create a reader upon the token 00317 Reader reader = new Reader (line.getHost); 00318 00319 // read file a line at a time. Method get() returns false when no more 00320 // tokens are found. 00321 while (line.get) 00322 { 00323 int x, y; 00324 00325 // reader is now bound to the content of the current line 00326 reader.get(x).get(y); 00327 00328 Stdout.put(x).put(y).cr(); 00329 } 00330 @endcode 00331 00332 You can use the same mechanism to bind subordinate Tokens: 00333 00334 @code 00335 // open a file for reading 00336 FileConduit fc = new FileConduit ("test.txt"); 00337 00338 // create a Token and bind it to both the file and a line-tokenizer 00339 HybridToken line = new HybridToken (Tokenizers.line, fc); 00340 00341 // now create a subordinate Token that splits on whitespace 00342 CompositeToken word = new CompositeToken (Tokenizers.space, line.getHost); 00343 00344 // read file a line at a time. Method get() returns false when no more 00345 // tokens are found. 00346 while (line.get) 00347 // extract space delimited tokens from each line 00348 while (word.get) 00349 Stdout.put(word).cr(); 00350 @endcode 00351 00352 00353 *******************************************************************************/ 00354 00355 class HybridToken : CompositeToken 00356 { 00357 private IBuffer host; 00358 00359 /*********************************************************************** 00360 00361 Set this token to use the provided Tokenizer, and bind it 00362 to the given buffer. 00363 00364 ***********************************************************************/ 00365 00366 this (ITokenizer tk, IBuffer buffer) 00367 { 00368 super (tk, buffer); 00369 00370 // create the hosting IBuffer 00371 host = buffer.create(); 00372 } 00373 00374 /*********************************************************************** 00375 00376 Set this token to use the provided Tokenizer, and bind it 00377 to the buffer associated with the given conduit. 00378 00379 ***********************************************************************/ 00380 00381 this (ITokenizer tk, IConduit conduit) 00382 { 00383 this (tk, conduit.createBuffer()); 00384 } 00385 00386 /*********************************************************************** 00387 00388 Return the associated host buffer. The host should be used 00389 for purposes of binding a subordinate Token or Reader onto 00390 the content of this token. Each call to next() will update 00391 this content appropriately, which is also reflected within 00392 said host buffer. 00393 00394 That is, token.toString() == token.getHost.toString(). 00395 00396 ***********************************************************************/ 00397 00398 IBuffer getHost () 00399 { 00400 return host; 00401 } 00402 00403 /*********************************************************************** 00404 00405 Extract the next token. 00406 00407 Returns true if a token was isolated, false if no more 00408 tokens were found. Note that one last token may still 00409 be present when this return false; this may happen if 00410 (for example) the last delimiter is missing before an 00411 Eof condition is seen. Check token.getLength() when 00412 this method returns false. 00413 00414 For example: 00415 00416 @code 00417 while (token.next || token.getLength) 00418 // do something 00419 00420 @endcode 00421 00422 ***********************************************************************/ 00423 00424 bool next () 00425 { 00426 // get the next token 00427 bool ret = super.next (); 00428 00429 // set host content 00430 host.setValidContent (toString()); 00431 00432 return ret; 00433 } 00434 }