Main Page | Class Hierarchy | Alphabetical List | Class List | Directories | File List | Class Members | File Members | Related Pages

BufferTokenizer.d

Go to the documentation of this file.
00001 /*******************************************************************************
00002 
00003         @file BufferTokenizer.d
00004         
00005         Copyright (c) 2004 Kris Bell
00006         
00007         This software is provided 'as-is', without any express or implied
00008         warranty. In no event will the authors be held liable for damages
00009         of any kind arising from the use of this software.
00010         
00011         Permission is hereby granted to anyone to use this software for any 
00012         purpose, including commercial applications, and to alter it and/or 
00013         redistribute it freely, subject to the following restrictions:
00014         
00015         1. The origin of this software must not be misrepresented; you must 
00016            not claim that you wrote the original software. If you use this 
00017            software in a product, an acknowledgment within documentation of 
00018            said product would be appreciated but is not required.
00019 
00020         2. Altered source versions must be plainly marked as such, and must 
00021            not be misrepresented as being the original software.
00022 
00023         3. This notice may not be removed or altered from any distribution
00024            of the source.
00025 
00026         4. Derivative works are permitted, but they must carry this notice
00027            in full and credit the original source.
00028 
00029 
00030                         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
00031 
00032 
00033         @version        Initial version, December 2005
00034               
00035         @author         Kris
00036 
00037 
00038 *******************************************************************************/
00039 
00040 module mango.io.BufferTokenizer;
00041 
00042 private import  mango.text.Token;
00043 
00044 private import  mango.io.Buffer;
00045 /+
00046 private import  mango.io.model.IBuffer,
00047                 mango.io.model.IConduit;
00048 +/
00049 /*******************************************************************************
00050 
00051         Tokenize input from a buffer or conduit
00052         
00053         All input is tokenized from the associated buffer, and exposed as a 
00054         freachable property via method opApply(). Empty token are passed to
00055         the caller ~ this may happen if, for example, an empty line is seen
00056         by a line-tokenizer.
00057 
00058 *******************************************************************************/
00059 
00060 class BufferTokenizerTemplate(T)
00061 {
00062         private TokenTemplate!(T)       token;
00063         private IBuffer                 buffer;
00064 
00065         /***********************************************************************
00066         
00067                 Construct a BufferTokenizer on the provided buffer, using 
00068                 the specified Tokenizer instead of the default one.
00069 
00070         ***********************************************************************/
00071 
00072         this (IBuffer buffer, TokenTemplate!(T) token)
00073         {
00074                 this.token = token;
00075                 this.buffer = buffer;
00076                 token.setRefill (&refill);
00077         }
00078         
00079         /***********************************************************************
00080                 
00081                 Construct a BufferTokenizer upon the buffer associated with 
00082                 the given conduit.
00083 
00084         ***********************************************************************/
00085 
00086         this (IConduit conduit, TokenTemplate!(T) token)
00087         {
00088                 this (new Buffer(conduit), token);
00089         }
00090 
00091         /**********************************************************************
00092 
00093                 Iterate over the set of tokens
00094 
00095         **********************************************************************/
00096 
00097         int opApply (int delegate(inout T[]) dg)
00098         {
00099                 return token.opApply (dg);
00100         }
00101 
00102         /***********************************************************************
00103  
00104                 Refill the token content from our buffer. Returns false 
00105                 upon reaching EOF
00106 
00107         ***********************************************************************/
00108 
00109         private bool refill (TokenTemplate!(T) token)
00110         {
00111                 bool more;
00112 
00113                 buffer.skip (token.eaten - buffer.getPosition);
00114                 if (buffer.getConduit)
00115                     more = cast(bool) (buffer.fill() != IConduit.Eof);
00116                 else
00117                    more = cast(bool) (buffer.readable > 0);
00118                 token.prime (buffer.toString);
00119                 return  more;
00120         }
00121 }
00122 
00123 
00124 // convenience alias
00125 alias BufferTokenizerTemplate!(char) BufferTokenizer;

Generated on Sat Dec 24 17:28:31 2005 for Mango by  doxygen 1.4.0