00001 /******************************************************************************* 00002 00003 @file BufferTokenizer.d 00004 00005 Copyright (c) 2004 Kris Bell 00006 00007 This software is provided 'as-is', without any express or implied 00008 warranty. In no event will the authors be held liable for damages 00009 of any kind arising from the use of this software. 00010 00011 Permission is hereby granted to anyone to use this software for any 00012 purpose, including commercial applications, and to alter it and/or 00013 redistribute it freely, subject to the following restrictions: 00014 00015 1. The origin of this software must not be misrepresented; you must 00016 not claim that you wrote the original software. If you use this 00017 software in a product, an acknowledgment within documentation of 00018 said product would be appreciated but is not required. 00019 00020 2. Altered source versions must be plainly marked as such, and must 00021 not be misrepresented as being the original software. 00022 00023 3. This notice may not be removed or altered from any distribution 00024 of the source. 00025 00026 4. Derivative works are permitted, but they must carry this notice 00027 in full and credit the original source. 00028 00029 00030 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 00031 00032 00033 @version Initial version, December 2005 00034 00035 @author Kris 00036 00037 00038 *******************************************************************************/ 00039 00040 module mango.io.BufferTokenizer; 00041 00042 private import mango.text.Token; 00043 00044 private import mango.io.Buffer; 00045 /+ 00046 private import mango.io.model.IBuffer, 00047 mango.io.model.IConduit; 00048 +/ 00049 /******************************************************************************* 00050 00051 Tokenize input from a buffer or conduit 00052 00053 All input is tokenized from the associated buffer, and exposed as a 00054 freachable property via method opApply(). Empty token are passed to 00055 the caller ~ this may happen if, for example, an empty line is seen 00056 by a line-tokenizer. 00057 00058 *******************************************************************************/ 00059 00060 class BufferTokenizerTemplate(T) 00061 { 00062 private TokenTemplate!(T) token; 00063 private IBuffer buffer; 00064 00065 /*********************************************************************** 00066 00067 Construct a BufferTokenizer on the provided buffer, using 00068 the specified Tokenizer instead of the default one. 00069 00070 ***********************************************************************/ 00071 00072 this (IBuffer buffer, TokenTemplate!(T) token) 00073 { 00074 this.token = token; 00075 this.buffer = buffer; 00076 token.setRefill (&refill); 00077 } 00078 00079 /*********************************************************************** 00080 00081 Construct a BufferTokenizer upon the buffer associated with 00082 the given conduit. 00083 00084 ***********************************************************************/ 00085 00086 this (IConduit conduit, TokenTemplate!(T) token) 00087 { 00088 this (new Buffer(conduit), token); 00089 } 00090 00091 /********************************************************************** 00092 00093 Iterate over the set of tokens 00094 00095 **********************************************************************/ 00096 00097 int opApply (int delegate(inout T[]) dg) 00098 { 00099 return token.opApply (dg); 00100 } 00101 00102 /*********************************************************************** 00103 00104 Refill the token content from our buffer. Returns false 00105 upon reaching EOF 00106 00107 ***********************************************************************/ 00108 00109 private bool refill (TokenTemplate!(T) token) 00110 { 00111 bool more; 00112 00113 buffer.skip (token.eaten - buffer.getPosition); 00114 if (buffer.getConduit) 00115 more = cast(bool) (buffer.fill() != IConduit.Eof); 00116 else 00117 more = cast(bool) (buffer.readable > 0); 00118 token.prime (buffer.toString); 00119 return more; 00120 } 00121 } 00122 00123 00124 // convenience alias 00125 alias BufferTokenizerTemplate!(char) BufferTokenizer;