00001 /******************************************************************************* 00002 00003 @file FileBucket.d 00004 00005 Copyright (c) 2004 Kris Bell 00006 00007 This software is provided 'as-is', without any express or implied 00008 warranty. In no event will the authors be held liable for damages 00009 of any kind arising from the use of this software. 00010 00011 Permission is hereby granted to anyone to use this software for any 00012 purpose, including commercial applications, and to alter it and/or 00013 redistribute it freely, subject to the following restrictions: 00014 00015 1. The origin of this software must not be misrepresented; you must 00016 not claim that you wrote the original software. If you use this 00017 software in a product, an acknowledgment within documentation of 00018 said product would be appreciated but is not required. 00019 00020 2. Altered source versions must be plainly marked as such, and must 00021 not be misrepresented as being the original software. 00022 00023 3. This notice may not be removed or altered from any distribution 00024 of the source. 00025 00026 4. Derivative works are permitted, but they must carry this notice 00027 in full and credit the original source. 00028 of the source. 00029 00030 00031 @version Initial version, April 2004 00032 @author Kris 00033 00034 00035 *******************************************************************************/ 00036 00037 00038 module mango.io.FileBucket; 00039 00040 public import mango.io.FilePath; 00041 00042 private import mango.io.Buffer, 00043 mango.io.FileStyle, 00044 mango.io.Exception, 00045 mango.io.FileConduit; 00046 00047 private import mango.io.model.IBitBucket; 00048 00049 /****************************************************************************** 00050 00051 FileBucket implements a simple mechanism to store and recover a 00052 large quantity of data for the duration of the hosting process. 00053 It is intended to act as a local-cache for a remote data-source, 00054 or as a spillover area for large in-memory cache instances. 00055 00056 Note that any and all stored data is rendered invalid the moment 00057 a FileBucket object is garbage-collected. 00058 00059 The implementation follows a fixed-capacity record scheme, where 00060 content can be rewritten in-place until said capacity is reached. 00061 At such time, the altered content is moved to a larger capacity 00062 record at end-of-file, and a hole remains at the prior location. 00063 These holes are not collected, since the lifespan of a FileBucket 00064 is limited to that of the host process. 00065 00066 All index keys must be unique. Writing to the FileBucket with an 00067 existing key will overwrite any previous content. What follows 00068 is a contrived example: 00069 00070 @code 00071 char[] text = "this is a test"; 00072 00073 FileBucket bucket = new FileBucket (new FilePath("bucket.bin"), FileBucket.HalfK); 00074 00075 // insert some data, and retrieve it again 00076 bucket.put ("a key", text); 00077 char[] b = cast(char[]) bucket.get ("a key"); 00078 00079 assert (b == text); 00080 bucket.close(); 00081 @endcode 00082 00083 ******************************************************************************/ 00084 00085 class FileBucket : IBitBucket 00086 { 00087 /********************************************************************** 00088 00089 Define the capacity (block-size) of each record 00090 00091 **********************************************************************/ 00092 00093 struct BlockSize 00094 { 00095 int capacity; 00096 } 00097 00098 // basic capacity for each record 00099 private FilePath path; 00100 00101 // basic capacity for each record 00102 private BlockSize block; 00103 00104 // where content is stored 00105 private FileConduit file; 00106 00107 // pointers to file records 00108 private Record[char[]] map; 00109 00110 // place for reading and writing from/into 00111 private Buffer buffer; 00112 00113 // current file size 00114 private long fileSize; 00115 00116 // current file usage 00117 private long waterLine; 00118 00119 // supported block sizes 00120 static const BlockSize EighthK = {128-1}, 00121 HalfK = {512-1}, 00122 OneK = {1024*1-1}, 00123 TwoK = {1024*2-1}, 00124 FourK = {1024*4-1}, 00125 EightK = {1024*8-1}, 00126 SixteenK = {1024*16-1}, 00127 ThirtyTwoK = {1024*32-1}, 00128 SixtyFourK = {1024*64-1}; 00129 00130 /********************************************************************** 00131 00132 Construct a FileBucket with the provided path and record- 00133 size. Selecting a record size that roughly matches the 00134 serialized content will limit 'thrashing'. 00135 00136 **********************************************************************/ 00137 00138 this (FilePath path, BlockSize block) 00139 { 00140 this (path, block, 100); 00141 } 00142 00143 /********************************************************************** 00144 00145 Construct a FileBucket with the provided path, record-size, 00146 and inital record count. The latter causes records to be 00147 pre-allocated, saving a certain amount of growth activity. 00148 Selecting a record size that roughly matches the serialized 00149 content will limit 'thrashing'. 00150 00151 **********************************************************************/ 00152 00153 this (FilePath path, BlockSize block, uint initialRecords) 00154 { 00155 this.path = path; 00156 this.block = block; 00157 00158 // open a storage file 00159 file = new FileConduit (path, FileStyle.ReadWriteCreate); 00160 00161 // set initial file size (can be zero) 00162 fileSize = initialRecords * block.capacity; 00163 file.seek (fileSize); 00164 file.truncate (); 00165 00166 // bind a buffer to the storage file, but don't allocate 00167 // memory since we'll explicitly set that as appropriate 00168 buffer = new Buffer; 00169 buffer.setConduit (file); 00170 } 00171 00172 /********************************************************************** 00173 00174 Destructor tries to close the file, if not already done. 00175 00176 **********************************************************************/ 00177 00178 ~this() 00179 { 00180 close (); 00181 } 00182 00183 /********************************************************************** 00184 00185 Return the block-size in use for this FileBucket 00186 00187 **********************************************************************/ 00188 00189 int getBufferSize () 00190 { 00191 return block.capacity+1; 00192 } 00193 00194 /********************************************************************** 00195 00196 Return where the FileBucket is located 00197 00198 **********************************************************************/ 00199 00200 FilePath getFilePath () 00201 { 00202 return path; 00203 } 00204 00205 /********************************************************************** 00206 00207 Return the currently populated size of this FileBucket 00208 00209 **********************************************************************/ 00210 00211 synchronized long length () 00212 { 00213 return waterLine; 00214 } 00215 00216 /********************************************************************** 00217 00218 Return the serialized data for the provided key. Returns 00219 null if the key was not found. 00220 00221 **********************************************************************/ 00222 00223 synchronized void[] get (char[] key) 00224 { 00225 Record r = null; 00226 00227 if (key in map) 00228 { 00229 r = map [key]; 00230 return r.read (this); 00231 } 00232 return null; 00233 } 00234 00235 /********************************************************************** 00236 00237 Remove the provided key from this FileBucket. 00238 00239 **********************************************************************/ 00240 00241 synchronized void remove (char[] key) 00242 { 00243 map.remove(key); 00244 } 00245 00246 /********************************************************************** 00247 00248 Write a serialized block of data, and associate it with 00249 the provided key. All keys must be unique, and it is the 00250 responsibility of the programmer to ensure this. Reusing 00251 an existing key will overwrite previous data. 00252 00253 Note that data is allowed to grow within the occupied 00254 bucket until it becomes larger than the allocated space. 00255 When this happens, the data is moved to a larger bucket 00256 at the file tail. 00257 00258 **********************************************************************/ 00259 00260 synchronized void put (char[] key, void[] data) 00261 { 00262 Record* r = key in map; 00263 00264 if (r is null) 00265 { 00266 Record rr = new Record (); 00267 map [key] = rr; 00268 r = &rr; 00269 } 00270 r.write (this, data, block); 00271 } 00272 00273 /********************************************************************** 00274 00275 Close this FileBucket -- all content is lost. 00276 00277 **********************************************************************/ 00278 00279 synchronized void close () 00280 { 00281 if (file) 00282 { 00283 file.close (); 00284 file = null; 00285 map = null; 00286 } 00287 } 00288 00289 /********************************************************************** 00290 00291 Each Record takes up a number of 'pages' within the file. 00292 The size of these pages is determined by the BlockSize 00293 provided during FileBucket construction. Additional space 00294 at the end of each block is potentially wasted, but enables 00295 content to grow in size without creating a myriad of holes. 00296 00297 **********************************************************************/ 00298 00299 private class Record 00300 { 00301 private long offset; 00302 private int length, 00303 capacity = -1; 00304 00305 /************************************************************** 00306 00307 **************************************************************/ 00308 00309 private static void eof (FileBucket bucket) 00310 { 00311 throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.toString()~"'"); 00312 } 00313 00314 /************************************************************** 00315 00316 This should be protected from thread-contention at 00317 a higher level. 00318 00319 **************************************************************/ 00320 00321 void[] read (FileBucket bucket) 00322 { 00323 void[] data = new void [length]; 00324 bucket.buffer.setContent (data, 0); 00325 00326 bucket.file.seek (offset); 00327 if (bucket.file.read (bucket.buffer) != length) 00328 eof (bucket); 00329 00330 return data; 00331 } 00332 00333 /************************************************************** 00334 00335 This should be protected from thread-contention at 00336 a higher level. 00337 00338 **************************************************************/ 00339 00340 void write (FileBucket bucket, void[] data, BlockSize block) 00341 { 00342 length = data.length; 00343 00344 // create new slot if we exceed capacity 00345 if (length > capacity) 00346 createBucket (bucket, length, block); 00347 00348 // locate to start of content 00349 bucket.file.seek (offset); 00350 00351 // write content 00352 bucket.buffer.setValidContent (data); 00353 bucket.buffer.flush (); 00354 } 00355 00356 /************************************************************** 00357 00358 **************************************************************/ 00359 00360 void createBucket (FileBucket bucket, int bytes, BlockSize block) 00361 { 00362 offset = bucket.waterLine; 00363 capacity = (bytes + block.capacity) & ~block.capacity; 00364 00365 bucket.waterLine += capacity; 00366 if (bucket.waterLine > bucket.fileSize) 00367 { 00368 // grow the filesize 00369 bucket.fileSize = bucket.waterLine * 2; 00370 00371 // expand the physical file size 00372 bucket.file.seek (bucket.fileSize); 00373 bucket.file.truncate (); 00374 } 00375 } 00376 } 00377 } 00378 00379