00001 /******************************************************************************* 00002 00003 @file FileBucket.d 00004 00005 Copyright (C) 2004 Kris Bell 00006 00007 00008 This software is provided 'as-is', without any express or implied 00009 warranty. In no event will the authors be held liable for damages 00010 of any kind arising from the use of this software. 00011 00012 Permission is hereby granted to anyone to use this software for any 00013 purpose, including commercial applications, and to alter it and/or 00014 redistribute it freely, subject to the following restrictions: 00015 00016 1. The origin of this software must not be misrepresented; you must 00017 not claim that you wrote the original software. If you use this 00018 software in a product, an acknowledgment within documentation of 00019 said product would be appreciated but is not required. 00020 00021 2. Altered source versions must be plainly marked as such, and must 00022 not be misrepresented as being the original software. 00023 00024 3. This notice may not be removed or altered from any distribution 00025 of the source. 00026 00027 00028 @version Initial version, April 2004 00029 @author Kris 00030 00031 00032 *******************************************************************************/ 00033 00034 00035 module mango.io.FileBucket; 00036 00037 public import mango.io.FilePath; 00038 00039 private import mango.io.Buffer, 00040 mango.io.FileStyle, 00041 mango.io.Exception, 00042 mango.io.FileConduit; 00043 00044 private import mango.io.model.IBitBucket; 00045 00046 /****************************************************************************** 00047 00048 FileBucket implements a simple mechanism to store and recover a 00049 large quantity of data for the duration of the hosting process. 00050 It is intended to act as a local-cache for a remote data-source, 00051 or as a spillover area for large in-memory cache instances. 00052 00053 Note that any and all stored data is rendered invalid the moment 00054 a FileBucket object is garbage-collected. 00055 00056 The implementation follows a fixed-capacity record scheme, where 00057 content can be rewritten in-place until said capacity is reached. 00058 At such time, the altered content is moved to a larger capacity 00059 record at end-of-file, and a hole remains at the prior location. 00060 These holes are not collected, since the lifespan of a FileBucket 00061 is limited to that of the host process. 00062 00063 All index keys must be unique. Writing to the FileBucket with an 00064 existing key will overwrite any previous content. What follows 00065 is a contrived example: 00066 00067 @code 00068 char[] text = "this is a test"; 00069 00070 FileBucket bucket = new FileBucket (new FilePath("bucket.bin"), FileBucket.HalfK); 00071 00072 // insert some data, and retrieve it again 00073 bucket.put ("a key", text); 00074 char[] b = cast(char[]) bucket.get ("a key"); 00075 00076 assert (b == text); 00077 bucket.close(); 00078 @endcode 00079 00080 ******************************************************************************/ 00081 00082 class FileBucket : IBitBucket 00083 { 00084 /********************************************************************** 00085 00086 Define the capacity (block-size) of each record 00087 00088 **********************************************************************/ 00089 00090 struct BlockSize 00091 { 00092 int capacity; 00093 } 00094 00095 // basic capacity for each record 00096 private FilePath path; 00097 00098 // basic capacity for each record 00099 private BlockSize block; 00100 00101 // where content is stored 00102 private FileConduit file; 00103 00104 // pointers to file records 00105 private Record[char[]] map; 00106 00107 // place for reading and writing from/into 00108 private Buffer buffer; 00109 00110 // current file size 00111 private long fileSize; 00112 00113 // current file usage 00114 private long waterLine; 00115 00116 // supported block sizes 00117 static const BlockSize EighthK = {128-1}, 00118 HalfK = {512-1}, 00119 OneK = {1024*1-1}, 00120 TwoK = {1024*2-1}, 00121 FourK = {1024*4-1}, 00122 EightK = {1024*8-1}, 00123 SixteenK = {1024*16-1}, 00124 ThirtyTwoK = {1024*32-1}, 00125 SixtyFourK = {1024*64-1}; 00126 00127 /********************************************************************** 00128 00129 Construct a FileBucket with the provided path and record- 00130 size. Selecting a record size that roughly matches the 00131 serialized content will limit 'thrashing'. 00132 00133 **********************************************************************/ 00134 00135 this (FilePath path, BlockSize block) 00136 { 00137 this (path, block, 100); 00138 } 00139 00140 /********************************************************************** 00141 00142 Construct a FileBucket with the provided path, record-size, 00143 and inital record count. The latter causes records to be 00144 pre-allocated, saving a certain amount of growth activity. 00145 Selecting a record size that roughly matches the serialized 00146 content will limit 'thrashing'. 00147 00148 **********************************************************************/ 00149 00150 this (FilePath path, BlockSize block, uint initialRecords) 00151 { 00152 this.path = path; 00153 this.block = block; 00154 00155 // open a storage file 00156 file = new FileConduit (path, FileStyle.ReadWriteCreate); 00157 00158 // set initial file size (can be zero) 00159 fileSize = initialRecords * block.capacity; 00160 file.seek (fileSize); 00161 file.truncate (); 00162 00163 // bind a buffer to the storage file, but don't allocate 00164 // memory since we'll explicitly set that as appropriate 00165 buffer = new Buffer; 00166 buffer.setConduit (file); 00167 } 00168 00169 /********************************************************************** 00170 00171 Destructor tries to close the file, if not already done. 00172 00173 **********************************************************************/ 00174 00175 ~this() 00176 { 00177 close (); 00178 } 00179 00180 /********************************************************************** 00181 00182 Return the block-size in use for this FileBucket 00183 00184 **********************************************************************/ 00185 00186 int getBufferSize () 00187 { 00188 return block.capacity+1; 00189 } 00190 00191 /********************************************************************** 00192 00193 Return where the FileBucket is located 00194 00195 **********************************************************************/ 00196 00197 FilePath getFilePath () 00198 { 00199 return path; 00200 } 00201 00202 /********************************************************************** 00203 00204 Return the currently populated size of this FileBucket 00205 00206 **********************************************************************/ 00207 00208 synchronized long length () 00209 { 00210 return waterLine; 00211 } 00212 00213 /********************************************************************** 00214 00215 Return the serialized data for the provided key. Returns 00216 null if the key was not found. 00217 00218 **********************************************************************/ 00219 00220 synchronized void[] get (char[] key) 00221 { 00222 Record r = null; 00223 00224 if (key in map) 00225 { 00226 r = map [key]; 00227 return r.read (this); 00228 } 00229 return null; 00230 } 00231 00232 /********************************************************************** 00233 00234 Remove the provided key from this FileBucket. 00235 00236 **********************************************************************/ 00237 00238 synchronized void remove (char[] key) 00239 { 00240 delete map [key]; 00241 } 00242 00243 /********************************************************************** 00244 00245 Write a serialized block of data, and associate it with 00246 the provided key. All keys must be unique, and it is the 00247 responsibility of the programmer to ensure this. Reusing 00248 an existing key will overwrite previous data. 00249 00250 Note that data is allowed to grow within the occupied 00251 bucket until it becomes larger than the allocated space. 00252 When this happens, the data is moved to a larger bucket 00253 at the file tail. 00254 00255 **********************************************************************/ 00256 00257 synchronized void put (char[] key, void[] data) 00258 { 00259 Record r = map [key]; 00260 00261 if (r is null) 00262 { 00263 r = new Record (); 00264 map [key] = r; 00265 } 00266 r.write (this, data, block); 00267 } 00268 00269 /********************************************************************** 00270 00271 Close this FileBucket -- all content is lost. 00272 00273 **********************************************************************/ 00274 00275 synchronized void close () 00276 { 00277 if (file) 00278 { 00279 file.close (); 00280 file = null; 00281 map = null; 00282 } 00283 } 00284 00285 /********************************************************************** 00286 00287 Each Record takes up a number of 'pages' within the file. 00288 The size of these pages is determined by the BlockSize 00289 provided during FileBucket construction. Additional space 00290 at the end of each block is potentially wasted, but enables 00291 content to grow in size without creating a myriad of holes. 00292 00293 **********************************************************************/ 00294 00295 private class Record 00296 { 00297 private long offset; 00298 private int length, 00299 capacity = -1; 00300 00301 /************************************************************** 00302 00303 **************************************************************/ 00304 00305 private static void eof (FileBucket bucket) 00306 { 00307 throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.toString()~"'"); 00308 } 00309 00310 /************************************************************** 00311 00312 This should be protected from thread-contention at 00313 a higher level. 00314 00315 **************************************************************/ 00316 00317 void[] read (FileBucket bucket) 00318 { 00319 void[] data = new void [length]; 00320 bucket.buffer.setContent (data, 0); 00321 00322 bucket.file.seek (offset); 00323 if (bucket.file.read (bucket.buffer) != length) 00324 eof (bucket); 00325 00326 return data; 00327 } 00328 00329 /************************************************************** 00330 00331 This should be protected from thread-contention at 00332 a higher level. 00333 00334 **************************************************************/ 00335 00336 void write (FileBucket bucket, void[] data, BlockSize block) 00337 { 00338 length = data.length; 00339 00340 // create new slot if we exceed capacity 00341 if (length > capacity) 00342 createBucket (bucket, length, block); 00343 00344 // locate to start of content 00345 bucket.file.seek (offset); 00346 00347 // write content 00348 bucket.buffer.setValidContent (data); 00349 if (! bucket.file.flush (bucket.buffer)) 00350 eof (bucket); 00351 } 00352 00353 /************************************************************** 00354 00355 **************************************************************/ 00356 00357 void createBucket (FileBucket bucket, int bytes, BlockSize block) 00358 { 00359 offset = bucket.waterLine; 00360 capacity = (bytes + block.capacity) & ~block.capacity; 00361 00362 bucket.waterLine += capacity; 00363 if (bucket.waterLine > bucket.fileSize) 00364 { 00365 // grow the filesize 00366 bucket.fileSize = bucket.waterLine * 2; 00367 00368 // expand the physical file size 00369 bucket.file.seek (bucket.fileSize); 00370 bucket.file.truncate (); 00371 } 00372 } 00373 } 00374 } 00375 00376