00001 /******************************************************************************* 00002 00003 @file FileBucket.d 00004 00005 Copyright (c) 2004 Kris Bell 00006 00007 This software is provided 'as-is', without any express or implied 00008 warranty. In no event will the authors be held liable for damages 00009 of any kind arising from the use of this software. 00010 00011 Permission is hereby granted to anyone to use this software for any 00012 purpose, including commercial applications, and to alter it and/or 00013 redistribute it freely, subject to the following restrictions: 00014 00015 1. The origin of this software must not be misrepresented; you must 00016 not claim that you wrote the original software. If you use this 00017 software in a product, an acknowledgment within documentation of 00018 said product would be appreciated but is not required. 00019 00020 2. Altered source versions must be plainly marked as such, and must 00021 not be misrepresented as being the original software. 00022 00023 3. This notice may not be removed or altered from any distribution 00024 of the source. 00025 00026 4. Derivative works are permitted, but they must carry this notice 00027 in full and credit the original source. 00028 of the source. 00029 00030 00031 @version Initial version, April 2004 00032 @author Kris 00033 00034 00035 *******************************************************************************/ 00036 00037 00038 module mango.io.FileBucket; 00039 00040 public import mango.io.FilePath; 00041 00042 private import mango.io.Exception, 00043 mango.io.FileConduit; 00044 00045 private import mango.io.model.IBitBucket; 00046 00047 /****************************************************************************** 00048 00049 FileBucket implements a simple mechanism to store and recover a 00050 large quantity of data for the duration of the hosting process. 00051 It is intended to act as a local-cache for a remote data-source, 00052 or as a spillover area for large in-memory cache instances. 00053 00054 Note that any and all stored data is rendered invalid the moment 00055 a FileBucket object is garbage-collected. 00056 00057 The implementation follows a fixed-capacity record scheme, where 00058 content can be rewritten in-place until said capacity is reached. 00059 At such time, the altered content is moved to a larger capacity 00060 record at end-of-file, and a hole remains at the prior location. 00061 These holes are not collected, since the lifespan of a FileBucket 00062 is limited to that of the host process. 00063 00064 All index keys must be unique. Writing to the FileBucket with an 00065 existing key will overwrite any previous content. What follows 00066 is a contrived example: 00067 00068 @code 00069 char[] text = "this is a test"; 00070 00071 FileBucket bucket = new FileBucket (new FilePath("bucket.bin"), FileBucket.HalfK); 00072 00073 // insert some data, and retrieve it again 00074 bucket.put ("a key", text); 00075 char[] b = cast(char[]) bucket.get ("a key"); 00076 00077 assert (b == text); 00078 bucket.close(); 00079 @endcode 00080 00081 ******************************************************************************/ 00082 00083 class FileBucket : IBitBucket 00084 { 00085 /********************************************************************** 00086 00087 Define the capacity (block-size) of each record 00088 00089 **********************************************************************/ 00090 00091 struct BlockSize 00092 { 00093 int capacity; 00094 } 00095 00096 // basic capacity for each record 00097 private FilePath path; 00098 00099 // basic capacity for each record 00100 private BlockSize block; 00101 00102 // where content is stored 00103 private FileConduit file; 00104 00105 // pointers to file records 00106 private Record[char[]] map; 00107 00108 // current file size 00109 private ulong fileSize; 00110 00111 // current file usage 00112 private ulong waterLine; 00113 00114 // supported block sizes 00115 static const BlockSize EighthK = {128-1}, 00116 HalfK = {512-1}, 00117 OneK = {1024*1-1}, 00118 TwoK = {1024*2-1}, 00119 FourK = {1024*4-1}, 00120 EightK = {1024*8-1}, 00121 SixteenK = {1024*16-1}, 00122 ThirtyTwoK = {1024*32-1}, 00123 SixtyFourK = {1024*64-1}; 00124 00125 /********************************************************************** 00126 00127 Construct a FileBucket with the provided path and record- 00128 size. Selecting a record size that roughly matches the 00129 serialized content will limit 'thrashing'. 00130 00131 **********************************************************************/ 00132 00133 this (FilePath path, BlockSize block) 00134 { 00135 this (path, block, 100); 00136 } 00137 00138 /********************************************************************** 00139 00140 Construct a FileBucket with the provided path, record-size, 00141 and inital record count. The latter causes records to be 00142 pre-allocated, saving a certain amount of growth activity. 00143 Selecting a record size that roughly matches the serialized 00144 content will limit 'thrashing'. 00145 00146 **********************************************************************/ 00147 00148 this (FilePath path, BlockSize block, uint initialRecords) 00149 { 00150 this.path = path; 00151 this.block = block; 00152 00153 // open a storage file 00154 file = new FileConduit (path, FileStyle.ReadWriteCreate); 00155 00156 // set initial file size (can be zero) 00157 fileSize = initialRecords * block.capacity; 00158 file.seek (fileSize); 00159 file.truncate (); 00160 } 00161 00162 /********************************************************************** 00163 00164 Return the block-size in use for this FileBucket 00165 00166 **********************************************************************/ 00167 00168 int getBufferSize () 00169 { 00170 return block.capacity+1; 00171 } 00172 00173 /********************************************************************** 00174 00175 Return where the FileBucket is located 00176 00177 **********************************************************************/ 00178 00179 FilePath getFilePath () 00180 { 00181 return path; 00182 } 00183 00184 /********************************************************************** 00185 00186 Return the currently populated size of this FileBucket 00187 00188 **********************************************************************/ 00189 00190 synchronized ulong length () 00191 { 00192 return waterLine; 00193 } 00194 00195 /********************************************************************** 00196 00197 Return the serialized data for the provided key. Returns 00198 null if the key was not found. 00199 00200 **********************************************************************/ 00201 00202 synchronized void[] get (char[] key) 00203 { 00204 Record r = null; 00205 00206 if (key in map) 00207 { 00208 r = map [key]; 00209 return r.read (this); 00210 } 00211 return null; 00212 } 00213 00214 /********************************************************************** 00215 00216 Remove the provided key from this FileBucket. 00217 00218 **********************************************************************/ 00219 00220 synchronized void remove (char[] key) 00221 { 00222 map.remove(key); 00223 } 00224 00225 /********************************************************************** 00226 00227 Write a serialized block of data, and associate it with 00228 the provided key. All keys must be unique, and it is the 00229 responsibility of the programmer to ensure this. Reusing 00230 an existing key will overwrite previous data. 00231 00232 Note that data is allowed to grow within the occupied 00233 bucket until it becomes larger than the allocated space. 00234 When this happens, the data is moved to a larger bucket 00235 at the file tail. 00236 00237 **********************************************************************/ 00238 00239 synchronized void put (char[] key, void[] data) 00240 { 00241 Record* r = key in map; 00242 00243 if (r is null) 00244 { 00245 Record rr = new Record (); 00246 map [key] = rr; 00247 r = &rr; 00248 } 00249 r.write (this, data, block); 00250 } 00251 00252 /********************************************************************** 00253 00254 Close this FileBucket -- all content is lost. 00255 00256 **********************************************************************/ 00257 00258 synchronized void close () 00259 { 00260 if (file) 00261 { 00262 file.close (); 00263 file = null; 00264 map = null; 00265 } 00266 } 00267 00268 /********************************************************************** 00269 00270 Each Record takes up a number of 'pages' within the file. 00271 The size of these pages is determined by the BlockSize 00272 provided during FileBucket construction. Additional space 00273 at the end of each block is potentially wasted, but enables 00274 content to grow in size without creating a myriad of holes. 00275 00276 **********************************************************************/ 00277 00278 private class Record 00279 { 00280 private ulong offset; 00281 private int length, 00282 capacity = -1; 00283 00284 /************************************************************** 00285 00286 **************************************************************/ 00287 00288 private static void eof (FileBucket bucket) 00289 { 00290 throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.toString()~"'"); 00291 } 00292 00293 /************************************************************** 00294 00295 This should be protected from thread-contention at 00296 a higher level. 00297 00298 **************************************************************/ 00299 00300 void[] read (FileBucket bucket) 00301 { 00302 void[] data = new ubyte [length]; 00303 00304 bucket.file.seek (offset); 00305 if (bucket.file.read (data) != length) 00306 eof (bucket); 00307 00308 return data; 00309 } 00310 00311 /************************************************************** 00312 00313 This should be protected from thread-contention at 00314 a higher level. 00315 00316 **************************************************************/ 00317 00318 void write (FileBucket bucket, void[] data, BlockSize block) 00319 { 00320 length = data.length; 00321 00322 // create new slot if we exceed capacity 00323 if (length > capacity) 00324 createBucket (bucket, length, block); 00325 00326 // locate to start of content 00327 bucket.file.seek (offset); 00328 00329 // write content 00330 if (bucket.file.write (data) != length) 00331 eof (bucket); 00332 } 00333 00334 /************************************************************** 00335 00336 **************************************************************/ 00337 00338 void createBucket (FileBucket bucket, int bytes, BlockSize block) 00339 { 00340 offset = bucket.waterLine; 00341 capacity = (bytes + block.capacity) & ~block.capacity; 00342 00343 bucket.waterLine += capacity; 00344 if (bucket.waterLine > bucket.fileSize) 00345 { 00346 // grow the filesize 00347 bucket.fileSize = bucket.waterLine * 2; 00348 00349 // expand the physical file size 00350 bucket.file.seek (bucket.fileSize); 00351 bucket.file.truncate (); 00352 } 00353 } 00354 } 00355 } 00356 00357