Main Page | Class Hierarchy | Alphabetical List | Class List | File List | Class Members | File Members | Related Pages

FileBucket.d

Go to the documentation of this file.
00001 /*******************************************************************************
00002 
00003         @file FileBucket.d
00004         
00005         Copyright (c) 2004 Kris Bell
00006         
00007         This software is provided 'as-is', without any express or implied
00008         warranty. In no event will the authors be held liable for damages
00009         of any kind arising from the use of this software.
00010         
00011         Permission is hereby granted to anyone to use this software for any 
00012         purpose, including commercial applications, and to alter it and/or 
00013         redistribute it freely, subject to the following restrictions:
00014         
00015         1. The origin of this software must not be misrepresented; you must 
00016            not claim that you wrote the original software. If you use this 
00017            software in a product, an acknowledgment within documentation of 
00018            said product would be appreciated but is not required.
00019 
00020         2. Altered source versions must be plainly marked as such, and must 
00021            not be misrepresented as being the original software.
00022 
00023         3. This notice may not be removed or altered from any distribution
00024            of the source.
00025 
00026         4. Derivative works are permitted, but they must carry this notice
00027            in full and credit the original source.
00028            of the source.
00029 
00030 
00031         @version        Initial version, April 2004
00032         @author         Kris
00033 
00034 
00035 *******************************************************************************/
00036 
00037 
00038 module mango.io.FileBucket;
00039 
00040 public  import  mango.io.FilePath;
00041 
00042 private import  mango.io.Buffer,
00043                 mango.io.FileStyle,
00044                 mango.io.Exception,
00045                 mango.io.FileConduit;
00046 
00047 private import  mango.io.model.IBitBucket;
00048 
00049 /******************************************************************************
00050 
00051         FileBucket implements a simple mechanism to store and recover a 
00052         large quantity of data for the duration of the hosting process.
00053         It is intended to act as a local-cache for a remote data-source, 
00054         or as a spillover area for large in-memory cache instances. 
00055         
00056         Note that any and all stored data is rendered invalid the moment
00057         a FileBucket object is garbage-collected.
00058 
00059         The implementation follows a fixed-capacity record scheme, where
00060         content can be rewritten in-place until said capacity is reached.
00061         At such time, the altered content is moved to a larger capacity
00062         record at end-of-file, and a hole remains at the prior location.
00063         These holes are not collected, since the lifespan of a FileBucket
00064         is limited to that of the host process.
00065 
00066         All index keys must be unique. Writing to the FileBucket with an
00067         existing key will overwrite any previous content. What follows
00068         is a contrived example:
00069         
00070         @code
00071         char[] text = "this is a test";
00072 
00073         FileBucket bucket = new FileBucket (new FilePath("bucket.bin"), FileBucket.HalfK);
00074 
00075         // insert some data, and retrieve it again
00076         bucket.put ("a key", text);
00077         char[] b = cast(char[]) bucket.get ("a key");
00078 
00079         assert (b == text);
00080         bucket.close();
00081         @endcode
00082 
00083 ******************************************************************************/
00084 
00085 class FileBucket : IBitBucket
00086 {
00087         /**********************************************************************
00088 
00089                 Define the capacity (block-size) of each record
00090 
00091         **********************************************************************/
00092 
00093         struct BlockSize
00094         {
00095                 int capacity;
00096         }
00097 
00098         // basic capacity for each record
00099         private FilePath                path;
00100 
00101         // basic capacity for each record
00102         private BlockSize               block;
00103 
00104         // where content is stored
00105         private FileConduit             file;
00106 
00107         // pointers to file records
00108         private Record[char[]]          map;
00109 
00110         // place for reading and writing from/into
00111         private Buffer                  buffer;
00112         
00113         // current file size
00114         private long                    fileSize;
00115 
00116         // current file usage
00117         private long                    waterLine;
00118 
00119         // supported block sizes
00120         static const BlockSize          EighthK  = {128-1},
00121                                         HalfK    = {512-1},
00122                                         OneK     = {1024*1-1},
00123                                         TwoK     = {1024*2-1},
00124                                         FourK    = {1024*4-1},
00125                                         EightK   = {1024*8-1},
00126                                         SixteenK = {1024*16-1},
00127                                         ThirtyTwoK = {1024*32-1},
00128                                         SixtyFourK = {1024*64-1};
00129 
00130         /**********************************************************************
00131 
00132                 Construct a FileBucket with the provided path and record-
00133                 size. Selecting a record size that roughly matches the 
00134                 serialized content will limit 'thrashing'.
00135 
00136         **********************************************************************/
00137 
00138         this (FilePath path, BlockSize block)
00139         {
00140                 this (path, block, 100);
00141         }
00142 
00143         /**********************************************************************
00144 
00145                 Construct a FileBucket with the provided path, record-size,
00146                 and inital record count. The latter causes records to be 
00147                 pre-allocated, saving a certain amount of growth activity.
00148                 Selecting a record size that roughly matches the serialized 
00149                 content will limit 'thrashing'. 
00150 
00151         **********************************************************************/
00152 
00153         this (FilePath path, BlockSize block, uint initialRecords)
00154         {
00155                 this.path = path;
00156                 this.block = block;
00157 
00158                 // open a storage file
00159                 file = new FileConduit (path, FileStyle.ReadWriteCreate);
00160 
00161                 // set initial file size (can be zero)
00162                 fileSize = initialRecords * block.capacity;
00163                 file.seek (fileSize);
00164                 file.truncate ();
00165 
00166                 // bind a buffer to the storage file, but don't allocate
00167                 // memory since we'll explicitly set that as appropriate
00168                 buffer = new Buffer;
00169                 buffer.setConduit (file);
00170         }
00171 
00172         /**********************************************************************
00173 
00174                 Destructor tries to close the file, if not already done.
00175 
00176         **********************************************************************/
00177 
00178         ~this()
00179         {
00180                 close ();
00181         }
00182 
00183         /**********************************************************************
00184 
00185                 Return the block-size in use for this FileBucket
00186 
00187         **********************************************************************/
00188 
00189         int getBufferSize ()
00190         {
00191                 return block.capacity+1;
00192         }
00193 
00194         /**********************************************************************
00195         
00196                 Return where the FileBucket is located
00197 
00198         **********************************************************************/
00199 
00200         FilePath getFilePath ()
00201         {
00202                 return path;
00203         }
00204 
00205         /**********************************************************************
00206 
00207                 Return the currently populated size of this FileBucket
00208 
00209         **********************************************************************/
00210 
00211         synchronized long length ()
00212         {
00213                 return waterLine;
00214         }
00215 
00216         /**********************************************************************
00217 
00218                 Return the serialized data for the provided key. Returns
00219                 null if the key was not found.
00220 
00221         **********************************************************************/
00222 
00223         synchronized void[] get (char[] key)
00224         {
00225                 Record r = null;
00226 
00227                 if (key in map)
00228                    {
00229                    r = map [key];
00230                    return r.read (this);
00231                    }
00232                 return null;
00233         }
00234 
00235         /**********************************************************************
00236 
00237                 Remove the provided key from this FileBucket.
00238 
00239         **********************************************************************/
00240 
00241         synchronized void remove (char[] key)
00242         {
00243                 delete map [key];
00244         }
00245 
00246         /**********************************************************************
00247 
00248                 Write a serialized block of data, and associate it with
00249                 the provided key. All keys must be unique, and it is the
00250                 responsibility of the programmer to ensure this. Reusing 
00251                 an existing key will overwrite previous data. 
00252 
00253                 Note that data is allowed to grow within the occupied 
00254                 bucket until it becomes larger than the allocated space.
00255                 When this happens, the data is moved to a larger bucket
00256                 at the file tail.
00257 
00258         **********************************************************************/
00259 
00260         synchronized void put (char[] key, void[] data)
00261         {
00262                 Record  r = map [key];
00263 
00264                 if (r is null)
00265                    {
00266                    r = new Record ();
00267                    map [key] =  r;
00268                    }
00269                 r.write (this, data, block);
00270         }
00271 
00272         /**********************************************************************
00273 
00274                 Close this FileBucket -- all content is lost.
00275 
00276         **********************************************************************/
00277 
00278         synchronized void close ()
00279         {
00280                 if (file)
00281                    {
00282                    file.close ();
00283                    file = null;
00284                    map = null;
00285                    }
00286         }
00287 
00288         /**********************************************************************
00289 
00290                 Each Record takes up a number of 'pages' within the file. 
00291                 The size of these pages is determined by the BlockSize 
00292                 provided during FileBucket construction. Additional space
00293                 at the end of each block is potentially wasted, but enables 
00294                 content to grow in size without creating a myriad of holes.
00295 
00296         **********************************************************************/
00297 
00298         private class Record
00299         {
00300                 private long            offset;
00301                 private int             length,
00302                                         capacity = -1;
00303 
00304                 /**************************************************************
00305 
00306                 **************************************************************/
00307 
00308                 private static void eof (FileBucket bucket)
00309                 {
00310                         throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.toString()~"'");
00311                 }
00312 
00313                 /**************************************************************
00314 
00315                         This should be protected from thread-contention at
00316                         a higher level.
00317 
00318                 **************************************************************/
00319 
00320                 void[] read (FileBucket bucket)
00321                 {
00322                         void[] data = new void [length];
00323                         bucket.buffer.setContent (data, 0);
00324 
00325                         bucket.file.seek (offset);
00326                         if (bucket.file.read (bucket.buffer) != length)
00327                             eof (bucket);
00328 
00329                         return data;
00330                 }
00331 
00332                 /**************************************************************
00333 
00334                         This should be protected from thread-contention at
00335                         a higher level.
00336 
00337                 **************************************************************/
00338 
00339                 void write (FileBucket bucket, void[] data, BlockSize block)
00340                 {
00341                         length = data.length;
00342 
00343                         // create new slot if we exceed capacity
00344                         if (length > capacity)
00345                             createBucket (bucket, length, block);
00346 
00347                         // locate to start of content 
00348                         bucket.file.seek (offset);
00349 
00350                         // write content
00351                         bucket.buffer.setValidContent (data);
00352                         if (! bucket.file.flush (bucket.buffer))
00353                               eof (bucket);
00354                 }
00355 
00356                 /**************************************************************
00357 
00358                 **************************************************************/
00359 
00360                 void createBucket (FileBucket bucket, int bytes, BlockSize block)
00361                 {
00362                         offset = bucket.waterLine;
00363                         capacity = (bytes + block.capacity) & ~block.capacity;
00364 
00365                         bucket.waterLine += capacity;
00366                         if (bucket.waterLine > bucket.fileSize)
00367                            {
00368                            // grow the filesize 
00369                            bucket.fileSize = bucket.waterLine * 2;
00370 
00371                            // expand the physical file size
00372                            bucket.file.seek (bucket.fileSize);
00373                            bucket.file.truncate ();
00374                            }
00375                 }
00376         }
00377 }
00378 
00379 

Generated on Sun Mar 6 00:30:56 2005 for Mango by doxygen 1.3.6