Main Page | Class Hierarchy | Alphabetical List | Class List | File List | Class Members | File Members | Related Pages

FileBucket.d

Go to the documentation of this file.
00001 /*******************************************************************************
00002 
00003         @file FileBucket.d
00004         
00005         Copyright (C) 2004 Kris Bell
00006 
00007         
00008         This software is provided 'as-is', without any express or implied
00009         warranty. In no event will the authors be held liable for damages
00010         of any kind arising from the use of this software.
00011         
00012         Permission is hereby granted to anyone to use this software for any 
00013         purpose, including commercial applications, and to alter it and/or 
00014         redistribute it freely, subject to the following restrictions:
00015         
00016         1. The origin of this software must not be misrepresented; you must 
00017            not claim that you wrote the original software. If you use this 
00018            software in a product, an acknowledgment within documentation of 
00019            said product would be appreciated but is not required.
00020 
00021         2. Altered source versions must be plainly marked as such, and must 
00022            not be misrepresented as being the original software.
00023 
00024         3. This notice may not be removed or altered from any distribution
00025            of the source.
00026 
00027 
00028         @version        Initial version, April 2004
00029         @author         Kris
00030 
00031 
00032 *******************************************************************************/
00033 
00034 
00035 module mango.io.FileBucket;
00036 
00037 public  import  mango.io.FilePath;
00038 
00039 private import  mango.io.Buffer,
00040                 mango.io.FileStyle,
00041                 mango.io.Exception,
00042                 mango.io.FileConduit;
00043 
00044 private import  mango.io.model.IBitBucket;
00045 
00046 /******************************************************************************
00047 
00048         FileBucket implements a simple mechanism to store and recover a 
00049         large quantity of data for the duration of the hosting process.
00050         It is intended to act as a local-cache for a remote data-source, 
00051         or as a spillover area for large in-memory cache instances. 
00052         
00053         Note that any and all stored data is rendered invalid the moment
00054         a FileBucket object is garbage-collected.
00055 
00056         The implementation follows a fixed-capacity record scheme, where
00057         content can be rewritten in-place until said capacity is reached.
00058         At such time, the altered content is moved to a larger capacity
00059         record at end-of-file, and a hole remains at the prior location.
00060         These holes are not collected, since the lifespan of a FileBucket
00061         is limited to that of the host process.
00062 
00063         All index keys must be unique. Writing to the FileBucket with an
00064         existing key will overwrite any previous content. What follows
00065         is a contrived example:
00066         
00067         @code
00068         char[] text = "this is a test";
00069 
00070         FileBucket bucket = new FileBucket (new FilePath("bucket.bin"), FileBucket.HalfK);
00071 
00072         // insert some data, and retrieve it again
00073         bucket.put ("a key", text);
00074         char[] b = cast(char[]) bucket.get ("a key");
00075 
00076         assert (b == text);
00077         bucket.close();
00078         @endcode
00079 
00080 ******************************************************************************/
00081 
00082 class FileBucket : IBitBucket
00083 {
00084         /**********************************************************************
00085 
00086                 Define the capacity (block-size) of each record
00087 
00088         **********************************************************************/
00089 
00090         struct BlockSize
00091         {
00092                 int capacity;
00093         }
00094 
00095         // basic capacity for each record
00096         private FilePath                path;
00097 
00098         // basic capacity for each record
00099         private BlockSize               block;
00100 
00101         // where content is stored
00102         private FileConduit             file;
00103 
00104         // pointers to file records
00105         private Record[char[]]          map;
00106 
00107         // place for reading and writing from/into
00108         private Buffer                  buffer;
00109         
00110         // current file size
00111         private long                    fileSize;
00112 
00113         // current file usage
00114         private long                    waterLine;
00115 
00116         // supported block sizes
00117         static const BlockSize          EighthK  = {128-1},
00118                                         HalfK    = {512-1},
00119                                         OneK     = {1024*1-1},
00120                                         TwoK     = {1024*2-1},
00121                                         FourK    = {1024*4-1},
00122                                         EightK   = {1024*8-1},
00123                                         SixteenK = {1024*16-1},
00124                                         ThirtyTwoK = {1024*32-1},
00125                                         SixtyFourK = {1024*64-1};
00126 
00127         /**********************************************************************
00128 
00129                 Construct a FileBucket with the provided path and record-
00130                 size. Selecting a record size that roughly matches the 
00131                 serialized content will limit 'thrashing'.
00132 
00133         **********************************************************************/
00134 
00135         this (FilePath path, BlockSize block)
00136         {
00137                 this (path, block, 100);
00138         }
00139 
00140         /**********************************************************************
00141 
00142                 Construct a FileBucket with the provided path, record-size,
00143                 and inital record count. The latter causes records to be 
00144                 pre-allocated, saving a certain amount of growth activity.
00145                 Selecting a record size that roughly matches the serialized 
00146                 content will limit 'thrashing'. 
00147 
00148         **********************************************************************/
00149 
00150         this (FilePath path, BlockSize block, uint initialRecords)
00151         {
00152                 this.path = path;
00153                 this.block = block;
00154 
00155                 // open a storage file
00156                 file = new FileConduit (path, FileStyle.ReadWriteCreate);
00157 
00158                 // set initial file size (can be zero)
00159                 fileSize = initialRecords * block.capacity;
00160                 file.seek (fileSize);
00161                 file.truncate ();
00162 
00163                 // bind a buffer to the storage file, but don't allocate
00164                 // memory since we'll explicitly set that as appropriate
00165                 buffer = new Buffer;
00166                 buffer.setConduit (file);
00167         }
00168 
00169         /**********************************************************************
00170 
00171                 Destructor tries to close the file, if not already done.
00172 
00173         **********************************************************************/
00174 
00175         ~this()
00176         {
00177                 close ();
00178         }
00179 
00180         /**********************************************************************
00181 
00182                 Return the block-size in use for this FileBucket
00183 
00184         **********************************************************************/
00185 
00186         int getBufferSize ()
00187         {
00188                 return block.capacity+1;
00189         }
00190 
00191         /**********************************************************************
00192         
00193                 Return where the FileBucket is located
00194 
00195         **********************************************************************/
00196 
00197         FilePath getFilePath ()
00198         {
00199                 return path;
00200         }
00201 
00202         /**********************************************************************
00203 
00204                 Return the currently populated size of this FileBucket
00205 
00206         **********************************************************************/
00207 
00208         synchronized long length ()
00209         {
00210                 return waterLine;
00211         }
00212 
00213         /**********************************************************************
00214 
00215                 Return the serialized data for the provided key. Returns
00216                 null if the key was not found.
00217 
00218         **********************************************************************/
00219 
00220         synchronized void[] get (char[] key)
00221         {
00222                 Record r = null;
00223 
00224                 if (key in map)
00225                    {
00226                    r = map [key];
00227                    return r.read (this);
00228                    }
00229                 return null;
00230         }
00231 
00232         /**********************************************************************
00233 
00234                 Remove the provided key from this FileBucket.
00235 
00236         **********************************************************************/
00237 
00238         synchronized void remove (char[] key)
00239         {
00240                 delete map [key];
00241         }
00242 
00243         /**********************************************************************
00244 
00245                 Write a serialized block of data, and associate it with
00246                 the provided key. All keys must be unique, and it is the
00247                 responsibility of the programmer to ensure this. Reusing 
00248                 an existing key will overwrite previous data. 
00249 
00250                 Note that data is allowed to grow within the occupied 
00251                 bucket until it becomes larger than the allocated space.
00252                 When this happens, the data is moved to a larger bucket
00253                 at the file tail.
00254 
00255         **********************************************************************/
00256 
00257         synchronized void put (char[] key, void[] data)
00258         {
00259                 Record  r = map [key];
00260 
00261                 if (r is null)
00262                    {
00263                    r = new Record ();
00264                    map [key] =  r;
00265                    }
00266                 r.write (this, data, block);
00267         }
00268 
00269         /**********************************************************************
00270 
00271                 Close this FileBucket -- all content is lost.
00272 
00273         **********************************************************************/
00274 
00275         synchronized void close ()
00276         {
00277                 if (file)
00278                    {
00279                    file.close ();
00280                    file = null;
00281                    map = null;
00282                    }
00283         }
00284 
00285         /**********************************************************************
00286 
00287                 Each Record takes up a number of 'pages' within the file. 
00288                 The size of these pages is determined by the BlockSize 
00289                 provided during FileBucket construction. Additional space
00290                 at the end of each block is potentially wasted, but enables 
00291                 content to grow in size without creating a myriad of holes.
00292 
00293         **********************************************************************/
00294 
00295         private class Record
00296         {
00297                 private long            offset;
00298                 private int             length,
00299                                         capacity = -1;
00300 
00301                 /**************************************************************
00302 
00303                 **************************************************************/
00304 
00305                 private static void eof (FileBucket bucket)
00306                 {
00307                         throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.toString()~"'");
00308                 }
00309 
00310                 /**************************************************************
00311 
00312                         This should be protected from thread-contention at
00313                         a higher level.
00314 
00315                 **************************************************************/
00316 
00317                 void[] read (FileBucket bucket)
00318                 {
00319                         void[] data = new void [length];
00320                         bucket.buffer.setContent (data, 0);
00321 
00322                         bucket.file.seek (offset);
00323                         if (bucket.file.read (bucket.buffer) != length)
00324                             eof (bucket);
00325 
00326                         return data;
00327                 }
00328 
00329                 /**************************************************************
00330 
00331                         This should be protected from thread-contention at
00332                         a higher level.
00333 
00334                 **************************************************************/
00335 
00336                 void write (FileBucket bucket, void[] data, BlockSize block)
00337                 {
00338                         length = data.length;
00339 
00340                         // create new slot if we exceed capacity
00341                         if (length > capacity)
00342                             createBucket (bucket, length, block);
00343 
00344                         // locate to start of content 
00345                         bucket.file.seek (offset);
00346 
00347                         // write content
00348                         bucket.buffer.setValidContent (data);
00349                         if (! bucket.file.flush (bucket.buffer))
00350                               eof (bucket);
00351                 }
00352 
00353                 /**************************************************************
00354 
00355                 **************************************************************/
00356 
00357                 void createBucket (FileBucket bucket, int bytes, BlockSize block)
00358                 {
00359                         offset = bucket.waterLine;
00360                         capacity = (bytes + block.capacity) & ~block.capacity;
00361 
00362                         bucket.waterLine += capacity;
00363                         if (bucket.waterLine > bucket.fileSize)
00364                            {
00365                            // grow the filesize 
00366                            bucket.fileSize = bucket.waterLine * 2;
00367 
00368                            // expand the physical file size
00369                            bucket.file.seek (bucket.fileSize);
00370                            bucket.file.truncate ();
00371                            }
00372                 }
00373         }
00374 }
00375 
00376 

Generated on Sun Nov 7 19:06:50 2004 for Mango by doxygen 1.3.6