Main Page | Class Hierarchy | Alphabetical List | Class List | Directories | File List | Class Members | File Members | Related Pages

FileBucket.d

Go to the documentation of this file.
00001 /*******************************************************************************
00002 
00003         @file FileBucket.d
00004         
00005         Copyright (c) 2004 Kris Bell
00006         
00007         This software is provided 'as-is', without any express or implied
00008         warranty. In no event will the authors be held liable for damages
00009         of any kind arising from the use of this software.
00010         
00011         Permission is hereby granted to anyone to use this software for any 
00012         purpose, including commercial applications, and to alter it and/or 
00013         redistribute it freely, subject to the following restrictions:
00014         
00015         1. The origin of this software must not be misrepresented; you must 
00016            not claim that you wrote the original software. If you use this 
00017            software in a product, an acknowledgment within documentation of 
00018            said product would be appreciated but is not required.
00019 
00020         2. Altered source versions must be plainly marked as such, and must 
00021            not be misrepresented as being the original software.
00022 
00023         3. This notice may not be removed or altered from any distribution
00024            of the source.
00025 
00026         4. Derivative works are permitted, but they must carry this notice
00027            in full and credit the original source.
00028            of the source.
00029 
00030 
00031         @version        Initial version, April 2004
00032         @author         Kris
00033 
00034 
00035 *******************************************************************************/
00036 
00037 
00038 module mango.io.FileBucket;
00039 
00040 public  import  mango.io.FilePath;
00041 
00042 private import  mango.io.Exception,
00043                 mango.io.FileConduit;
00044 
00045 private import  mango.io.model.IBitBucket;
00046 
00047 /******************************************************************************
00048 
00049         FileBucket implements a simple mechanism to store and recover a 
00050         large quantity of data for the duration of the hosting process.
00051         It is intended to act as a local-cache for a remote data-source, 
00052         or as a spillover area for large in-memory cache instances. 
00053         
00054         Note that any and all stored data is rendered invalid the moment
00055         a FileBucket object is garbage-collected.
00056 
00057         The implementation follows a fixed-capacity record scheme, where
00058         content can be rewritten in-place until said capacity is reached.
00059         At such time, the altered content is moved to a larger capacity
00060         record at end-of-file, and a hole remains at the prior location.
00061         These holes are not collected, since the lifespan of a FileBucket
00062         is limited to that of the host process.
00063 
00064         All index keys must be unique. Writing to the FileBucket with an
00065         existing key will overwrite any previous content. What follows
00066         is a contrived example:
00067         
00068         @code
00069         char[] text = "this is a test";
00070 
00071         FileBucket bucket = new FileBucket (new FilePath("bucket.bin"), FileBucket.HalfK);
00072 
00073         // insert some data, and retrieve it again
00074         bucket.put ("a key", text);
00075         char[] b = cast(char[]) bucket.get ("a key");
00076 
00077         assert (b == text);
00078         bucket.close();
00079         @endcode
00080 
00081 ******************************************************************************/
00082 
00083 class FileBucket : IBitBucket
00084 {
00085         /**********************************************************************
00086 
00087                 Define the capacity (block-size) of each record
00088 
00089         **********************************************************************/
00090 
00091         struct BlockSize
00092         {
00093                 int capacity;
00094         }
00095 
00096         // basic capacity for each record
00097         private FilePath                path;
00098 
00099         // basic capacity for each record
00100         private BlockSize               block;
00101 
00102         // where content is stored
00103         private FileConduit             file;
00104 
00105         // pointers to file records
00106         private Record[char[]]          map;
00107 
00108         // current file size
00109         private ulong                   fileSize;
00110 
00111         // current file usage
00112         private ulong                   waterLine;
00113 
00114         // supported block sizes
00115         static const BlockSize          EighthK  = {128-1},
00116                                         HalfK    = {512-1},
00117                                         OneK     = {1024*1-1},
00118                                         TwoK     = {1024*2-1},
00119                                         FourK    = {1024*4-1},
00120                                         EightK   = {1024*8-1},
00121                                         SixteenK = {1024*16-1},
00122                                         ThirtyTwoK = {1024*32-1},
00123                                         SixtyFourK = {1024*64-1};
00124 
00125         /**********************************************************************
00126 
00127                 Construct a FileBucket with the provided path and record-
00128                 size. Selecting a record size that roughly matches the 
00129                 serialized content will limit 'thrashing'.
00130 
00131         **********************************************************************/
00132 
00133         this (FilePath path, BlockSize block)
00134         {
00135                 this (path, block, 100);
00136         }
00137 
00138         /**********************************************************************
00139 
00140                 Construct a FileBucket with the provided path, record-size,
00141                 and inital record count. The latter causes records to be 
00142                 pre-allocated, saving a certain amount of growth activity.
00143                 Selecting a record size that roughly matches the serialized 
00144                 content will limit 'thrashing'. 
00145 
00146         **********************************************************************/
00147 
00148         this (FilePath path, BlockSize block, uint initialRecords)
00149         {
00150                 this.path = path;
00151                 this.block = block;
00152 
00153                 // open a storage file
00154                 file = new FileConduit (path, FileStyle.ReadWriteCreate);
00155 
00156                 // set initial file size (can be zero)
00157                 fileSize = initialRecords * block.capacity;
00158                 file.seek (fileSize);
00159                 file.truncate ();
00160         }
00161 
00162         /**********************************************************************
00163 
00164                 Return the block-size in use for this FileBucket
00165 
00166         **********************************************************************/
00167 
00168         int getBufferSize ()
00169         {
00170                 return block.capacity+1;
00171         }
00172 
00173         /**********************************************************************
00174         
00175                 Return where the FileBucket is located
00176 
00177         **********************************************************************/
00178 
00179         FilePath getFilePath ()
00180         {
00181                 return path;
00182         }
00183 
00184         /**********************************************************************
00185 
00186                 Return the currently populated size of this FileBucket
00187 
00188         **********************************************************************/
00189 
00190         synchronized ulong length ()
00191         {
00192                 return waterLine;
00193         }
00194 
00195         /**********************************************************************
00196 
00197                 Return the serialized data for the provided key. Returns
00198                 null if the key was not found.
00199 
00200         **********************************************************************/
00201 
00202         synchronized void[] get (char[] key)
00203         {
00204                 Record r = null;
00205 
00206                 if (key in map)
00207                    {
00208                    r = map [key];
00209                    return r.read (this);
00210                    }
00211                 return null;
00212         }
00213 
00214         /**********************************************************************
00215 
00216                 Remove the provided key from this FileBucket.
00217 
00218         **********************************************************************/
00219 
00220         synchronized void remove (char[] key)
00221         {
00222                 map.remove(key);
00223         }
00224 
00225         /**********************************************************************
00226 
00227                 Write a serialized block of data, and associate it with
00228                 the provided key. All keys must be unique, and it is the
00229                 responsibility of the programmer to ensure this. Reusing 
00230                 an existing key will overwrite previous data. 
00231 
00232                 Note that data is allowed to grow within the occupied 
00233                 bucket until it becomes larger than the allocated space.
00234                 When this happens, the data is moved to a larger bucket
00235                 at the file tail.
00236 
00237         **********************************************************************/
00238 
00239         synchronized void put (char[] key, void[] data)
00240         {
00241                 Record* r = key in map;
00242 
00243                 if (r is null)
00244                    {
00245                    Record rr = new Record ();
00246                    map [key] =  rr;
00247                    r = &rr;
00248                    }
00249                 r.write (this, data, block);
00250         }
00251 
00252         /**********************************************************************
00253 
00254                 Close this FileBucket -- all content is lost.
00255 
00256         **********************************************************************/
00257 
00258         synchronized void close ()
00259         {
00260                 if (file)
00261                    {
00262                    file.close ();
00263                    file = null;
00264                    map = null;
00265                    }
00266         }
00267 
00268         /**********************************************************************
00269 
00270                 Each Record takes up a number of 'pages' within the file. 
00271                 The size of these pages is determined by the BlockSize 
00272                 provided during FileBucket construction. Additional space
00273                 at the end of each block is potentially wasted, but enables 
00274                 content to grow in size without creating a myriad of holes.
00275 
00276         **********************************************************************/
00277 
00278         private class Record
00279         {
00280                 private ulong           offset;
00281                 private int             length,
00282                                         capacity = -1;
00283 
00284                 /**************************************************************
00285 
00286                 **************************************************************/
00287 
00288                 private static void eof (FileBucket bucket)
00289                 {
00290                         throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.toString()~"'");
00291                 }
00292 
00293                 /**************************************************************
00294 
00295                         This should be protected from thread-contention at
00296                         a higher level.
00297 
00298                 **************************************************************/
00299 
00300                 void[] read (FileBucket bucket)
00301                 {
00302                         void[] data = new ubyte [length];
00303 
00304                         bucket.file.seek (offset);
00305                         if (bucket.file.read (data) != length)
00306                             eof (bucket);
00307 
00308                         return data;
00309                 }
00310 
00311                 /**************************************************************
00312 
00313                         This should be protected from thread-contention at
00314                         a higher level.
00315 
00316                 **************************************************************/
00317 
00318                 void write (FileBucket bucket, void[] data, BlockSize block)
00319                 {
00320                         length = data.length;
00321 
00322                         // create new slot if we exceed capacity
00323                         if (length > capacity)
00324                             createBucket (bucket, length, block);
00325 
00326                         // locate to start of content 
00327                         bucket.file.seek (offset);
00328         
00329                         // write content
00330                         if (bucket.file.write (data) != length)
00331                             eof (bucket);
00332                 }
00333 
00334                 /**************************************************************
00335 
00336                 **************************************************************/
00337 
00338                 void createBucket (FileBucket bucket, int bytes, BlockSize block)
00339                 {
00340                         offset = bucket.waterLine;
00341                         capacity = (bytes + block.capacity) & ~block.capacity;
00342 
00343                         bucket.waterLine += capacity;
00344                         if (bucket.waterLine > bucket.fileSize)
00345                            {
00346                            // grow the filesize 
00347                            bucket.fileSize = bucket.waterLine * 2;
00348 
00349                            // expand the physical file size
00350                            bucket.file.seek (bucket.fileSize);
00351                            bucket.file.truncate ();
00352                            }
00353                 }
00354         }
00355 }
00356 
00357 

Generated on Sat Dec 24 17:28:32 2005 for Mango by  doxygen 1.4.0