00001 /******************************************************************************* 00002 00003 @file FileConduit.d 00004 00005 Copyright (c) 2004 Kris Bell 00006 00007 This software is provided 'as-is', without any express or implied 00008 warranty. In no event will the authors be held liable for damages 00009 of any kind arising from the use of this software. 00010 00011 Permission is hereby granted to anyone to use this software for any 00012 purpose, including commercial applications, and to alter it and/or 00013 redistribute it freely, subject to the following restrictions: 00014 00015 1. The origin of this software must not be misrepresented; you must 00016 not claim that you wrote the original software. If you use this 00017 software in a product, an acknowledgment within documentation of 00018 said product would be appreciated but is not required. 00019 00020 2. Altered source versions must be plainly marked as such, and must 00021 not be misrepresented as being the original software. 00022 00023 3. This notice may not be removed or altered from any distribution 00024 of the source. 00025 00026 4. Derivative works are permitted, but they must carry this notice 00027 in full and credit the original source. 00028 00029 00030 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 00031 00032 00033 @version Initial version, March 2004 00034 @author Kris 00035 John Reimer 00036 Anders F Bjorklund (Darwin patches) 00037 Chris Sauls (Win95 file support) 00038 00039 *******************************************************************************/ 00040 00041 module mango.io.FileConduit; 00042 00043 public import mango.base.System; 00044 00045 public import mango.io.Buffer, 00046 mango.io.FilePath, 00047 mango.io.FileProxy, 00048 mango.io.FileStyle; 00049 00050 private import mango.io.Conduit, 00051 mango.io.Exception, 00052 mango.io.ConduitStyle; 00053 00054 /******************************************************************************* 00055 00056 Bring in native Unix functions. Note that the functions read() 00057 and close() will actually hide the inherited Conduit.read() and 00058 Resource.close() if the import is placed within the body of the 00059 FileConduit class. Apparently this behaviour is 'by design', so 00060 fair warning to those who might expect otherwise. 00061 00062 *******************************************************************************/ 00063 00064 version (Posix) 00065 { 00066 version (linux) 00067 { 00068 private import std.c.linux.linux; 00069 alias std.c.linux.linux posix; 00070 } 00071 00072 version (darwin) 00073 { 00074 private import std.c.darwin.darwin; 00075 alias std.c.darwin.darwin posix; 00076 } 00077 00078 private extern (C) int ftruncate (int, int); 00079 } 00080 00081 /******************************************************************************* 00082 00083 Bring in native Windows functions 00084 00085 *******************************************************************************/ 00086 00087 version (Win32) 00088 { 00089 private import std.c.windows.windows; 00090 00091 private extern (Windows) 00092 { 00093 HANDLE GetStdHandle (DWORD); 00094 BOOL SetEndOfFile (HANDLE); 00095 00096 BOOL UnmapViewOfFile (LPCVOID); 00097 BOOL FlushViewOfFile (LPCVOID, DWORD); 00098 LPVOID MapViewOfFile (HANDLE, DWORD, DWORD, DWORD, DWORD); 00099 HANDLE CreateFileMappingA (HANDLE, LPSECURITY_ATTRIBUTES, DWORD, DWORD, DWORD, LPCTSTR); 00100 } 00101 } 00102 00103 00104 /******************************************************************************* 00105 00106 Implements a means of reading and writing a generic file. Conduits 00107 are the primary means of accessing external data, and are usually 00108 routed through a Buffer. File conduit extends the generic conduit 00109 by providing file-specific methods to set the file size, seek to a 00110 specific file position, and so on. Also provided is a factory for 00111 create a memory-mapped Buffer upon a file. 00112 00113 Serial input and output is straightforward. In this example we 00114 copy a file directly to the console: 00115 00116 @code 00117 // open a file for reading 00118 FileConduit fc = new FileConduit ("test.txt"); 00119 00120 // stream directly to console 00121 Stdio.stdout.copy (fc); 00122 @endcode 00123 00124 And here we copy one file to another: 00125 00126 @code 00127 // open a file for reading 00128 FileConduit from = new FileConduit ("test.txt"); 00129 00130 // open another for writing 00131 FileConduit to = new FileConduit ("copy.txt", FileStyle.WriteTruncate); 00132 00133 // copy file 00134 to.copy (from); 00135 @endcode 00136 00137 FileConduit can just as easily handle random IO. Here we see how 00138 a Reader and Writer are used to perform simple input and output: 00139 00140 @code 00141 // open a file for reading 00142 FileConduit fc = new FileConduit ("random.bin", FileStyle.ReadWriteCreate); 00143 00144 // construct (binary) reader & writer upon this conduit 00145 Reader r = new Reader(fc); 00146 Writer w = new Writer(fc); 00147 00148 int x=10, y=20; 00149 00150 // write some data 00151 w.put(x).put(y); 00152 00153 // flush output since IO is buffered 00154 w.flush(); 00155 00156 // rewind to file start 00157 fc.seek (0); 00158 00159 // read data back again, but swap destinations 00160 r.get(y).get(x); 00161 00162 assert (y==10); 00163 assert (x==20); 00164 00165 fc.close(); 00166 @endcode 00167 00168 See FilePath, FileProxy, and FileSystem for additional functionality 00169 related to file manipulation. Note that classes CompositeWriter and 00170 CompositeReader may be more appropriate for the writing & reading of 00171 structured data. Doxygen has a hard time with D version() statements, 00172 so part of this class is documented within FileConduit::VersionWin32 00173 instead. 00174 00175 Compile with -version=Win32SansUnicode to enable Win95 & Win32s file 00176 support. 00177 00178 *******************************************************************************/ 00179 00180 class FileConduit : Conduit, ISeekable 00181 { 00182 // the file we're working with 00183 private FilePath path; 00184 00185 // expose conduit copy() method also (alias is truly non-intuitive ...) 00186 alias Conduit.copy copy; 00187 00188 /*********************************************************************** 00189 00190 Create a FileConduit with the provided path and style. 00191 00192 ***********************************************************************/ 00193 00194 this (char[] name, FileStyle style = FileStyle.ReadExisting) 00195 { 00196 this (new FilePath(name), style); 00197 } 00198 00199 /*********************************************************************** 00200 00201 Create a FileConduit from the provided proxy and style. 00202 00203 ***********************************************************************/ 00204 00205 this (FileProxy proxy, FileStyle style = FileStyle.ReadExisting) 00206 { 00207 this (proxy.getPath(), style); 00208 } 00209 00210 /*********************************************************************** 00211 00212 Create a FileConduit with the provided path and style. 00213 00214 ***********************************************************************/ 00215 00216 this (FilePath path, FileStyle style = FileStyle.ReadExisting) 00217 { 00218 // say we're seekable 00219 super (style, true); 00220 00221 // remember who we are 00222 this.path = path; 00223 00224 // open the file 00225 _open (style); 00226 00227 // bump lock count 00228 acquire(); 00229 } 00230 00231 /*********************************************************************** 00232 00233 Create a FileConduit on the provided FileDevice. This is 00234 strictly for adapting existing devices such as Stdout and 00235 friends. 00236 00237 ***********************************************************************/ 00238 00239 package this (FileDevice device) 00240 { 00241 // say we're not seekable 00242 super (device, false); 00243 00244 // devices don't have a path (bug?) 00245 this.path = null; 00246 00247 // open the file 00248 _reopen (device); 00249 00250 // bump lock count 00251 acquire(); 00252 } 00253 00254 /*********************************************************************** 00255 00256 Callback to close the file. This is invoked from the Resource 00257 base-class when the resource is being closed. 00258 00259 ***********************************************************************/ 00260 00261 protected override void closure () 00262 { 00263 super.closure (); 00264 _close (); 00265 } 00266 00267 /*********************************************************************** 00268 00269 Return the FilePath used by this file. 00270 00271 ***********************************************************************/ 00272 00273 FilePath getPath () 00274 { 00275 return path; 00276 } 00277 00278 /*********************************************************************** 00279 00280 Move the file position to the given offset (from the file 00281 start) and return the adjusted postion. 00282 00283 ***********************************************************************/ 00284 00285 long seek (long offset) 00286 { 00287 return seek (offset, SeekAnchor.Begin); 00288 } 00289 00290 /*********************************************************************** 00291 00292 Return the current file position. 00293 00294 ***********************************************************************/ 00295 00296 long getPosition () 00297 { 00298 return seek (0, SeekAnchor.Current); 00299 } 00300 00301 00302 /*********************************************************************** 00303 00304 Create a Buffer of the default FileConduit size, and 00305 associate it with this conduit 00306 00307 ***********************************************************************/ 00308 00309 override IBuffer createBuffer() 00310 { 00311 const int DefaultFileBufferSize = 1024 * 16; 00312 00313 IBuffer buffer = new Buffer (DefaultFileBufferSize); 00314 00315 buffer.setConduit (this); 00316 return buffer; 00317 } 00318 00319 /*********************************************************************** 00320 00321 Return the total length of this file. 00322 00323 ***********************************************************************/ 00324 00325 long length () 00326 { 00327 long pos, 00328 ret; 00329 00330 pos = getPosition (); 00331 ret = seek (0, SeekAnchor.End); 00332 seek (pos); 00333 return ret; 00334 } 00335 00336 /*********************************************************************** 00337 00338 Transfer the content of another file to this one. Returns a 00339 reference to this class on success, or throws an IOException 00340 upon failure. 00341 00342 ***********************************************************************/ 00343 00344 FileConduit copy (FilePath source) 00345 { 00346 auto FileConduit fc = new FileConduit (source); 00347 super.copy (fc); 00348 return this; 00349 } 00350 00351 /*********************************************************************** 00352 00353 Windows-specific code 00354 00355 ***********************************************************************/ 00356 00357 version(Win32) 00358 { 00359 private HANDLE handle; 00360 private bool console, 00361 appending; 00362 00363 /*************************************************************** 00364 00365 Throw an IOException noting the last error 00366 00367 ***************************************************************/ 00368 00369 private void exception () 00370 { 00371 char[] name = (path) ? path.toString : "console"; 00372 00373 throw new IOException (name ~ ": " ~ System.error); 00374 } 00375 00376 /*************************************************************** 00377 00378 Gain access to the standard IO handles (console etc). 00379 00380 ***************************************************************/ 00381 00382 private void _reopen (FileDevice device) 00383 { 00384 static const DWORD[] id = [ 00385 cast(DWORD) -10, 00386 cast(DWORD) -11, 00387 cast(DWORD) -12 00388 ]; 00389 static const char[][] f = [ 00390 "CONIN$\0", 00391 "CONOUT$\0", 00392 "CONOUT$\0" 00393 ]; 00394 00395 handle = GetStdHandle (id[device.id]); 00396 if (! handle) 00397 handle = CreateFileA (f[device.id], 00398 GENERIC_READ | GENERIC_WRITE, 00399 FILE_SHARE_READ | FILE_SHARE_WRITE, 00400 null, OPEN_EXISTING, 0, null); 00401 if (! handle) 00402 exception(); 00403 console = true; 00404 } 00405 00406 /*************************************************************** 00407 00408 Open a file with the provided style. 00409 00410 ***************************************************************/ 00411 00412 private void _open (FileStyle style) 00413 { 00414 DWORD attr, 00415 share, 00416 access, 00417 create; 00418 00419 alias DWORD[] Flags; 00420 00421 static const Flags Access = 00422 [ 00423 0, // invalid 00424 GENERIC_READ, 00425 GENERIC_WRITE, 00426 GENERIC_READ | GENERIC_WRITE, 00427 ]; 00428 00429 static const Flags Create = 00430 [ 00431 OPEN_EXISTING, // must exist 00432 CREATE_ALWAYS, // create always 00433 TRUNCATE_EXISTING, // must exist 00434 CREATE_ALWAYS, // (for appending) 00435 ]; 00436 00437 static const Flags Share = 00438 [ 00439 FILE_SHARE_READ, 00440 FILE_SHARE_WRITE, 00441 FILE_SHARE_READ | FILE_SHARE_WRITE, 00442 ]; 00443 00444 static const Flags Attr = 00445 [ 00446 0, 00447 FILE_FLAG_RANDOM_ACCESS, 00448 FILE_FLAG_SEQUENTIAL_SCAN, 00449 0, 00450 FILE_FLAG_WRITE_THROUGH, 00451 ]; 00452 00453 attr = Attr[style.cache]; 00454 share = Share[style.share]; 00455 create = Create[style.open]; 00456 access = Access[style.access]; 00457 00458 version (Win32SansUnicode) 00459 handle = CreateFileA (path.toUtf8, access, share, 00460 null, create, 00461 attr | FILE_ATTRIBUTE_NORMAL, 00462 cast(HANDLE) null); 00463 else 00464 handle = CreateFileW (path.toUtf16, access, share, 00465 null, create, 00466 attr | FILE_ATTRIBUTE_NORMAL, 00467 cast(HANDLE) null); 00468 00469 if (handle == INVALID_HANDLE_VALUE) 00470 exception (); 00471 00472 // move to end of file? 00473 if (style.open == style.Open.Append) 00474 appending = true; 00475 } 00476 00477 /*************************************************************** 00478 00479 Close the underlying file 00480 00481 ***************************************************************/ 00482 00483 private void _close () 00484 { 00485 if (! CloseHandle (handle)) 00486 exception (); 00487 } 00488 00489 /*************************************************************** 00490 00491 Read a chunk of bytes from the file into the provided 00492 array (typically that belonging to an IBuffer) 00493 00494 ***************************************************************/ 00495 00496 protected override int reader (void[] dst) 00497 { 00498 DWORD read; 00499 void *p = dst; 00500 00501 if (! ReadFile (handle, p, dst.length, &read, null)) 00502 exception (); 00503 00504 if (read == 0 && dst.length > 0) 00505 return Eof; 00506 return read; 00507 } 00508 00509 /*************************************************************** 00510 00511 Write a chunk of bytes to the file from the provided 00512 array (typically that belonging to an IBuffer) 00513 00514 ***************************************************************/ 00515 00516 protected override int writer (void[] src) 00517 { 00518 const int Max = 0x7fff; 00519 DWORD written = src.length; 00520 00521 // Win32 console has problems with content over 00522 // 32KB in length, so handle it specifically 00523 if (console && written > Max) 00524 { 00525 DWORD len = written; 00526 do { 00527 DWORD i = len; 00528 if (i > Max) 00529 i = Max; 00530 if (! WriteFile (handle, src, i, &i, null)) 00531 exception (); 00532 src = src [i .. length]; 00533 len -= i; 00534 } while (len); 00535 } 00536 else 00537 { 00538 // try to emulate the Unix O_APPEND mode 00539 if (appending) 00540 SetFilePointer (handle, 0, null, SeekAnchor.End); 00541 00542 if (! WriteFile (handle, src, src.length, &written, null)) 00543 exception (); 00544 } 00545 00546 return written; 00547 } 00548 00549 /*************************************************************** 00550 00551 Set the file size to be that of the current seek 00552 position. The file must be writable for this to 00553 succeed. 00554 00555 ***************************************************************/ 00556 00557 void truncate () 00558 { 00559 // must have Generic_Write access 00560 if (! SetEndOfFile (handle)) 00561 exception (); 00562 } 00563 00564 /*************************************************************** 00565 00566 Set the file seek position to the specified offset 00567 from the given anchor. 00568 00569 ***************************************************************/ 00570 00571 long seek (long offset, SeekAnchor anchor) 00572 { 00573 LONG high = (offset >> 32); 00574 long result = SetFilePointer (handle, cast(LONG) offset, 00575 &high, anchor); 00576 00577 if (result == -1 && 00578 GetLastError() != ERROR_SUCCESS) 00579 exception (); 00580 00581 return result + (cast(long) high << 32); 00582 } 00583 } 00584 00585 00586 /*********************************************************************** 00587 00588 Unix-specific code. Note that some methods are 32bit only 00589 00590 ***********************************************************************/ 00591 00592 version (Posix) 00593 { 00594 private int handle = -1; 00595 00596 /*************************************************************** 00597 00598 Throw an IOException noting the last error 00599 00600 ***************************************************************/ 00601 00602 private void exception () 00603 { 00604 throw new IOException (path.toString() ~ ": " ~ 00605 System.error); 00606 } 00607 00608 /*************************************************************** 00609 00610 Gain access to the standard IO handles (console etc). 00611 00612 ***************************************************************/ 00613 00614 private void _reopen (FileDevice device) 00615 { 00616 handle = device.id; 00617 } 00618 00619 /*************************************************************** 00620 00621 Open a file with the provided style. 00622 00623 ***************************************************************/ 00624 00625 private void _open (FileStyle style) 00626 { 00627 int share, 00628 access; 00629 00630 alias int[] Flags; 00631 00632 static const Flags Access = 00633 [ 00634 0, // invalid 00635 O_RDONLY, 00636 O_WRONLY, 00637 O_RDWR, 00638 ]; 00639 00640 static const Flags Create = 00641 [ 00642 0, // open existing 00643 O_CREAT, // create always 00644 O_TRUNC, // must exist 00645 O_APPEND | O_CREAT, 00646 ]; 00647 00648 // this is not the same as Windows sharing, 00649 // but it's perhaps a reasonable approximation 00650 static const Flags Share = 00651 [ 00652 0640, // read access 00653 0620, // write access 00654 0660, // read & write 00655 ]; 00656 00657 share = Share[style.share]; 00658 access = Access[style.access] | Create[style.open]; 00659 00660 handle = posix.open (path.toUtf8, access, share); 00661 if (handle == -1) 00662 exception (); 00663 } 00664 00665 /*************************************************************** 00666 00667 Close the underlying file 00668 00669 ***************************************************************/ 00670 00671 private void _close () 00672 { 00673 if (posix.close (handle) == -1) 00674 exception (); 00675 } 00676 00677 /*************************************************************** 00678 00679 Read a chunk of bytes from the file into the provided 00680 array (typically that belonging to an IBuffer) 00681 00682 ***************************************************************/ 00683 00684 protected override int reader (void[] dst) 00685 { 00686 int read = posix.read (handle, dst, dst.length); 00687 if (read == -1) 00688 exception (); 00689 else 00690 if (read == 0 && dst.length > 0) 00691 return Eof; 00692 return read; 00693 } 00694 00695 /*************************************************************** 00696 00697 Write a chunk of bytes to the file from the provided 00698 array (typically that belonging to an IBuffer) 00699 00700 ***************************************************************/ 00701 00702 protected override int writer (void[] src) 00703 { 00704 int written = posix.write (handle, src, src.length); 00705 if (written == -1) 00706 exception (); 00707 return written; 00708 } 00709 00710 /*************************************************************** 00711 00712 32bit only ... 00713 00714 Set the file size to be that of the current seek 00715 position. The file must be writable for this to 00716 succeed. 00717 00718 ***************************************************************/ 00719 00720 void truncate () 00721 { 00722 // set filesize to be current seek-position 00723 if (ftruncate (handle, getPosition()) == -1) 00724 exception (); 00725 } 00726 00727 /*************************************************************** 00728 00729 32bit only ... 00730 00731 Set the file seek position to the specified offset 00732 from the given anchor. 00733 00734 ***************************************************************/ 00735 00736 long seek (long offset, SeekAnchor anchor) 00737 { 00738 int result = posix.lseek (handle, offset, anchor); 00739 if (result == -1) 00740 exception (); 00741 return result; 00742 } 00743 } 00744 } 00745 00746 00747 /******************************************************************************* 00748 00749 Class to handle memory-mapped files 00750 00751 *******************************************************************************/ 00752 00753 class MappedFile : MappedBuffer 00754 { 00755 version (Windows) 00756 { 00757 private FileConduit host; 00758 private void* base; 00759 private HANDLE mmFile; 00760 00761 /*************************************************************** 00762 00763 Construct a MappedBuffer upon the given FileConduit. 00764 One should set the file size using seek() & truncate() 00765 to setup the available working space. 00766 00767 ***************************************************************/ 00768 00769 this (FileConduit host) 00770 { 00771 this.host = host; 00772 00773 // can only do 32bit mapping on 32bit platform 00774 long size = host.length; 00775 ConduitStyle style = host.getStyle; 00776 00777 DWORD flags = PAGE_READWRITE; 00778 if (style.access == ConduitStyle.Access.Read) 00779 flags = PAGE_READONLY; 00780 00781 mmFile = CreateFileMappingA (host.handle, null, flags, 0, 0, null); 00782 if (mmFile is null) 00783 host.exception (); 00784 00785 flags = FILE_MAP_WRITE; 00786 if (style.access == ConduitStyle.Access.Read) 00787 flags = FILE_MAP_READ; 00788 00789 base = MapViewOfFile (mmFile, flags, 0, 0, 0); 00790 if (base is null) 00791 host.exception (); 00792 00793 void[] mem = base[0..size]; 00794 setValidContent (mem); 00795 } 00796 00797 /*************************************************************** 00798 00799 Ensure this is closed when GC'd 00800 00801 ***************************************************************/ 00802 00803 ~this () 00804 { 00805 close(); 00806 } 00807 00808 /*************************************************************** 00809 00810 Close this mapped buffer 00811 00812 ***************************************************************/ 00813 00814 void close () 00815 { 00816 if (base) 00817 UnmapViewOfFile (base); 00818 00819 if (mmFile) 00820 CloseHandle (mmFile); 00821 00822 mmFile = null; 00823 base = null; 00824 } 00825 00826 /*************************************************************** 00827 00828 Flush dirty content out to the drive. This 00829 fails with error 33 if the file content is 00830 virgin. Opening a file for ReadWriteExists 00831 followed by a flush() will cause this. 00832 00833 ***************************************************************/ 00834 00835 void flush () 00836 { 00837 // flush all dirty pages 00838 if (! FlushViewOfFile (base, 0)) 00839 host.exception(); 00840 } 00841 } 00842 00843 00844 version (Posix) 00845 { 00846 // not yet supported 00847 private this (){}; 00848 } 00849 } 00850 00851 00852 /******************************************************************************* 00853 00854 Class used to wrap an existing file-oriented handle, such as Stdout 00855 and its cohorts. 00856 00857 *******************************************************************************/ 00858 00859 class FileDevice : ConduitStyle 00860 { 00861 private uint id; 00862 00863 this (uint id, Access access) 00864 in { 00865 assert (id < 3); 00866 } 00867 body 00868 { 00869 super (access); 00870 this.id = id; 00871 } 00872 } 00873 00874