00001 /******************************************************************************* 00002 00003 @file HttpClient.d 00004 00005 Copyright (c) 2004 Kris Bell 00006 00007 This software is provided 'as-is', without any express or implied 00008 warranty. In no event will the authors be held liable for damages 00009 of any kind arising from the use of this software. 00010 00011 Permission is hereby granted to anyone to use this software for any 00012 purpose, including commercial applications, and to alter it and/or 00013 redistribute it freely, subject to the following restrictions: 00014 00015 1. The origin of this software must not be misrepresented; you must 00016 not claim that you wrote the original software. If you use this 00017 software in a product, an acknowledgment within documentation of 00018 said product would be appreciated but is not required. 00019 00020 2. Altered source versions must be plainly marked as such, and must 00021 not be misrepresented as being the original software. 00022 00023 3. This notice may not be removed or altered from any distribution 00024 of the source. 00025 00026 4. Derivative works are permitted, but they must carry this notice 00027 in full and credit the original source. 00028 00029 00030 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 00031 00032 00033 @version Initial version, April 2004 00034 @author Kris 00035 00036 00037 Redirection handling guided via 00038 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html 00039 00040 *******************************************************************************/ 00041 00042 module mango.http.client.HttpClient; 00043 00044 private import mango.sys.System; 00045 00046 private import mango.convert.Atoi; 00047 00048 private import mango.io.Uri, 00049 mango.io.Token, 00050 mango.io.Buffer, 00051 mango.io.SocketConduit, 00052 mango.io.Exception, 00053 mango.io.Tokenizer, 00054 mango.io.DisplayWriter; 00055 00056 private import mango.http.server.HttpParams, 00057 mango.http.server.HttpHeaders, 00058 mango.http.server.HttpCookies, 00059 mango.http.server.HttpResponse; 00060 00061 00062 /******************************************************************************* 00063 00064 Supports the basic needs of a client making requests of an HTTP 00065 server. The following is an example of how this might be used: 00066 00067 @code 00068 // callback for client reader 00069 void sink (char[] content) 00070 { 00071 Stdout.put (content); 00072 } 00073 00074 // create client for a GET request 00075 auto HttpClient client = new HttpClient (HttpClient.Get, "http://www.yahoo.com"); 00076 00077 // make request 00078 client.open (); 00079 00080 // check return status for validity 00081 if (client.isResponseOK) 00082 { 00083 // extract content length 00084 int length = client.getResponseHeaders.getInt (HttpHeader.ContentLength, int.max); 00085 00086 // display all returned headers 00087 Stdout.put (client.getResponseHeaders); 00088 00089 // display remaining content 00090 client.read (&sink, length); 00091 } 00092 else 00093 Stderr.put (client.getResponse); 00094 00095 client.close (); 00096 @endcode 00097 00098 *******************************************************************************/ 00099 00100 class HttpClient 00101 { 00102 // this is struct rather than typedef to avoid compiler bugs 00103 private struct RequestMethod 00104 { 00105 final char[] name; 00106 } 00107 00108 // class members; there's a surprising amount of stuff here! 00109 private MutableUri uri; 00110 private Buffer input, 00111 output; 00112 private SocketConduit socket; 00113 private RequestMethod method; 00114 private InternetAddress address; 00115 private HttpMutableParams paramsOut; 00116 private HttpHeaders headersIn; 00117 private HttpMutableHeaders headersOut; 00118 private HttpMutableCookies cookiesOut; 00119 private ResponseLine responseLine; 00120 00121 // default to three second timeout on read operations ... 00122 private static final uint DefaultReadTimeout = System.Interval.Second * 3; 00123 00124 // use HTTP v1.0 ? 00125 private static const char[] DefaultHttpVersion = "HTTP/1.0"; 00126 00127 // standard set of request methods ... 00128 static const RequestMethod Get = {"GET"}, 00129 Put = {"PUT"}, 00130 Head = {"HEAD"}, 00131 Post = {"POST"}, 00132 Trace = {"TRACE"}, 00133 Delete = {"DELETE"}, 00134 Options = {"OPTIONS"}, 00135 Connect = {"CONNECT"}; 00136 00137 /*********************************************************************** 00138 00139 Create a client for the given URL. The argument should be 00140 fully qualified with an "http:" or "https:" scheme, or an 00141 explicit port should be provided. 00142 00143 ***********************************************************************/ 00144 00145 this (RequestMethod method, char[] url) 00146 { 00147 this (method, new MutableUri(url)); 00148 } 00149 00150 /*********************************************************************** 00151 00152 Create a client with the provided Uri instance. The Uri should 00153 be fully qualified with an "http:" or "https:" scheme, or an 00154 explicit port should be provided. 00155 00156 ***********************************************************************/ 00157 00158 this (RequestMethod method, MutableUri uri) 00159 { 00160 this.uri = uri; 00161 this.method = method; 00162 00163 responseLine = new ResponseLine (); 00164 headersIn = new HttpHeaders (); 00165 00166 paramsOut = new HttpMutableParams (new Buffer (1024 * 1)); 00167 headersOut = new HttpMutableHeaders (new Buffer (1024 * 4)); 00168 cookiesOut = new HttpMutableCookies (headersOut); 00169 00170 // decode the host name (may take a second or two) 00171 address = new InternetAddress (uri.getHost(), uri.getValidPort()); 00172 } 00173 00174 /*********************************************************************** 00175 00176 Get the current input headers, as returned by the host request. 00177 00178 ***********************************************************************/ 00179 00180 HttpHeaders getResponseHeaders() 00181 { 00182 return headersIn; 00183 } 00184 00185 /*********************************************************************** 00186 00187 Gain access to the request headers. Use this to add whatever 00188 headers are required for a request. 00189 00190 ***********************************************************************/ 00191 00192 HttpMutableHeaders getRequestHeaders() 00193 { 00194 return headersOut; 00195 } 00196 00197 /*********************************************************************** 00198 00199 Gain access to the request parameters. Use this to add x=y 00200 style parameters to the request. These will be appended to 00201 the request assuming the original Uri does not contain any 00202 of its own. 00203 00204 ***********************************************************************/ 00205 00206 HttpMutableParams getRequestParams() 00207 { 00208 return paramsOut; 00209 } 00210 00211 /*********************************************************************** 00212 00213 Return the Uri associated with this client 00214 00215 ***********************************************************************/ 00216 00217 Uri getUri() 00218 { 00219 return uri; 00220 } 00221 00222 /*********************************************************************** 00223 00224 Return the response-line for the latest request. This takes 00225 the form of "version status reason" as defined in the HTTP 00226 RFC. 00227 00228 ***********************************************************************/ 00229 00230 ResponseLine getResponse() 00231 { 00232 return responseLine; 00233 } 00234 00235 /*********************************************************************** 00236 00237 Return the HTTP status code set by the remote server 00238 00239 ***********************************************************************/ 00240 00241 int getStatus() 00242 { 00243 return responseLine.getStatus(); 00244 } 00245 00246 /*********************************************************************** 00247 00248 Return whether the response was OK or not 00249 00250 ***********************************************************************/ 00251 00252 bool isResponseOK() 00253 { 00254 return cast(bool) (getStatus() == HttpResponseCode.OK); 00255 } 00256 00257 /*********************************************************************** 00258 00259 Add a cookie to the outgoing headers 00260 00261 ***********************************************************************/ 00262 00263 void addCookie (Cookie cookie) 00264 { 00265 cookiesOut.add (cookie); 00266 } 00267 00268 /*********************************************************************** 00269 00270 Close all resources used by a request. You must invoke this 00271 between successive open() calls. 00272 00273 ***********************************************************************/ 00274 00275 void close () 00276 { 00277 if (socket) 00278 { 00279 socket.shutdown (); 00280 socket.close (); 00281 socket = null; 00282 } 00283 } 00284 00285 /*********************************************************************** 00286 00287 Reset the client such that it is ready for a new request. 00288 00289 ***********************************************************************/ 00290 00291 void reset () 00292 { 00293 headersIn.reset(); 00294 headersOut.reset(); 00295 paramsOut.reset(); 00296 } 00297 00298 /*********************************************************************** 00299 00300 Overridable method to create a Socket. You may find a need 00301 to override this for some purpose; perhaps to add input or 00302 output filters. 00303 00304 ***********************************************************************/ 00305 00306 protected SocketConduit createSocket () 00307 { 00308 return new SocketConduit (true); 00309 } 00310 00311 /*********************************************************************** 00312 00313 Make a request for the resource specified via the constructor, 00314 using a callback for pumping additional data to the host. This 00315 defaults to a three-second timeout period. The return value 00316 represents the input buffer, from which all returned headers 00317 and content may be accessed. 00318 00319 ***********************************************************************/ 00320 00321 IBuffer open (IWritable pump) 00322 { 00323 return open (DefaultReadTimeout, pump); 00324 } 00325 00326 /*********************************************************************** 00327 00328 Make a request for the resource specified via the constructor, 00329 using the specified timeout period (in milli-seconds).The 00330 return value represents the input buffer, from which all 00331 returned headers and content may be accessed. 00332 00333 ***********************************************************************/ 00334 00335 IBuffer open (uint timeout = DefaultReadTimeout) 00336 { 00337 return open (timeout, null); 00338 } 00339 00340 /*********************************************************************** 00341 00342 Make a request for the resource specified via the constructor 00343 using the specified timeout period (in micro-seconds), and a 00344 user-defined callback for pumping additional data to the host. 00345 The callback would be used when uploading data during a 'put' 00346 operation (or equivalent). The return value represents the 00347 input buffer, from which all returned headers and content may 00348 be accessed. 00349 00350 ***********************************************************************/ 00351 00352 IBuffer open (uint timeout, IWritable pump) 00353 { 00354 return open (timeout, pump, method); 00355 } 00356 00357 /*********************************************************************** 00358 00359 Make a request for the resource specified via the constructor 00360 using the specified timeout period (in micro-seconds), and a 00361 user-defined callback for pumping additional data to the host. 00362 The callback would be used when uploading data during a 'put' 00363 operation (or equivalent). The return value represents the 00364 input buffer, from which all returned headers and content may 00365 be accessed. 00366 00367 ***********************************************************************/ 00368 00369 private IBuffer open (uint timeout, IWritable pump, RequestMethod method) 00370 { 00371 IWriter emit; 00372 00373 // create socket, and connect it 00374 socket = createSocket; 00375 socket.setTimeout (timeout); 00376 socket.connect (address); 00377 00378 // create buffers for input and output 00379 input = new Buffer (socket); 00380 output = new Buffer (socket); 00381 00382 // bind a writer to the socket 00383 emit = new DisplayWriter (output); 00384 00385 // setup a Host header 00386 if (headersOut.get (HttpHeader.Host, null) is null) 00387 headersOut.add (HttpHeader.Host, uri.getHost); 00388 00389 // setup request path 00390 char[] path = uri.getPath; 00391 if (path.length is 0) 00392 path = "/"; 00393 00394 // attach/extend query parameters if user has added some 00395 char[] query = uri.extendQuery (paramsOut.toOutputString); 00396 00397 // format request 00398 emit (method.name) 00399 (' ') 00400 (path); 00401 00402 if (query.length) 00403 emit ('?') (query); 00404 00405 emit (' ') 00406 (DefaultHttpVersion) 00407 (CR) 00408 (headersOut) 00409 (CR); 00410 00411 // user has more data to send? 00412 if (pump) 00413 pump.write (emit); 00414 00415 // send entire request 00416 emit.flush (); 00417 00418 // Token for initial parsing of input header lines 00419 CompositeToken line = new CompositeToken (Tokenizers.line, input); 00420 00421 // skip any blank lines 00422 while (line.next() && line.getLength() == 0) 00423 {} 00424 00425 // is this a bogus request? 00426 if (input.readable == 0) 00427 responseLine.error ("truncated response"); 00428 00429 // read response line 00430 responseLine.parse (line.toString); 00431 00432 // parse headers and go home 00433 headersIn.reset (); 00434 headersIn.parse (input); 00435 00436 switch (responseLine.getStatus) 00437 { 00438 case HttpResponseCode.SeeOther: 00439 case HttpResponseCode.MovedPermanently: 00440 case HttpResponseCode.MovedTemporarily: 00441 case HttpResponseCode.TemporaryRedirect: 00442 socket.close(); 00443 uri.parse (headersIn.get (HttpHeader.Location)); 00444 if (method is Get || method is Head) 00445 return open (timeout, pump, method); 00446 else 00447 if (method is Post) 00448 return redirectPost (timeout, pump, responseLine.getStatus); 00449 else 00450 responseLine.error ("unexpected redirect for method "~method.name); 00451 default: 00452 break; 00453 } 00454 return input; 00455 } 00456 00457 /*********************************************************************** 00458 00459 ***********************************************************************/ 00460 00461 void read (void delegate (char[]) sink, long length = long.max) 00462 { 00463 do { 00464 length -= input.readable; 00465 sink (input.toString); 00466 input.clear (); 00467 } while (length > 0 && input.fill() != socket.Eof); 00468 } 00469 00470 /*********************************************************************** 00471 00472 Handle redirection of Post 00473 00474 Guidance for this default behaviour came from this page: 00475 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html 00476 00477 ***********************************************************************/ 00478 00479 IBuffer redirectPost (uint timeout, IWritable pump, int status) 00480 { 00481 switch (status) 00482 { 00483 case HttpResponseCode.SeeOther: 00484 case HttpResponseCode.MovedTemporarily: 00485 return open (timeout, null, Get); 00486 00487 case HttpResponseCode.MovedPermanently: 00488 case HttpResponseCode.TemporaryRedirect: 00489 if (canRepost (status)) 00490 return open (timeout, pump, this.method); 00491 default: 00492 } 00493 responseLine.error ("Illegal redirection of Post"); 00494 return null; 00495 } 00496 00497 /*********************************************************************** 00498 00499 Handle user-notification of Post redirection. This should 00500 be overridden appropriately. 00501 00502 Guidance for this default behaviour came from this page: 00503 http://ppewww.ph.gla.ac.uk/~flavell/www/post-redirect.html 00504 00505 ***********************************************************************/ 00506 00507 bool canRepost (uint status) 00508 { 00509 return false; 00510 } 00511 } 00512 00513 00514 /****************************************************************************** 00515 00516 Class to represent an HTTP response-line 00517 00518 ******************************************************************************/ 00519 00520 private class ResponseLine : IWritable 00521 { 00522 private Buffer buf; 00523 private BoundToken token; 00524 private char[] code, 00525 reason, 00526 vershion; 00527 private int status; 00528 00529 /********************************************************************** 00530 00531 Construct response-line with pre-allocated buffer & token 00532 00533 **********************************************************************/ 00534 00535 this () 00536 { 00537 buf = new Buffer; 00538 00539 // bind tokens to a space-tokenizer 00540 token = new BoundToken (new SpaceTokenizer()); 00541 } 00542 00543 /********************************************************************** 00544 00545 Parse the the given response-line into its constituent 00546 components. 00547 00548 **********************************************************************/ 00549 00550 void parse (char[] line) 00551 { 00552 // setup our buffer with the request 00553 buf.setValidContent (line); 00554 00555 if (token.next (buf)) 00556 { 00557 vershion = token.toString(); 00558 if (token.next (buf)) 00559 { 00560 code = token.toString(); 00561 reason = line[buf.getPosition..line.length]; 00562 00563 status = cast(int) Atoi.convert (code); 00564 if (status > 0) 00565 return; 00566 } 00567 } 00568 error ("Invalid HTTP response-line: '"~line~"'"); 00569 } 00570 00571 /********************************************************************** 00572 00573 Return HTTP version 00574 00575 **********************************************************************/ 00576 00577 char[] getVersion () 00578 { 00579 return vershion; 00580 } 00581 00582 /********************************************************************** 00583 00584 Return reason text 00585 00586 **********************************************************************/ 00587 00588 char[] getReason () 00589 { 00590 return reason; 00591 } 00592 00593 /********************************************************************** 00594 00595 Return status integer 00596 00597 **********************************************************************/ 00598 00599 int getStatus () 00600 { 00601 return status; 00602 } 00603 00604 /********************************************************************** 00605 00606 convert back to original string 00607 00608 **********************************************************************/ 00609 00610 override char[] toString () 00611 { 00612 return vershion~" "~code~" "~reason; 00613 } 00614 00615 /********************************************************************** 00616 00617 Output the string via the given writer 00618 00619 **********************************************************************/ 00620 00621 void write (IWriter writer) 00622 { 00623 writer.put(toString).cr(); 00624 } 00625 00626 /********************************************************************** 00627 00628 throw an exception 00629 00630 **********************************************************************/ 00631 00632 static void error (char[] msg) 00633 { 00634 throw new IOException (msg); 00635 } 00636 } 00637 00638