Main Page | Modules | Class Hierarchy | Alphabetical List | Class List | Directories | File List | Class Members | File Members | Related Pages

wvhttppool.h

00001 /* -*- Mode: C++ -*-
00002  * Worldvisions Weaver Software:
00003  *   Copyright (C) 1997-2002 Net Integration Technologies, Inc.
00004  *
00005  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
00006  * 
00007  * Just create a WvHttpPool object, add it to your list, and use pool.addurl()
00008  * to get a WvStream* that gives you the file you requested.
00009  */ 
00010 #ifndef __WVHTTPPOOL_H
00011 #define __WVHTTPPOOL_H
00012 
00013 #include "ftpparse.h"
00014 #include "wvurl.h"
00015 #include "wvistreamlist.h"
00016 #include "wvstreamclone.h"
00017 #include "wvlog.h"
00018 #include "wvhashtable.h"
00019 #include "wvhttp.h"
00020 #include "wvbufstream.h"
00021 #include "wvbuf.h"
00022 #include "wvcont.h"
00023 
00024 class WvBufUrlStream;
00025 class WvUrlStream;
00026 class WvHttpStream;
00027 
00028 static const WvString DEFAULT_ANON_PW("weasels@");
00029 
00030 class WvUrlRequest
00031 {
00032 public:
00033     WvUrl url;
00034     WvString headers;
00035     WvUrlStream *instream;
00036     WvBufUrlStream *outstream;
00037     WvStream *putstream;
00038 
00039     bool pipeline_test;
00040     bool inuse;
00041     bool is_dir;
00042     bool create_dirs;
00043     WvString method;
00044     
00045     WvUrlRequest(WvStringParm _url, WvStringParm _method, WvStringParm _headers,
00046                  WvStream *content_source, bool _create_dirs, bool _pipeline_test);
00047     ~WvUrlRequest();
00048     
00049     void done();
00050 };
00051 
00052 DeclareWvList(WvUrlRequest);
00053 
00054 
00055 struct WvUrlLink
00056 {
00057     WvString linkname;
00058     WvUrl url;
00059 
00060     WvUrlLink::WvUrlLink(WvStringParm _linkname, WvStringParm _url)
00061         : linkname(_linkname), url(_url)
00062     {}
00063 };
00064 DeclareWvList(WvUrlLink);
00065 
00066 
00067 class WvBufUrlStream : public WvBufStream
00068 {
00069 public:
00070     WvString url;
00071     WvString proto;
00072     WvUrlLinkList links;  // HTML links or FTP directory listing
00073 
00074     // HTTP stuff...
00075     WvString version;
00076     int status;
00077     WvHTTPHeaderDict headers; 
00078 
00079     WvBufUrlStream() : status(0), headers(10)
00080         {}
00081     virtual ~WvBufUrlStream()
00082         {}
00083 };
00084 
00085 DeclareWvTable(WvIPPortAddr);
00086 
00087 
00088 class WvUrlStream : public WvStreamClone
00089 {
00090 public:
00091     class Target
00092     {
00093     public:
00094         WvIPPortAddr remaddr;
00095         WvString username;
00096 
00097         Target(const WvIPPortAddr &_remaddr, WvStringParm _username)
00098             : remaddr(_remaddr), username(_username) {}
00099 
00100         ~Target() {}
00101 
00102         bool operator== (const Target &n2) const
00103         { return (username == n2.username && remaddr == n2.remaddr); }
00104     };
00105     Target target;
00106     static int max_requests;
00107 
00108 protected:
00109     WvLog log;
00110     WvUrlRequestList urls, waiting_urls;
00111     int request_count;
00112     WvUrlRequest *curl; // current url
00113     virtual void doneurl() = 0;
00114     virtual void request_next() = 0;
00115 
00116 public:
00117     WvUrlStream(const WvIPPortAddr &_remaddr, WvStringParm _username, 
00118         WvStringParm logname)
00119         : WvStreamClone(new WvTCPConn(_remaddr)), target(_remaddr, _username),
00120           log(logname, WvLog::Debug)
00121     {
00122         request_count = 0;
00123         curl = NULL;
00124     }
00125 
00126     virtual ~WvUrlStream() {};
00127 
00128     virtual void close() = 0;
00129     void addurl(WvUrlRequest *url);
00130     void delurl(WvUrlRequest *url);
00131     // only implemented in WvHttpStream
00132     virtual size_t remaining()
00133     { return 0; }
00134     
00135     virtual void execute() = 0;
00136 };
00137 
00138 unsigned WvHash(const WvUrlStream::Target &n);
00139 
00140 DeclareWvDict(WvUrlStream, WvUrlStream::Target, target);
00141 
00142 
00143 class WvHttpStream : public WvUrlStream
00144 {
00145 public:
00146     static bool global_enable_pipelining;
00147     bool enable_pipelining;
00148     
00149 private:
00150     int pipeline_test_count;
00151     bool ssl;
00152     bool sent_url_request;      // Have we sent a request to the server yet?
00153     WvIPPortAddrTable &pipeline_incompatible;
00154     WvString http_response, pipeline_test_response;
00155     WvDynBuf putstream_data;
00156     
00157     enum { Unknown, Chunked, ContentLength, Infinity,
00158            PostHeadInfinity, PostHeadChunked, PostHeadStream,
00159            ChuckInfinity, ChuckChunked, ChuckStream } encoding;
00160     size_t bytes_remaining;
00161     bool in_chunk_trailer, last_was_pipeline_test, in_doneurl;
00162 
00163     virtual void doneurl();
00164     virtual void request_next();
00165     void start_pipeline_test(WvUrl *url);
00166     WvString request_str(WvUrlRequest *url, bool keep_alive);
00167     void send_request(WvUrlRequest *url);
00168     void pipelining_is_broken(int why);
00169     
00170 public:
00171     WvHttpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
00172          bool ssl, WvIPPortAddrTable &_pipeline_incompatible);
00173     virtual ~WvHttpStream();
00174 
00175     virtual void close();
00176     virtual bool pre_select(SelectInfo &si);
00177     virtual bool post_select(SelectInfo &si);
00178     virtual void execute();
00179     virtual size_t remaining()
00180     { return bytes_remaining; }
00181 };
00182 
00183 
00184 class WvFtpStream : public WvUrlStream
00185 {
00186     bool logged_in, pasv_acked;
00187     WvString password;
00188     WvTCPConn *data;
00189     time_t last_request_time;
00190 
00191     virtual void doneurl();
00192     virtual void request_next();
00193 
00194     // Disregard all lines that are of the form "xxx-", meaning that another
00195     // line follows.  Only the last line is important for us.
00196     char *get_important_line();
00197 
00198     // Parse response to "PASV" command and returns a pointer to the address
00199     // of the data port (or NULL if it can't parse the response)..
00200     // This mucks about with line.
00201     WvIPPortAddr *parse_pasv_response(char *line);
00202 
00203     WvString parse_for_links(char *line);
00204 
00205     WvCont cont;
00206     void* real_execute(void*);
00207 
00208 public:
00209     WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
00210                 WvStringParm _password);
00211 
00212     virtual bool pre_select(SelectInfo &si);
00213     virtual bool post_select(SelectInfo &si);
00214     virtual void close();
00215     virtual void execute();
00216 };
00217 
00218 
00219 // FIXME: Rename this to WvUrlPool someday.
00220 class WvHttpPool : public WvIStreamList
00221 {
00222     WvLog log;
00223     WvResolver dns;
00224     WvUrlStreamDict conns;
00225     WvUrlRequestList urls;
00226     int num_streams_created;
00227     
00228     WvIPPortAddrTable pipeline_incompatible;
00229     
00230 public:
00231     WvHttpPool();
00232     virtual ~WvHttpPool();
00233     
00234     virtual bool pre_select(SelectInfo &si);
00235     virtual void execute();
00236     
00237     WvBufUrlStream *addurl(WvStringParm _url, WvStringParm _method = "GET",
00238                             WvStringParm _headers = "",
00239                             WvStream *content_source = NULL,
00240                             bool create_dirs = false);
00241 
00242     // For URL uploads.  create_dirs should be true if you want all
00243     // non-existent directories in _url to be created.
00244 //    WvBufUrlStream *addputurl(WvStringParm _url, WvStringParm _headers,
00245 //                            WvStream *s, bool create_dirs = false);
00246 private:
00247     void unconnect(WvUrlStream *s);
00248     
00249 public:
00250     bool idle() const 
00251         { return !urls.count(); }
00252 };
00253 
00254 
00255 #endif // __WVHTTPPOOL_H

Generated on Tue Jul 12 02:59:27 2005 for WvStreams by  doxygen 1.4.0