FGFS-4.1/simgear
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

io: refactor and improve HTTP modules.

Browse Source 
- refactor code used multiple times spread over sg/fg into
   one single location.
 - allow aborting requests.
 - Provide two common request types:
  * FileRequest: Save response into file
  * MemoryRequest: Keep resonse in memory (std::string)
 - extend HTTP::Client interface:
  * urlretrieve: Save url to file (shortcut for making a
                 FileRequest)
  * urlload: Get respons into memory (shortcut for making
             a MemoryRequest)
This commit is contained in:
Thomas Geymayer 2013-10-27 18:40:14 +01:00
parent 050f3791cc
commit f93fead8f2
15 changed files with 789 additions and 354 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
simgear/io/CMakeLists.txt
View File

@ -15,6 +15,8 @@ set(HEADERS
sg_socket.hxx
sg_socket_udp.hxx
HTTPClient.hxx
HTTPFileRequest.hxx
HTTPMemoryRequest.hxx
HTTPRequest.hxx
HTTPContentDecode.hxx
DAVMultiStatus.hxx
@ -36,6 +38,8 @@ set(SOURCES
sg_socket.cxx
sg_socket_udp.cxx
HTTPClient.cxx
HTTPFileRequest.cxx
HTTPMemoryRequest.cxx
HTTPRequest.cxx
HTTPContentDecode.cxx
DAVMultiStatus.cxx

159
simgear/io/HTTPClient.cxx
View File

@ -22,12 +22,12 @@
//
#include "HTTPClient.hxx"
#include "HTTPFileRequest.hxx"
#include <sstream>
#include <cassert>
#include <cstdlib> // rand()
#include <list>
#include <iostream>
#include <errno.h>
#include <map>
#include <stdexcept>
@ -51,10 +51,6 @@
# endif
#endif
using std::string;
using std::stringstream;
using std::vector;
namespace simgear
{
@ -103,8 +99,21 @@ public:
virtual ~Connection()
{
}
virtual void handleBufferRead (NetBuffer& buffer)
{
if( !activeRequest || !activeRequest->isComplete() )
return NetChat::handleBufferRead(buffer);
// Request should be aborted (signaled by setting its state to complete).
// force the state to GETTING_BODY, to simplify logic in
// responseComplete and handleClose
state = STATE_GETTING_BODY;
responseComplete();
}
void setServer(const string& h, short p)
void setServer(const std::string& h, short p)
{
host = h;
port = p;
@ -224,6 +233,10 @@ public:
void tryStartNextRequest()
{
while( !queuedRequests.empty()
&& queuedRequests.front()->isComplete() )
queuedRequests.pop_front();
if (queuedRequests.empty()) {
idleTime.stamp();
return;
@ -244,28 +257,28 @@ public:
Request_ptr r = queuedRequests.front();
r->requestStart();
requestBodyBytesToSend = r->requestBodyLength();
stringstream headerData;
string path = r->path();
std::stringstream headerData;
std::string path = r->path();
assert(!path.empty());
string query = r->query();
string bodyData;
std::string query = r->query();
std::string bodyData;
if (!client->proxyHost().empty()) {
path = r->scheme() + "://" + r->host() + r->path();
}
if (r->requestBodyType() == CONTENT_TYPE_URL_ENCODED) {
if (r->bodyType() == CONTENT_TYPE_URL_ENCODED) {
headerData << r->method() << " " << path << " HTTP/1.1\r\n";
bodyData = query.substr(1); // URL-encode, drop the leading '?'
headerData << "Content-Type:" << CONTENT_TYPE_URL_ENCODED << "\r\n";
headerData << "Content-Length:" << bodyData.size() << "\r\n";
} else {
headerData << r->method() << " " << path << query << " HTTP/1.1\r\n";
if (requestBodyBytesToSend >= 0) {
headerData << "Content-Length:" << requestBodyBytesToSend << "\r\n";
headerData << "Content-Type:" << r->requestBodyType() << "\r\n";
if( r->hasBodyData() )
{
headerData << "Content-Length:" << r->bodyLength() << "\r\n";
headerData << "Content-Type:" << r->bodyType() << "\r\n";
}
}
@ -276,8 +289,8 @@ public:
headerData << "Proxy-Authorization: " << client->proxyAuth() << "\r\n";
}
BOOST_FOREACH(string h, r->requestHeaders()) {
headerData << h << ": " << r->header(h) << "\r\n";
BOOST_FOREACH(const StringMap::value_type& h, r->requestHeaders()) {
headerData << h.first << ": " << h.second << "\r\n";
}
headerData << "\r\n"; // final CRLF to terminate the headers
@ -292,33 +305,42 @@ public:
// drain down before trying to start any more requests.
return;
}
while (requestBodyBytesToSend > 0) {
char buf[4096];
int len = r->getBodyData(buf, 4096);
if (len > 0) {
requestBodyBytesToSend -= len;
if (!bufferSend(buf, len)) {
SG_LOG(SG_IO, SG_WARN, "overflow the HTTP::Connection output buffer");
state = STATE_SOCKET_ERROR;
return;
if( r->hasBodyData() )
for(size_t body_bytes_sent = 0; body_bytes_sent < r->bodyLength();)
{
char buf[4096];
size_t len = r->getBodyData(buf, body_bytes_sent, 4096);
if( len )
{
if( !bufferSend(buf, len) )
{
SG_LOG(SG_IO,
SG_WARN,
"overflow the HTTP::Connection output buffer");
state = STATE_SOCKET_ERROR;
return;
}
body_bytes_sent += len;
}
else
{
SG_LOG(SG_IO,
SG_WARN,
"HTTP asynchronous request body generation is unsupported");
break;
}
// SG_LOG(SG_IO, SG_INFO, "sent body:\n" << string(buf, len) << "\n%%%%%%%%%");
} else {
SG_LOG(SG_IO, SG_WARN, "HTTP asynchronous request body generation is unsupported");
break;
}
}
// SG_LOG(SG_IO, SG_INFO, "did start request:" << r->url() <<
// "\n\t @ " << reinterpret_cast<void*>(r.ptr()) <<
// "\n\t on connection " << this);
// successfully sent, remove from queue, and maybe send the next
// SG_LOG(SG_IO, SG_INFO, "did start request:" << r->url() <<
// "\n\t @ " << reinterpret_cast<void*>(r.ptr()) <<
// "\n\t on connection " << this);
// successfully sent, remove from queue, and maybe send the next
queuedRequests.pop_front();
sentRequests.push_back(r);
state = STATE_WAITING_FOR_RESPONSE;
state = STATE_WAITING_FOR_RESPONSE;
// pipelining, let's maybe send the next request right away
// pipelining, let's maybe send the next request right away
tryStartNextRequest();
}
@ -326,12 +348,12 @@ public:
{
idleTime.stamp();
client->receivedBytes(static_cast<unsigned int>(n));
if ((state == STATE_GETTING_BODY) || (state == STATE_GETTING_CHUNKED_BYTES)) {
_contentDecoder.receivedBytes(s, n);
} else {
buffer += string(s, n);
}
if( (state == STATE_GETTING_BODY)
|| (state == STATE_GETTING_CHUNKED_BYTES) )
_contentDecoder.receivedBytes(s, n);
else
buffer.append(s, n);
}
virtual void foundTerminator(void)
@ -428,7 +450,7 @@ private:
void processHeader()
{
string h = strutils::simplify(buffer);
std::string h = strutils::simplify(buffer);
if (h.empty()) { // blank line terminates headers
headersComplete();
return;
@ -440,9 +462,9 @@ private:
return;
}
string key = strutils::simplify(buffer.substr(0, colonPos));
string lkey = boost::to_lower_copy(key);
string value = strutils::strip(buffer.substr(colonPos + 1));
std::string key = strutils::simplify(buffer.substr(0, colonPos));
std::string lkey = boost::to_lower_copy(key);
std::string value = strutils::strip(buffer.substr(colonPos + 1));
// only consider these if getting headers (as opposed to trailers
// of a chunked transfer)
@ -466,7 +488,7 @@ private:
activeRequest->responseHeader(lkey, value);
}
void processTransferEncoding(const string& te)
void processTransferEncoding(const std::string& te)
{
if (te == "chunked") {
chunkedTransfer = true;
@ -534,7 +556,7 @@ private:
void responseComplete()
{
Request_ptr completedRequest = activeRequest;
Request_ptr completedRequest = activeRequest;
_contentDecoder.finish();
assert(sentRequests.front() == activeRequest);
@ -581,14 +603,13 @@ private:
Client* client;
Request_ptr activeRequest;
ConnectionState state;
string host;
std::string host;
short port;
std::string buffer;
int bodyTransferSize;
SGTimeStamp idleTime;
bool chunkedTransfer;
bool noMessageBody;
int requestBodyBytesToSend;
RequestList queuedRequests;
RequestList sentRequests;
@ -669,6 +690,9 @@ void Client::update(int waitTimeout)
void Client::makeRequest(const Request_ptr& r)
{
if( r->isComplete() )
return;
if( r->url().find("://") == std::string::npos ) {
r->setFailure(EINVAL, "malformed URL");
return;
@ -679,7 +703,7 @@ void Client::makeRequest(const Request_ptr& r)
return;
}
string host = r->host();
std::string host = r->host();
int port = r->port();
if (!d->proxy.empty()) {
host = d->proxy;
@ -687,9 +711,9 @@ void Client::makeRequest(const Request_ptr& r)
}
Connection* con = NULL;
stringstream ss;
std::stringstream ss;
ss << host << "-" << port;
string connectionId = ss.str();
std::string connectionId = ss.str();
bool havePending = !d->pendingRequests.empty();
bool atConnectionsLimit = d->connections.size() >= d->maxConnections;
ConnectionDict::iterator consEnd = d->connections.end();
@ -741,12 +765,29 @@ void Client::makeRequest(const Request_ptr& r)
con->queueRequest(r);
}
//------------------------------------------------------------------------------
FileRequestRef Client::urlretrieve( const std::string& url,
const std::string& filename )
{
FileRequestRef req = new FileRequest(url, filename);
makeRequest(req);
return req;
}
//------------------------------------------------------------------------------
MemoryRequestRef Client::urlload(const std::string& url)
{
MemoryRequestRef req = new MemoryRequest(url);
makeRequest(req);
return req;
}
void Client::requestFinished(Connection* con)
{
}
void Client::setUserAgent(const string& ua)
void Client::setUserAgent(const std::string& ua)
{
d->userAgent = ua;
}
@ -766,7 +807,9 @@ const std::string& Client::proxyAuth() const
return d->proxyAuth;
}
void Client::setProxy(const string& proxy, int port, const string& auth)
void Client::setProxy( const std::string& proxy,
int port,
const std::string& auth )
{
d->proxy = proxy;
d->proxyPort = port;

22
simgear/io/HTTPClient.hxx
View File

@ -27,7 +27,8 @@
#include <memory> // for std::auto_ptr
#include <stdint.h> // for uint_64t
#include <simgear/io/HTTPRequest.hxx>
#include <simgear/io/HTTPFileRequest.hxx>
#include <simgear/io/HTTPMemoryRequest.hxx>
namespace simgear
{
@ -47,7 +48,24 @@ public:
void update(int waitTimeout = 0);
void makeRequest(const Request_ptr& r);
/**
* Download a resource and save it to a file.
*
* @param url The resource to download
* @param filename Path to the target file
* @param data Data for POST request
*/
FileRequestRef urlretrieve( const std::string& url,
const std::string& filename );
/**
* Request a resource and keep it in memory.
*
* @param url The resource to download
*/
MemoryRequestRef urlload(const std::string& url);
void setUserAgent(const std::string& ua);
void setProxy(const std::string& proxy, int port, const std::string& auth = "");

82
simgear/io/HTTPFileRequest.cxx Normal file
View File

@ -0,0 +1,82 @@
// HTTP request writing response to a file.
//
// Copyright (C) 2013 Thomas Geymayer <tomgey@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
#include "HTTPFileRequest.hxx"
#include <simgear/debug/logstream.hxx>
namespace simgear
{
namespace HTTP
{
//----------------------------------------------------------------------------
FileRequest::FileRequest(const std::string& url, const std::string& path):
Request(url, "GET"),
_filename(path)
{
}
//----------------------------------------------------------------------------
void FileRequest::responseHeadersComplete()
{
Request::responseHeadersComplete();
if( !_filename.empty() )
// TODO validate path? (would require to expose fgValidatePath somehow to
// simgear)
_file.open(_filename.c_str(), std::ios::binary | std::ios::trunc);
if( !_file )
{
SG_LOG
(
SG_IO,
SG_WARN,
"HTTP::FileRequest: failed to open file '" << _filename << "'"
);
abort("Failed to open file.");
}
}
//----------------------------------------------------------------------------
void FileRequest::gotBodyData(const char* s, int n)
{
if( !_file )
{
SG_LOG
(
SG_IO,
SG_WARN,
"HTTP::FileRequest: error writing to '" << _filename << "'"
);
return;
}
_file.write(s, n);
}
//----------------------------------------------------------------------------
void FileRequest::onAlways()
{
_file.close();
}
} // namespace HTTP
} // namespace simgear

56
simgear/io/HTTPFileRequest.hxx Normal file
View File

@ -0,0 +1,56 @@
///@file HTTP request writing response to a file.
//
// Copyright (C) 2013 Thomas Geymayer <tomgey@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
#ifndef SG_HTTP_FILEREQUEST_HXX_
#define SG_HTTP_FILEREQUEST_HXX_
#include "HTTPRequest.hxx"
#include <fstream>
namespace simgear
{
namespace HTTP
{
class FileRequest:
public Request
{
public:
/**
*
* @param url Adress to download from
* @param path Path to file for saving response
*/
FileRequest(const std::string& url, const std::string& path);
protected:
std::string _filename;
std::ofstream _file;
virtual void responseHeadersComplete();
virtual void gotBodyData(const char* s, int n);
virtual void onAlways();
};
typedef SGSharedPtr<FileRequest> FileRequestRef;
} // namespace HTTP
} // namespace simgear
#endif /* SG_HTTP_FILEREQUEST_HXX_ */

55
simgear/io/HTTPMemoryRequest.cxx Normal file
View File

@ -0,0 +1,55 @@
// HTTP request keeping response in memory.
//
// Copyright (C) 2013 Thomas Geymayer <tomgey@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
#include "HTTPMemoryRequest.hxx"
namespace simgear
{
namespace HTTP
{
//----------------------------------------------------------------------------
MemoryRequest::MemoryRequest(const std::string& url):
Request(url, "GET")
{
}
//----------------------------------------------------------------------------
const std::string& MemoryRequest::responseBody() const
{
return _response;
}
//----------------------------------------------------------------------------
void MemoryRequest::responseHeadersComplete()
{
Request::responseHeadersComplete();
if( responseLength() )
_response.reserve( responseLength() );
}
//----------------------------------------------------------------------------
void MemoryRequest::gotBodyData(const char* s, int n)
{
_response.append(s, n);
}
} // namespace HTTP
} // namespace simgear

58
simgear/io/HTTPMemoryRequest.hxx Normal file
View File

@ -0,0 +1,58 @@
///@file HTTP request keeping response in memory.
//
// Copyright (C) 2013 Thomas Geymayer <tomgey@gmail.com>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Library General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Library General Public License for more details.
//
// You should have received a copy of the GNU Library General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
#ifndef SG_HTTP_MEMORYREQUEST_HXX_
#define SG_HTTP_MEMORYREQUEST_HXX_
#include "HTTPRequest.hxx"
#include <fstream>
namespace simgear
{
namespace HTTP
{
class MemoryRequest:
public Request
{
public:
/**
*
* @param url Adress to download from
*/
MemoryRequest(const std::string& url);
/**
* Body contents of server response.
*/
const std::string& responseBody() const;
protected:
std::string _response;
virtual void responseHeadersComplete();
virtual void gotBodyData(const char* s, int n);
};
typedef SGSharedPtr<MemoryRequest> MemoryRequestRef;
} // namespace HTTP
} // namespace simgear
#endif /* SG_HTTP_MEMORYREQUEST_HXX_ */

365
simgear/io/HTTPRequest.cxx
View File

@ -1,59 +1,116 @@
#include "HTTPRequest.hxx"
#include <simgear/misc/strutils.hxx>
#include <simgear/compiler.h>
#include <simgear/debug/logstream.hxx>
using std::string;
using std::map;
#include <simgear/misc/strutils.hxx>
#include <simgear/props/props_io.hxx>
namespace simgear
{
namespace HTTP
{
extern const int DEFAULT_HTTP_PORT;
Request::Request(const string& url, const string method) :
_method(method),
_url(url),
_responseVersion(HTTP_VERSION_UNKNOWN),
_responseStatus(0),
_responseLength(0),
_receivedBodyBytes(0),
_willClose(false)
//------------------------------------------------------------------------------
Request::Request(const std::string& url, const std::string method):
_method(method),
_url(url),
_responseVersion(HTTP_VERSION_UNKNOWN),
_responseStatus(0),
_responseLength(0),
_receivedBodyBytes(0),
_ready_state(UNSENT),
_willClose(false)
{
}
//------------------------------------------------------------------------------
Request::~Request()
{
}
void Request::setUrl(const string& url)
//------------------------------------------------------------------------------
Request* Request::done(const Callback& cb)
{
_url = url;
if( _ready_state == DONE )
cb(this);
else
_cb_done = cb;
return this;
}
string_list Request::requestHeaders() const
//------------------------------------------------------------------------------
Request* Request::fail(const Callback& cb)
{
string_list r;
return r;
if( _ready_state == FAILED )
cb(this);
else
_cb_fail = cb;
return this;
}
string Request::header(const std::string& name) const
//------------------------------------------------------------------------------
Request* Request::always(const Callback& cb)
{
return string();
if( isComplete() )
cb(this);
else
_cb_always = cb;
return this;
}
//------------------------------------------------------------------------------
void Request::setBodyData( const std::string& data,
const std::string& type )
{
_request_data = data;
_request_media_type = type;
if( !data.empty() && _method == "GET" )
_method = "POST";
}
//----------------------------------------------------------------------------
void Request::setBodyData(const SGPropertyNode* data)
{
if( !data )
setBodyData("");
std::stringstream buf;
writeProperties(buf, data, true);
setBodyData(buf.str(), "application/xml");
}
//------------------------------------------------------------------------------
void Request::setUrl(const std::string& url)
{
_url = url;
}
//------------------------------------------------------------------------------
void Request::requestStart()
{
setReadyState(OPENED);
}
void Request::responseStart(const string& r)
//------------------------------------------------------------------------------
Request::HTTPVersion decodeHTTPVersion(const std::string& v)
{
if( v == "HTTP/1.1" ) return Request::HTTP_1_1;
if( v == "HTTP/1.0" ) return Request::HTTP_1_0;
if( strutils::starts_with(v, "HTTP/0.") ) return Request::HTTP_0_x;
return Request::HTTP_VERSION_UNKNOWN;
}
//------------------------------------------------------------------------------
void Request::responseStart(const std::string& r)
{
const int maxSplit = 2; // HTTP/1.1 nnn reason-string
string_list parts = strutils::split(r, NULL, maxSplit);
@ -63,42 +120,72 @@ void Request::responseStart(const string& r)
return;
}
_responseVersion = decodeVersion(parts[0]);
_responseVersion = decodeHTTPVersion(parts[0]);
_responseStatus = strutils::to_int(parts[1]);
_responseReason = parts[2];
}
void Request::responseHeader(const string& key, const string& value)
//------------------------------------------------------------------------------
void Request::responseHeader(const std::string& key, const std::string& value)
{
if (key == "connection") {
_willClose = (value.find("close") != string::npos);
}
_responseHeaders[key] = value;
if( key == "connection" )
_willClose = (value.find("close") != std::string::npos);
_responseHeaders[key] = value;
}
//------------------------------------------------------------------------------
void Request::responseHeadersComplete()
{
// no op
}
void Request::processBodyBytes(const char* s, int n)
{
_receivedBodyBytes += n;
gotBodyData(s, n);
}
void Request::gotBodyData(const char* s, int n)
{
setReadyState(HEADERS_RECEIVED);
}
//------------------------------------------------------------------------------
void Request::responseComplete()
{
if( !isComplete() )
setReadyState(DONE);
}
string Request::scheme() const
//------------------------------------------------------------------------------
void Request::gotBodyData(const char* s, int n)
{
setReadyState(LOADING);
}
//------------------------------------------------------------------------------
void Request::onDone()
{
}
//------------------------------------------------------------------------------
void Request::onFail()
{
SG_LOG
(
SG_IO,
SG_INFO,
"request failed:" << url() << " : "
<< responseCode() << "/" << responseReason()
);
}
//------------------------------------------------------------------------------
void Request::onAlways()
{
}
//------------------------------------------------------------------------------
void Request::processBodyBytes(const char* s, int n)
{
_receivedBodyBytes += n;
gotBodyData(s, n);
}
//------------------------------------------------------------------------------
std::string Request::scheme() const
{
int firstColon = url().find(":");
if (firstColon > 0) {
@ -107,10 +194,11 @@ string Request::scheme() const
return ""; // couldn't parse scheme
}
string Request::path() const
//------------------------------------------------------------------------------
std::string Request::path() const
{
string u(url());
std::string u(url());
int schemeEnd = u.find("://");
if (schemeEnd < 0) {
return ""; // couldn't parse scheme
@ -132,10 +220,10 @@ string Request::path() const
return u.substr(hostEnd, query - hostEnd);
}
string Request::query() const
//------------------------------------------------------------------------------
std::string Request::query() const
{
string u(url());
std::string u(url());
int query = u.find('?');
if (query < 0) {
return ""; //no query string found
@ -144,104 +232,153 @@ string Request::query() const
return u.substr(query); //includes question mark
}
string Request::host() const
//------------------------------------------------------------------------------
std::string Request::host() const
{
string hp(hostAndPort());
int colonPos = hp.find(':');
if (colonPos >= 0) {
return hp.substr(0, colonPos); // trim off the colon and port
} else {
return hp; // no port specifier
}
std::string hp(hostAndPort());
int colonPos = hp.find(':');
if (colonPos >= 0) {
return hp.substr(0, colonPos); // trim off the colon and port
} else {
return hp; // no port specifier
}
}
//------------------------------------------------------------------------------
unsigned short Request::port() const
{
string hp(hostAndPort());
int colonPos = hp.find(':');
if (colonPos >= 0) {
return (unsigned short) strutils::to_int(hp.substr(colonPos + 1));
} else {
return DEFAULT_HTTP_PORT;
}
std::string hp(hostAndPort());
int colonPos = hp.find(':');
if (colonPos >= 0) {
return (unsigned short) strutils::to_int(hp.substr(colonPos + 1));
} else {
return DEFAULT_HTTP_PORT;
}
}
string Request::hostAndPort() const
//------------------------------------------------------------------------------
std::string Request::hostAndPort() const
{
string u(url());
int schemeEnd = u.find("://");
if (schemeEnd < 0) {
return ""; // couldn't parse scheme
}
int hostEnd = u.find('/', schemeEnd + 3);
if (hostEnd < 0) { // all remainder of URL is host
return u.substr(schemeEnd + 3);
}
return u.substr(schemeEnd + 3, hostEnd - (schemeEnd + 3));
std::string u(url());
int schemeEnd = u.find("://");
if (schemeEnd < 0) {
return ""; // couldn't parse scheme
}
int hostEnd = u.find('/', schemeEnd + 3);
if (hostEnd < 0) { // all remainder of URL is host
return u.substr(schemeEnd + 3);
}
return u.substr(schemeEnd + 3, hostEnd - (schemeEnd + 3));
}
//------------------------------------------------------------------------------
void Request::setResponseLength(unsigned int l)
{
_responseLength = l;
_responseLength = l;
}
//------------------------------------------------------------------------------
unsigned int Request::responseLength() const
{
// if the server didn't supply a content length, use the number
// of bytes we actually received (so far)
if ((_responseLength == 0) && (_receivedBodyBytes > 0)) {
return _receivedBodyBytes;
}
return _responseLength;
// if the server didn't supply a content length, use the number
// of bytes we actually received (so far)
if( (_responseLength == 0) && (_receivedBodyBytes > 0) )
return _receivedBodyBytes;
return _responseLength;
}
//------------------------------------------------------------------------------
void Request::setFailure(int code, const std::string& reason)
{
_responseStatus = code;
_responseReason = reason;
failed();
_responseStatus = code;
_responseReason = reason;
setReadyState(FAILED);
}
void Request::failed()
//------------------------------------------------------------------------------
void Request::setReadyState(ReadyState state)
{
SG_LOG(SG_IO, SG_INFO, "request failed:" << url() << " : "
<< responseCode() << "/" << responseReason());
_ready_state = state;
if( state == DONE )
{
if( _cb_done )
_cb_done(this);
onDone();
}
else if( state == FAILED )
{
if( _cb_fail )
_cb_fail(this);
onFail();
}
else
return;
if( _cb_always )
_cb_always(this);
onAlways();
}
Request::HTTPVersion Request::decodeVersion(const string& v)
//------------------------------------------------------------------------------
void Request::abort()
{
if (v == "HTTP/1.1") return HTTP_1_1;
if (v == "HTTP/1.0") return HTTP_1_0;
if (strutils::starts_with(v, "HTTP/0.")) return HTTP_0_x;
return HTTP_VERSION_UNKNOWN;
abort("Request aborted.");
}
//----------------------------------------------------------------------------
void Request::abort(const std::string& reason)
{
if( isComplete() )
return;
setFailure(-1, reason);
_willClose = true;
}
//------------------------------------------------------------------------------
bool Request::closeAfterComplete() const
{
// for non HTTP/1.1 connections, assume server closes
return _willClose || (_responseVersion != HTTP_1_1);
}
int Request::requestBodyLength() const
{
return -1;
// for non HTTP/1.1 connections, assume server closes
return _willClose || (_responseVersion != HTTP_1_1);
}
std::string Request::requestBodyType() const
//------------------------------------------------------------------------------
bool Request::isComplete() const
{
return "text/plain";
return _ready_state == DONE || _ready_state == FAILED;
}
int Request::getBodyData(char*, int maxCount) const
//------------------------------------------------------------------------------
bool Request::hasBodyData() const
{
return 0;
return !_request_media_type.empty();
}
//------------------------------------------------------------------------------
std::string Request::bodyType() const
{
return _request_media_type;
}
//------------------------------------------------------------------------------
size_t Request::bodyLength() const
{
return _request_data.length();
}
//------------------------------------------------------------------------------
size_t Request::getBodyData(char* s, size_t offset, size_t max_count) const
{
size_t bytes_available = _request_data.size() - offset;
size_t bytes_to_read = std::min(bytes_available, max_count);
memcpy(s, _request_data.data() + offset, bytes_to_read);
return bytes_to_read;
}
} // of namespace HTTP
} // of namespace simgear

156
simgear/io/HTTPRequest.hxx
View File

@ -3,21 +3,85 @@
#include <map>
#include <simgear/structure/map.hxx>
#include <simgear/structure/SGReferenced.hxx>
#include <simgear/structure/SGSharedPtr.hxx>
#include <simgear/math/sg_types.hxx>
#include <boost/function.hpp>
class SGPropertyNode;
namespace simgear
{
namespace HTTP
{
class Request : public SGReferenced
class Request:
public SGReferenced
{
public:
typedef boost::function<void(Request*)> Callback;
enum ReadyState
{
UNSENT = 0,
OPENED,
HEADERS_RECEIVED,
LOADING,
DONE,
FAILED
};
virtual ~Request();
/**
*
*/
StringMap& requestHeaders() { return _request_headers; }
const StringMap& requestHeaders() const { return _request_headers; }
std::string& requestHeader(const std::string& key)
{ return _request_headers[key]; }
const std::string requestHeader(const std::string& key) const
{ return _request_headers.get(key); }
/**
* Set the handler to be called when the request successfully completes.
*
* @note If the request is already complete, the handler is called
* immediately.
*/
Request* done(const Callback& cb);
/**
* Set the handler to be called when the request completes or aborts with an
* error.
*
* @note If the request has already failed, the handler is called
* immediately.
*/
Request* fail(const Callback& cb);
/**
* Set the handler to be called when the request either successfully
* completes or fails.
*
* @note If the request is already complete or has already failed, the
* handler is called immediately.
*/
Request* always(const Callback& cb);
/**
* Set the data for the body of the request. The request is automatically
* send using the POST method.
*
* @param data Body data
* @param type Media Type (aka MIME) of the body data
*/
void setBodyData( const std::string& data,
const std::string& type = "text/plain" );
void setBodyData( const SGPropertyNode* data );
virtual void setUrl(const std::string& url);
virtual std::string method() const
@ -32,8 +96,8 @@ public:
virtual unsigned short port() const;
virtual std::string query() const;
virtual string_list requestHeaders() const;
virtual std::string header(const std::string& name) const;
StringMap const& responseHeaders() const
{ return _responseHeaders; }
virtual int responseCode() const
{ return _responseStatus; }
@ -41,26 +105,30 @@ public:
virtual std::string responseReason() const
{ return _responseReason; }
void setResponseLength(unsigned int l);
void setResponseLength(unsigned int l);
virtual unsigned int responseLength() const;
/**
* Query the size of the request body. -1 (the default value) means no
* request body
* Check if request contains body data.
*/
virtual int requestBodyLength() const;
virtual bool hasBodyData() const;
/**
* Retrieve the request body content type.
*/
virtual std::string bodyType() const;
/**
* Retrieve the size of the request body.
*/
virtual size_t bodyLength() const;
/**
* Retrieve the body data bytes. Will be passed the maximum body bytes
* to return in the buffer, and must return the actual number
* of bytes written.
*/
virtual int getBodyData(char* s, int count) const;
/**
* retrieve the request body content type. Default is text/plain
*/
virtual std::string requestBodyType() const;
virtual size_t getBodyData(char* s, size_t offset, size_t max_count) const;
/**
* running total of body bytes received so far. Can be used
@ -80,9 +148,21 @@ public:
HTTPVersion responseVersion() const
{ return _responseVersion; }
static HTTPVersion decodeVersion(const std::string& v);
ReadyState readyState() const { return _ready_state; }
/**
* Request aborting this request.
*/
void abort();
/**
* Request aborting this request and specify the reported reaseon for it.
*/
void abort(const std::string& reason);
bool closeAfterComplete() const;
bool isComplete() const;
protected:
Request(const std::string& url, const std::string method = "GET");
@ -91,34 +171,48 @@ protected:
virtual void responseHeader(const std::string& key, const std::string& value);
virtual void responseHeadersComplete();
virtual void responseComplete();
virtual void failed();
virtual void gotBodyData(const char* s, int n);
virtual void onDone();
virtual void onFail();
virtual void onAlways();
private:
friend class Client;
friend class Connection;
friend class ContentDecoder;
Request(const Request&); // = delete;
Request& operator=(const Request&); // = delete;
void processBodyBytes(const char* s, int n);
void setFailure(int code, const std::string& reason);
void setReadyState(ReadyState state);
std::string _method;
std::string _url;
HTTPVersion _responseVersion;
int _responseStatus;
std::string _responseReason;
unsigned int _responseLength;
unsigned int _receivedBodyBytes;
bool _willClose;
typedef std::map<std::string, std::string> HeaderDict;
HeaderDict _responseHeaders;
std::string _method;
std::string _url;
StringMap _request_headers;
std::string _request_data;
std::string _request_media_type;
HTTPVersion _responseVersion;
int _responseStatus;
std::string _responseReason;
StringMap _responseHeaders;
unsigned int _responseLength;
unsigned int _receivedBodyBytes;
Callback _cb_done,
_cb_fail,
_cb_always;
ReadyState _ready_state;
bool _willClose;
};
typedef SGSharedPtr<Request> Request_ptr;
} // of namespace HTTP
} // of namespace simgear
#endif // of SG_HTTP_REQUEST_HXX

102
simgear/io/SVNRepository.cxx
View File

@ -119,40 +119,9 @@ namespace { // anonmouse
Request(repo->baseUrl, "PROPFIND"),
_repo(repo)
{
}
virtual string_list requestHeaders() const
{
string_list r;
r.push_back("Depth");
return r;
}
virtual string header(const string& name) const
{
if (name == "Depth") {
return "0";
}
return string();
}
virtual string requestBodyType() const
{
return "application/xml; charset=\"utf-8\"";
}
virtual int requestBodyLength() const
{
return strlen(PROPFIND_REQUEST_BODY);
}
virtual int getBodyData(char* buf, int count) const
{
int bodyLen = strlen(PROPFIND_REQUEST_BODY);
assert(count >= bodyLen);
memcpy(buf, PROPFIND_REQUEST_BODY, bodyLen);
return bodyLen;
requestHeader("Depth") = "0";
setBodyData( PROPFIND_REQUEST_BODY,
"application/xml; charset=\"utf-8\"" );
}
protected:
@ -169,7 +138,7 @@ namespace { // anonmouse
}
}
virtual void responseComplete()
virtual void onDone()
{
if (responseCode() == 207) {
_davStatus.finishParse();
@ -189,9 +158,9 @@ namespace { // anonmouse
_davStatus.parseXML(s, n);
}
virtual void failed()
virtual void onFail()
{
HTTP::Request::failed();
HTTP::Request::onFail();
_repo->propFindFailed(this, SVNRepository::SVN_ERROR_SOCKET);
}
@ -200,64 +169,43 @@ namespace { // anonmouse
DAVMultiStatus _davStatus;
};
class UpdateReportRequest : public HTTP::Request
class UpdateReportRequest:
public HTTP::Request
{
public:
UpdateReportRequest(SVNRepoPrivate* repo,
const std::string& aVersionName,
bool startEmpty) :
HTTP::Request("", "REPORT"),
_requestSent(0),
_parser(repo->p),
_repo(repo),
_failed(false)
{
setUrl(repo->vccUrl);
_request =
std::string request =
"<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n"
"<S:update-report send-all=\"true\" xmlns:S=\"svn:\">\n"
"<S:src-path>" + repo->baseUrl + "</S:src-path>\n"
"<S:depth>unknown</S:depth>\n";
"<S:depth>unknown</S:depth>\n"
"<S:entry rev=\"" + aVersionName + "\" depth=\"infinity\" start-empty=\"true\"/>\n";
_request += "<S:entry rev=\"" + aVersionName + "\" depth=\"infinity\" start-empty=\"true\"/>\n";
if (!startEmpty) {
string_list entries;
_repo->rootCollection->mergeUpdateReportDetails(0, entries);
BOOST_FOREACH(string e, entries) {
_request += e + "\n";
}
if( !startEmpty )
{
string_list entries;
_repo->rootCollection->mergeUpdateReportDetails(0, entries);
BOOST_FOREACH(string e, entries)
{
request += e + "\n";
}
}
_request += "</S:update-report>";
}
request += "</S:update-report>";
virtual string requestBodyType() const
{
return "application/xml; charset=\"utf-8\"";
}
virtual int requestBodyLength() const
{
return _request.size();
}
virtual int getBodyData(char* buf, int count) const
{
int len = std::min(count, requestBodyLength() - _requestSent);
memcpy(buf, _request.c_str() + _requestSent, len);
_requestSent += len;
return len;
setBodyData(request, "application/xml; charset=\"utf-8\"");
}
protected:
virtual void responseHeadersComplete()
{
}
virtual void responseComplete()
virtual void onDone()
{
if (_failed) {
return;
@ -300,14 +248,12 @@ protected:
}
}
virtual void failed()
virtual void onFail()
{
HTTP::Request::failed();
HTTP::Request::onFail();
_repo->updateFailed(this, SVNRepository::SVN_ERROR_SOCKET);
}
private:
string _request;
mutable int _requestSent;
SVNReportParser _parser;
SVNRepoPrivate* _repo;
bool _failed;

29
simgear/io/httpget.cxx
View File

@ -48,35 +48,11 @@ public:
}
string key = h.substr(0, colonPos);
_headers[key] = h.substr(colonPos + 1);
requestHeader(key) = h.substr(colonPos + 1);
}
virtual string_list requestHeaders() const
{
string_list r;
std::map<string, string>::const_iterator it;
for (it = _headers.begin(); it != _headers.end(); ++it) {
r.push_back(it->first);
}
return r;
}
virtual string header(const string& name) const
{
std::map<string, string>::const_iterator it = _headers.find(name);
if (it == _headers.end()) {
return string();
}
return it->second;
}
protected:
virtual void responseHeadersComplete()
{
}
virtual void responseComplete()
virtual void onDone()
{
_complete = true;
}
@ -88,7 +64,6 @@ protected:
private:
bool _complete;
SGFile* _file;
std::map<string, string> _headers;
};
int main(int argc, char* argv[])

2
simgear/io/sg_netChat.hxx
View File

@ -70,6 +70,8 @@ class NetChat : public NetBufferChannel
{
std::string terminator;
int bytesToCollect;
protected:
virtual void handleBufferRead (NetBuffer& buffer) ;
public:

44
simgear/io/test_HTTP.cxx
View File

@ -53,48 +53,23 @@ public:
bool complete;
bool failed;
string bodyData;
string bodyContentType;
TestRequest(const std::string& url, const std::string method = "GET") :
HTTP::Request(url, method),
complete(false)
{
bodyContentType = "text/plain";
}
std::map<string, string> sendHeaders;
std::map<string, string> headers;
protected:
string_list requestHeaders() const
{
string_list r;
std::map<string, string>::const_iterator it;
for (it = sendHeaders.begin(); it != sendHeaders.end(); ++it) {
r.push_back(it->first);
}
return r;
}
string header(const string& name) const
{
std::map<string, string>::const_iterator it = sendHeaders.find(name);
if (it == sendHeaders.end()) {
return string();
}
return it->second;
}
virtual void responseHeadersComplete()
{
}
virtual void responseComplete()
virtual void onDone()
{
complete = true;
}
virtual void failure()
virtual void onFail()
{
failed = true;
}
@ -105,11 +80,6 @@ protected:
bodyData += string(s, n);
}
virtual std::string requestBodyType() const
{
return bodyContentType;
}
virtual void responseHeader(const string& header, const string& value)
{
headers[header] = value;
@ -535,8 +505,8 @@ int main(int argc, char* argv[])
{
TestRequest* tr = new TestRequest("http://localhost:2000/test_headers");
HTTP::Request_ptr own(tr);
tr->sendHeaders["X-Foo"] = "Bar";
tr->sendHeaders["X-AnotherHeader"] = "A longer value";
tr->requestHeader("X-Foo") = "Bar";
tr->requestHeader("X-AnotherHeader") = "A longer value";
cl.makeRequest(tr);
waitForComplete(&cl, tr);
@ -721,7 +691,7 @@ int main(int argc, char* argv[])
{
cout << "POST" << endl;
TestRequest* tr = new TestRequest("http://localhost:2000/test_post?foo=abc&bar=1234&username=johndoe", "POST");
tr->bodyContentType = "application/x-www-form-urlencoded";
tr->setBodyData("", "application/x-www-form-urlencoded");
HTTP::Request_ptr own(tr);
cl.makeRequest(tr);

7
simgear/package/Catalog.cxx
View File

@ -50,17 +50,12 @@ public:
}
protected:
virtual void responseHeadersComplete()
{
}
virtual void gotBodyData(const char* s, int n)
{
m_buffer += std::string(s, n);
}
virtual void responseComplete()
virtual void onDone()
{
if (responseCode() != 200) {
SG_LOG(SG_GENERAL, SG_ALERT, "catalog download failure:" << m_owner->url());

2
simgear/package/Install.cxx
View File

@ -81,7 +81,7 @@ protected:
m_owner->installProgress(m_buffer.size(), responseLength());
}
virtual void responseComplete()
virtual void onDone()
{
if (responseCode() != 200) {
SG_LOG(SG_GENERAL, SG_ALERT, "download failure");