mirror of https://github.com/davisking/dlib.git
--HG-- extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%402300pull/2/head
Davis King
17 years ago
parent
bdd16302eb
commit
2801ca62c7
@ -0,0 +1,740 @@
|
||||
|
||||
|
||||
#include "../sockets.h"
|
||||
#include "../string.h"
|
||||
#include "../logger.h"
|
||||
#include "../sockstreambuf.h"
|
||||
#include "../timeout.h"
|
||||
#include "http_client.h"
|
||||
#include <time.h>
|
||||
#include <stdio.h>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <iostream>
|
||||
|
||||
namespace dlib
|
||||
{
|
||||
|
||||
typedef dlib::shared_ptr< dlib::timeout::kernel_1a > timeout_ptr;
|
||||
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define BR_CASECMP strnicmp
|
||||
#else
|
||||
#define BR_CASECMP strncasecmp
|
||||
#endif
|
||||
// Default timeout after 60 seconds
|
||||
#define DEFAULT_TIMEOUT 60000
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
inline bool isXdigit( char c )
|
||||
{
|
||||
return (c >= '0' && c <= '9') ||
|
||||
(c >= 'A' && c <= 'Z') ||
|
||||
(c >= 'a' && c <= 'z');
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
std::string http_client::urldecode( const std::string& s )
|
||||
{
|
||||
std::stringstream ss;
|
||||
|
||||
for ( char const * p_read = s.c_str(), * p_end = (s.c_str() + s.size()); p_read < p_end; p_read++ )
|
||||
{
|
||||
if ( p_read[0] == '%' && p_read+1 != p_end && p_read+2 != p_end && isXdigit(p_read[1]) && isXdigit(p_read[2]) )
|
||||
{
|
||||
ss << static_cast<char>((( (p_read[1] & 0xf) + ((p_read[1] >= 'A') ? 9 : 0) ) << 4 ) | ( (p_read[2] & 0xf) + ((p_read[2] >= 'A') ? 9 : 0) ));
|
||||
p_read += 2;
|
||||
}
|
||||
else if ( p_read[0] == '+' )
|
||||
{
|
||||
// Undo the encoding that replaces spaces with plus signs.
|
||||
ss << ' ';
|
||||
}
|
||||
else
|
||||
{
|
||||
ss << p_read[0];
|
||||
}
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
//! \return modified string ``s'' with spaces trimmed from left
|
||||
inline std::string& triml(std::string& s)
|
||||
{
|
||||
int pos(0);
|
||||
for ( ; s[pos] == ' ' || s[pos] == '\t' || s[pos] == '\r' || s[pos] == '\n' ; ++pos );
|
||||
s.erase(0, pos);
|
||||
return s;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
//! \return modified string ``s'' with spaces trimmed from right
|
||||
inline std::string& trimr(std::string& s)
|
||||
{
|
||||
int pos(s.size());
|
||||
for ( ; pos && (s[pos-1] == ' ' || s[pos-1] == '\t' || s[pos-1] == '\r' || s[pos-1] == '\n') ; --pos );
|
||||
s.erase(pos, s.size()-pos);
|
||||
return s;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
//! \return modified string ``s'' with spaces trimmed from edges
|
||||
inline std::string& trim(std::string& s)
|
||||
{
|
||||
return triml(trimr(s));
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
http_client::
|
||||
http_client(
|
||||
) :
|
||||
http_return(0),
|
||||
timeout(DEFAULT_TIMEOUT),
|
||||
OnDownload(0)
|
||||
{
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
std::string http_client::get_header(const std::string& header_name) const
|
||||
{
|
||||
stringmap::const_iterator ci = headers.find(header_name);
|
||||
return ci != headers.end() ? ci->second : std::string();
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
void http_client::set_header(const std::string& header_name, long header_value)
|
||||
{
|
||||
char buf[21] = { 0 };
|
||||
#ifdef __WXMSW__
|
||||
::ltoa(header_value, buf, 10);
|
||||
#else
|
||||
sprintf(buf, "%ld", header_value);
|
||||
#endif
|
||||
set_header(header_name, buf);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
void http_client::set_header(const std::string& header_name, const std::string& header_value)
|
||||
{
|
||||
headers[header_name] = header_value;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
bool http_client::is_header_set(const std::string& header_name) const
|
||||
{
|
||||
stringmap::const_iterator ci = headers.find(header_name);
|
||||
return ci != headers.end() && !ci->second.empty();
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
void http_client::remove_header(const std::string& header_name)
|
||||
{
|
||||
headers.erase(header_name);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
void http_client::set_cookie(const std::string& cookie_name, long cookie_value)
|
||||
{
|
||||
char buf[21] = { 0 };
|
||||
#ifdef __WXMSW__
|
||||
::ltoa(cookie_value, buf, 10);
|
||||
#else
|
||||
sprintf(buf, "%ld", cookie_value);
|
||||
#endif
|
||||
set_cookie(cookie_name, buf);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
void http_client::set_cookie(const std::string& cookie_name, const std::string& cookie_value)
|
||||
{
|
||||
cookies[cookie_name] = cookie_value;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
void http_client::remove_cookie(const std::string& cookie_name)
|
||||
{
|
||||
cookies.erase(cookie_name);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
// POST
|
||||
const std::string& http_client::post_url (const std::string& url, const string_to_stringmap& postvars, const string_to_stringmap& filenames)
|
||||
{
|
||||
std::string CT;
|
||||
std::string postBody = build_post(CT, postvars, filenames);
|
||||
set_header("Content-Type", CT);
|
||||
set_header("Content-Length", static_cast<long>(postBody.size()));
|
||||
|
||||
grab_url(url, "POST", postBody);
|
||||
|
||||
return returned_body;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
const std::string& http_client::post_url (const std::string& url, const std::string& postbuffer)
|
||||
{
|
||||
if ( !is_header_set("Content-Type") ) // Maybe they just forgot it?
|
||||
set_header("Content-Type", "application/x-www-form-urlencoded");
|
||||
|
||||
set_header("Content-Length", static_cast<long>(postbuffer.size()));
|
||||
|
||||
grab_url(url, "POST", postbuffer);
|
||||
|
||||
return returned_body;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
std::string http_client::get_random_string( size_t length ) const
|
||||
{
|
||||
static bool has_seeded(false);
|
||||
static std::string allowed_chars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789");
|
||||
|
||||
if ( !has_seeded )
|
||||
{
|
||||
has_seeded = true;
|
||||
::srand( static_cast<unsigned int>(::time(NULL)) );
|
||||
}
|
||||
|
||||
std::string retVal; retVal.reserve(length);
|
||||
while ( retVal.size() < length )
|
||||
{
|
||||
retVal += allowed_chars[(rand() % allowed_chars.size())];
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
// static
|
||||
std::string http_client::urlencode(const std::string& in, bool post_encode)
|
||||
{
|
||||
static std::string allowed_chars("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_");
|
||||
|
||||
std::stringstream ss;
|
||||
ss << std::hex;
|
||||
for (std::string::const_iterator ci = in.begin(); ci != in.end(); ++ci)
|
||||
{
|
||||
if ( allowed_chars.find(*ci) != std::string::npos )
|
||||
{
|
||||
ss << *ci;
|
||||
}
|
||||
else if ( post_encode && *ci == ' ' )
|
||||
{
|
||||
ss << '+';
|
||||
}
|
||||
else
|
||||
{
|
||||
ss << '%' << std::setfill('0') << std::setw(2) << std::right << static_cast<int>(*ci);
|
||||
}
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
std::string http_client::get_basename( const std::string& filename ) const
|
||||
{
|
||||
std::string::size_type pos = filename.find_last_of("\\/");
|
||||
if ( pos == std::string::npos )
|
||||
return filename;
|
||||
else
|
||||
return filename.substr(pos+1);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
bool http_client::parse_url(
|
||||
const std::string& url,
|
||||
std::string& scheme,
|
||||
std::string& user,
|
||||
std::string& pass,
|
||||
std::string& host,
|
||||
short& port,
|
||||
std::string& path
|
||||
) const
|
||||
{
|
||||
scheme.clear();
|
||||
user.clear();
|
||||
pass.clear();
|
||||
host.clear();
|
||||
path.clear();
|
||||
port = 0;
|
||||
|
||||
// Find scheme
|
||||
std::string::size_type pos_scheme = url.find("://");
|
||||
if ( pos_scheme == std::string::npos )
|
||||
{
|
||||
pos_scheme = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
scheme = strtolower(url.substr(0, pos_scheme));
|
||||
pos_scheme += 3;
|
||||
}
|
||||
|
||||
std::string::size_type pos_path = url.find('/', pos_scheme);
|
||||
if ( pos_path == std::string::npos )
|
||||
{
|
||||
host = url.substr(pos_scheme);
|
||||
}
|
||||
else
|
||||
{
|
||||
host = url.substr(pos_scheme, pos_path - pos_scheme);
|
||||
path = url.substr(pos_path);
|
||||
}
|
||||
|
||||
std::string::size_type pos_at = host.find('@');
|
||||
if ( pos_at != std::string::npos )
|
||||
{
|
||||
std::string::size_type pos_dp = host.find(':');
|
||||
if ( pos_dp != std::string::npos && pos_dp < pos_at )
|
||||
{
|
||||
user = host.substr(0, pos_dp);
|
||||
pass = host.substr(pos_dp+1, pos_at-pos_dp-1);
|
||||
}
|
||||
else
|
||||
{
|
||||
user = host.substr(0, pos_at);
|
||||
}
|
||||
host = host.substr(pos_at+1);
|
||||
}
|
||||
|
||||
std::string::size_type pos_dp = host.find(':');
|
||||
if ( pos_dp != std::string::npos )
|
||||
{
|
||||
port = dlib::string_cast<short>(host.substr(pos_dp+1));
|
||||
host = host.substr(0, pos_dp);
|
||||
}
|
||||
|
||||
host = strtolower(host);
|
||||
|
||||
if ( port == 0 )
|
||||
{
|
||||
if ( scheme == "http" )
|
||||
port = 80;
|
||||
else if ( scheme == "ftp" )
|
||||
port = 21;
|
||||
else if ( scheme == "https" )
|
||||
port = 443;
|
||||
}
|
||||
|
||||
return !host.empty();
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
std::string http_client::strtolower(const std::string& in) const
|
||||
{
|
||||
std::string retVal = in;
|
||||
|
||||
for (std::string::iterator ii = retVal.begin(); ii != retVal.end(); ++ii)
|
||||
{
|
||||
*ii = ::tolower(*ii);
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
std::string http_client::strtoupper(const std::string& in) const
|
||||
{
|
||||
std::string retVal = in;
|
||||
|
||||
for (std::string::iterator ii = retVal.begin(); ii != retVal.end(); ++ii)
|
||||
{
|
||||
*ii = ::toupper(*ii);
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
// GET
|
||||
const std::string& http_client::get_url(const std::string& url)
|
||||
{
|
||||
std::string CT = get_header("Content-Type");
|
||||
|
||||
// You do a GET with a POST header??
|
||||
if ( CT == "application/x-www-form-urlencoded" || CT == "multipart/form-data" )
|
||||
remove_header("Content-Type");
|
||||
|
||||
grab_url(url);
|
||||
|
||||
return returned_body;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
std::string http_client::build_post(std::string& content_type, const string_to_stringmap& postvars, const string_to_stringmap& filenames_in) const
|
||||
{
|
||||
if ( postvars.empty() && filenames_in.empty() )
|
||||
return std::string();
|
||||
|
||||
string_to_stringmap filenames = filenames_in;
|
||||
|
||||
// sanitize the files
|
||||
if ( !filenames.empty() )
|
||||
{
|
||||
string_to_stringmap::iterator var_names = filenames.begin();
|
||||
while (var_names != filenames.end())
|
||||
{
|
||||
stringmap::iterator fnames = var_names->second.begin();
|
||||
|
||||
while( fnames != var_names->second.end() )
|
||||
{
|
||||
FILE *fp = ::fopen(fnames->second.c_str(), "rb");
|
||||
if ( fp == NULL )
|
||||
{
|
||||
stringmap::iterator old_one = fnames++;
|
||||
var_names->second.erase(old_one);
|
||||
}
|
||||
else
|
||||
{
|
||||
fclose(fp);
|
||||
++fnames;
|
||||
}
|
||||
}
|
||||
|
||||
if ( fnames->second.empty() )
|
||||
{
|
||||
string_to_stringmap::iterator old_one = var_names++;
|
||||
filenames.erase(old_one);
|
||||
}
|
||||
else
|
||||
{
|
||||
++var_names;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
content_type = !filenames.empty() ? "multipart/form-data" : "application/x-www-form-urlencoded";
|
||||
std::stringstream postBody;
|
||||
if ( !filenames.empty() )
|
||||
{
|
||||
std::string mime_boundary = get_random_string(32);
|
||||
|
||||
// First add the form vars
|
||||
for (string_to_stringmap::const_iterator ci = postvars.begin(); ci != postvars.end(); ++ci)
|
||||
{
|
||||
for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
|
||||
{
|
||||
postBody << "--" << mime_boundary << "\r\n"
|
||||
"Content-Disposition: form-data; name=\"" << ci->first << "\"\r\n\r\n"
|
||||
<< si->second << "\r\n";
|
||||
}
|
||||
}
|
||||
|
||||
// Then add the files
|
||||
for (string_to_stringmap::const_iterator ci = filenames.begin(); ci != filenames.end(); ++ci)
|
||||
{
|
||||
for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
|
||||
{
|
||||
std::ifstream in(si->second.c_str());
|
||||
postBody << "--" << mime_boundary << "\r\n"
|
||||
"Content-Disposition: form-data; name=\"" << ci->first << "\"; filename=\"" << get_basename(si->second) << "\"\r\n\r\n"
|
||||
<< in << "\r\n";
|
||||
}
|
||||
}
|
||||
|
||||
postBody << "--" << mime_boundary << "--\r\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
// No files...
|
||||
for (string_to_stringmap::const_iterator ci = postvars.begin(); ci != postvars.end(); ++ci)
|
||||
{
|
||||
for (stringmap::const_iterator si = ci->second.begin(); si != ci->second.end(); ++si)
|
||||
{
|
||||
postBody << urlencode(ci->first) << '=' << urlencode(si->second) << '&';
|
||||
}
|
||||
}
|
||||
|
||||
// read the last '&'
|
||||
char c;
|
||||
postBody.read(&c, 1);
|
||||
}
|
||||
|
||||
return postBody.str();
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
bool http_client::grab_url(const std::string& url, const std::string& method, const std::string& post_body)
|
||||
{
|
||||
error_field.clear();
|
||||
returned_headers.clear();
|
||||
http_return = 0;
|
||||
returned_body.clear();
|
||||
|
||||
std::string to_use_method = strtoupper(method);
|
||||
|
||||
std::string scheme, user, pass, host, path;
|
||||
short port;
|
||||
if ( !parse_url(url, scheme, user, pass, host, port, path) )
|
||||
{
|
||||
error_field = "Couldn't parse the URL!";
|
||||
return false;
|
||||
}
|
||||
|
||||
// Build request
|
||||
std::stringstream ret;
|
||||
ret << to_use_method << ' ' << path << " HTTP/1.0\r\n"
|
||||
<< "Host: " << host;
|
||||
if (port != 80 && port != 443) ret << ':' << port;
|
||||
ret << "\r\n";
|
||||
|
||||
bool content_length_said = false;
|
||||
|
||||
set_header("Connection", "Close");
|
||||
for (stringmap::iterator ci = headers.begin(); ci != headers.end(); ++ci)
|
||||
{
|
||||
std::string head = strtolower(ci->first);
|
||||
|
||||
if ( head == "content-length" )
|
||||
{
|
||||
content_length_said = true;
|
||||
}
|
||||
|
||||
ret << ci->first << ':' << ' ' << ci->second << "\r\n";
|
||||
}
|
||||
|
||||
if ( !content_length_said && to_use_method != "GET" )
|
||||
ret << "Content-Length: " << static_cast<unsigned int>(post_body.size()) << "\r\n";
|
||||
|
||||
std::stringstream cookie_ss;
|
||||
for (stringmap::iterator ci = cookies.begin(); ci != cookies.end(); ++ci)
|
||||
{
|
||||
std::string var = ci->first ; trim(var);
|
||||
std::string val = ci->second; trim(val);
|
||||
|
||||
if ( val.empty() || var.empty() )
|
||||
continue;
|
||||
|
||||
if ( !cookie_ss.str().empty() )
|
||||
cookie_ss << ';' << ' ';
|
||||
|
||||
cookie_ss << urlencode(var) << '=' << urlencode(val);
|
||||
}
|
||||
|
||||
if ( !cookie_ss.str().empty() )
|
||||
ret << "Cookie: " << cookie_ss.str() << "\r\n";
|
||||
|
||||
ret << "\r\n";
|
||||
ret << post_body;
|
||||
|
||||
std::string request_build = ret.str();
|
||||
|
||||
std::stringstream ss;
|
||||
{
|
||||
dlib::connection * conn(0);
|
||||
try
|
||||
{
|
||||
conn = dlib::connect(host, port);
|
||||
}
|
||||
catch (const dlib::socket_error& e)
|
||||
{
|
||||
error_field = e.what();
|
||||
return false;
|
||||
}
|
||||
|
||||
// Implement a timeout
|
||||
timeout_ptr t;
|
||||
if ( timeout > 0 )
|
||||
t.reset( new dlib::timeout::kernel_1a(*conn, &dlib::connection::shutdown, timeout) );
|
||||
|
||||
// Write our request
|
||||
conn->write(request_build.c_str(), static_cast<long>(request_build.size()));
|
||||
|
||||
t.reset();
|
||||
|
||||
// And read the response
|
||||
char buf[512];
|
||||
long bytes_read(0), bytes_total(0);
|
||||
bool read_headers(true);
|
||||
|
||||
if ( timeout > 0 )
|
||||
t.reset( new dlib::timeout::kernel_1a(*conn, &dlib::connection::shutdown, timeout) );
|
||||
|
||||
while ( (bytes_read = conn->read(buf, 512)) > 0 )
|
||||
{
|
||||
ss.write(buf, bytes_read);
|
||||
|
||||
// Incremental read headers
|
||||
if ( read_headers )
|
||||
{
|
||||
std::string body_with_headers = ss.str();
|
||||
std::string::size_type ctr(0);
|
||||
|
||||
while ( true )
|
||||
{
|
||||
std::string::size_type pos = body_with_headers.find("\r\n", ctr);
|
||||
if ( pos == std::string::npos )
|
||||
{
|
||||
// This is our last position of "\r\n"
|
||||
ss.str("");
|
||||
ss.write( body_with_headers.substr(ctr).c_str(), body_with_headers.size() - ctr );
|
||||
break;
|
||||
}
|
||||
|
||||
std::string header = body_with_headers.substr(ctr, pos-ctr);
|
||||
if ( header.empty() )
|
||||
{
|
||||
// Ok, we're done reading the headers
|
||||
read_headers = false;
|
||||
// What follows now is the body
|
||||
ss.str("");
|
||||
ss.write( body_with_headers.substr(pos + 2).c_str(), body_with_headers.size() - pos - 2 );
|
||||
break;
|
||||
}
|
||||
ctr = pos + 2;
|
||||
|
||||
if ( returned_headers.empty() )
|
||||
{
|
||||
if (
|
||||
header[0] == 'H' &&
|
||||
header[1] == 'T' &&
|
||||
header[2] == 'T' &&
|
||||
header[3] == 'P' &&
|
||||
header[4] == '/' &&
|
||||
(header[5] >= '0' && header[5] <= '9') &&
|
||||
header[6] == '.' &&
|
||||
(header[7] >= '0' && header[7] <= '9') &&
|
||||
header[8] == ' '
|
||||
)
|
||||
{
|
||||
http_return = (header[9 ] - '0') * 100 +
|
||||
(header[10] - '0') * 10 +
|
||||
(header[11] - '0');
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
std::string::size_type pos_dp = header.find_first_of(':');
|
||||
std::string header_name, header_value;
|
||||
if ( pos_dp == std::string::npos )
|
||||
{
|
||||
// **TODO** what should I do here??
|
||||
header_name = header;
|
||||
}
|
||||
else
|
||||
{
|
||||
header_name = trim(header.substr(0, pos_dp));
|
||||
header_value = trim(header.substr(pos_dp+1));
|
||||
}
|
||||
|
||||
returned_headers[ header_name ].push_back(header_value);
|
||||
|
||||
if ( BR_CASECMP(header_name.c_str(), "Content-Length", 14) == 0 )
|
||||
{
|
||||
bytes_total = atol( header_value.c_str() );
|
||||
}
|
||||
else if ( BR_CASECMP(header_name.c_str(), "Set-Cookie", 10) == 0 )
|
||||
{
|
||||
std::string::size_type cur_pos(0), pos_pk, pos_is;
|
||||
std::string work, var, val;
|
||||
for ( cur_pos = 0; cur_pos < header_value.size(); cur_pos++ )
|
||||
{
|
||||
pos_pk = header_value.find(';', cur_pos);
|
||||
work = trim( header_value.substr(cur_pos, pos_pk - cur_pos) );
|
||||
|
||||
pos_is = work.find('=');
|
||||
if ( pos_is != std::string::npos )
|
||||
{ // Hmmm? what in the else case?
|
||||
var = trim( http_client::urldecode( work.substr(0, pos_is) ) );
|
||||
val = trim( http_client::urldecode( work.substr(pos_is + 1) ) );
|
||||
|
||||
if ( var != "expires" && var != "domain" && var != "path" )
|
||||
set_cookie( var, val );
|
||||
}
|
||||
cur_pos = pos_pk == std::string::npos ? pos_pk - 1 : pos_pk;
|
||||
}
|
||||
} // Set-Cookie?
|
||||
|
||||
} // while (true)
|
||||
} // read_headers?
|
||||
|
||||
// Call the OnDownload function if it's set
|
||||
if ( OnDownload && !read_headers )
|
||||
{
|
||||
if ( (*OnDownload)(static_cast<long>(ss.tellp()), bytes_total, user_info) == false )
|
||||
{
|
||||
t.reset();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ( bytes_total != 0 && static_cast<long>(ss.tellp()) == bytes_total )
|
||||
{
|
||||
t.reset();
|
||||
break;
|
||||
}
|
||||
|
||||
if ( timeout > 0 )
|
||||
t.reset( new dlib::timeout::kernel_1a(*conn, &dlib::connection::shutdown, timeout) );
|
||||
} // while still data to read
|
||||
|
||||
t.reset();
|
||||
|
||||
delete conn;
|
||||
|
||||
|
||||
switch ( bytes_read )
|
||||
{
|
||||
case dlib::TIMEOUT: error_field = "Timeout"; return false; break;
|
||||
case dlib::WOULDBLOCK: error_field = "Would block"; return false; break;
|
||||
case dlib::OTHER_ERROR: error_field = "Other error"; return false; break;
|
||||
case dlib::SHUTDOWN: error_field = "Timeout"; return false; break;
|
||||
case dlib::PORTINUSE: error_field = "Port in use"; return false; break;
|
||||
}
|
||||
}
|
||||
|
||||
returned_body = ss.str();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
void http_client::clear()
|
||||
{
|
||||
headers.clear();
|
||||
cookies.clear();
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
void http_client::prepare_for_next_url( )
|
||||
{
|
||||
remove_header("Content-Type");
|
||||
remove_header("Content-Length");
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,101 @@
|
||||
#ifndef DLIB__BROWSER_H
|
||||
#define DLIB__BROWSER_H
|
||||
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "http_client_abstract.h"
|
||||
|
||||
|
||||
// Default timeout after 60 seconds
|
||||
#define DEFAULT_TIMEOUT 60000
|
||||
|
||||
namespace dlib
|
||||
{
|
||||
|
||||
// Function which is called when there is data available.
|
||||
// Return false to stop the download process...
|
||||
typedef bool (*fnOnDownload)(long already_downloaded, long total_to_download, void * userInfo);
|
||||
|
||||
|
||||
class http_client
|
||||
{
|
||||
public:
|
||||
http_client();
|
||||
|
||||
typedef std::map< std::string, std::string > stringmap;
|
||||
typedef std::map< std::string, stringmap > string_to_stringmap;
|
||||
typedef std::map< std::string, std::vector<std::string> > string_to_stringvector;
|
||||
|
||||
// Header functions
|
||||
void set_header(const std::string& header_name, const std::string& header_value);
|
||||
void set_header(const std::string& header_name, long header_value);
|
||||
std::string get_header(const std::string& header_name) const;
|
||||
void remove_header(const std::string& header_name);
|
||||
bool is_header_set(const std::string& header_name) const;
|
||||
|
||||
// This function will clear out all cookies & headers set until now
|
||||
void clear();
|
||||
// This function will clear out the Content-Type header
|
||||
void prepare_for_next_url();
|
||||
|
||||
void set_callback_function( fnOnDownload od, void * _user_info ) { OnDownload = od; user_info = _user_info; }
|
||||
|
||||
void set_cookie(const std::string& cookie_name, const std::string& cookie_value);
|
||||
void set_cookie(const std::string& cookie_name, long cookie_value);
|
||||
void remove_cookie(const std::string& cookie_name);
|
||||
|
||||
void set_user_agent(const std::string& new_agent) { set_header("User-Agent", new_agent); }
|
||||
|
||||
|
||||
void set_timeout( unsigned int milliseconds = DEFAULT_TIMEOUT ) { timeout = milliseconds; }
|
||||
|
||||
|
||||
string_to_stringvector get_returned_headers() const { return returned_headers; }
|
||||
short get_http_return () const { return http_return; }
|
||||
const std::string& get_body () const { return returned_body; }
|
||||
|
||||
// POST
|
||||
const std::string& post_url (const std::string& url, const string_to_stringmap& postvars, const string_to_stringmap& filenames = string_to_stringmap());
|
||||
const std::string& post_url (const std::string& url, const std::string& postbuffer);
|
||||
// GET
|
||||
const std::string& get_url (const std::string& url);
|
||||
|
||||
bool has_error( ) const { return !error_field.empty(); }
|
||||
const std::string& get_error( ) const { return error_field; }
|
||||
|
||||
static std::string urlencode(const std::string& in, bool post_encode = false);
|
||||
static std::string urldecode(const std::string& in);
|
||||
private:
|
||||
bool grab_url(const std::string& url, const std::string& method = "GET", const std::string& post_body = "");
|
||||
std::string build_post(std::string& content_type, const string_to_stringmap& postvars, const string_to_stringmap& filenames) const;
|
||||
|
||||
std::string get_random_string( size_t length = 32 ) const;
|
||||
std::string get_basename( const std::string& filename ) const;
|
||||
std::string strtolower(const std::string& in) const;
|
||||
std::string strtoupper(const std::string& in) const;
|
||||
|
||||
bool parse_url(const std::string& url, std::string& scheme, std::string& user, std::string& pass, std::string& host, short& port, std::string& path) const;
|
||||
|
||||
stringmap headers;
|
||||
stringmap cookies;
|
||||
|
||||
string_to_stringvector returned_headers;
|
||||
short http_return;
|
||||
std::string returned_body, error_field;
|
||||
|
||||
unsigned int timeout;
|
||||
|
||||
fnOnDownload OnDownload;
|
||||
void * user_info;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#ifdef NO_MAKEFILE
|
||||
#include "http_client.cpp"
|
||||
#endif
|
||||
|
||||
#endif // DLIB__BROWSER_H
|
||||
|
||||
@ -0,0 +1,218 @@
|
||||
#undef DLIB__BROWSER_ABSTRACT_
|
||||
#ifdef DLIB__BROWSER_ABSTRACT_
|
||||
|
||||
|
||||
|
||||
namespace dlib
|
||||
{
|
||||
|
||||
// Function which is called when there is data available.
|
||||
// Return false to stop the download process...
|
||||
typedef bool (*fnOnDownload)(long already_downloaded, long total_to_download, void * userInfo);
|
||||
|
||||
|
||||
// ----------------------------------------------------------------------------------------
|
||||
/*
|
||||
TODO:
|
||||
- Timed cookie support
|
||||
- POSTing files: check it!
|
||||
- Don't timeout when still downloading!
|
||||
*/
|
||||
// ----------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Browser
|
||||
{
|
||||
/*!
|
||||
WHAT THIS OBJECT REPRESENTS
|
||||
This object represents a possibility for the end user to download webpages (HTTP/1.0)
|
||||
from the internet like a normal webbrowser would do.
|
||||
!*/
|
||||
|
||||
public:
|
||||
|
||||
Browser(
|
||||
);
|
||||
/*!
|
||||
Constructor
|
||||
!*/
|
||||
|
||||
void set_header(
|
||||
const std::string& header_name,
|
||||
const std::string& header_value
|
||||
);
|
||||
/*!
|
||||
Set a header to a certain value
|
||||
Example: set_header("User-Agent", "Internet Explorer")
|
||||
!*/
|
||||
|
||||
|
||||
void set_header(
|
||||
const std::string& header_name,
|
||||
long header_value
|
||||
);
|
||||
/*!
|
||||
Set a header to a certain number
|
||||
Example: set_header("Content-Length", 1234)
|
||||
!*/
|
||||
|
||||
std::string get_header(
|
||||
const std::string& header_name
|
||||
) const;
|
||||
/*!
|
||||
Get the value of the header or an empty string when it's not set.
|
||||
Example: get_header("Content-Length") would return "1234"
|
||||
!*/
|
||||
|
||||
void remove_header(
|
||||
const std::string& header_name
|
||||
);
|
||||
/*!
|
||||
Removes a certain header
|
||||
!*/
|
||||
|
||||
bool is_header_set(
|
||||
const std::string& header_name
|
||||
) const;
|
||||
/*!
|
||||
Returns when a header is set and is not empty
|
||||
!*/
|
||||
|
||||
void set_user_agent(
|
||||
const std::string& new_agent
|
||||
) { set_header("User-Agent", new_agent); }
|
||||
/*!
|
||||
Convenience function for setting a user agent
|
||||
!*/
|
||||
|
||||
void clear(
|
||||
);
|
||||
/*!
|
||||
Clear out all cookies & headers set until now
|
||||
!*/
|
||||
|
||||
void prepare_for_next_url(
|
||||
);
|
||||
/*!
|
||||
Clear out any header and/or cookie which would obstruct getting a next page.
|
||||
At this moment this is cleared:
|
||||
- the Content-Type header
|
||||
!*/
|
||||
|
||||
void set_callback_function(
|
||||
fnOnDownload od,
|
||||
void * _user_info
|
||||
);
|
||||
/*!
|
||||
Set a callback function for one of the following events:
|
||||
- OnDownload: this will tell you how much is downloaded and how much will need to be downloaded
|
||||
!*/
|
||||
|
||||
void set_cookie(
|
||||
const std::string& cookie_name,
|
||||
const std::string& cookie_value
|
||||
);
|
||||
/*!
|
||||
Set a cookie
|
||||
!*/
|
||||
|
||||
void set_cookie(
|
||||
const std::string& cookie_name,
|
||||
long cookie_value
|
||||
);
|
||||
/*!
|
||||
Set a cookie
|
||||
!*/
|
||||
|
||||
void remove_cookie(
|
||||
const std::string& cookie_name
|
||||
);
|
||||
/*!
|
||||
Remove a cookie if it's set
|
||||
!*/
|
||||
|
||||
void set_timeout(
|
||||
unsigned int milliseconds
|
||||
);
|
||||
/*!
|
||||
Set the maximum time how long a request can take. Setting this to 0 disables
|
||||
this behavior.
|
||||
!*/
|
||||
|
||||
string_to_stringvector get_returned_headers(
|
||||
) const;
|
||||
/*!
|
||||
Returns all the headers which are returned in the download of the webpage.
|
||||
!*/
|
||||
|
||||
short get_http_return (
|
||||
) const;
|
||||
/*!
|
||||
Retrieves the HTTP return code.
|
||||
!*/
|
||||
|
||||
const std::string& get_body (
|
||||
) const;
|
||||
/*!
|
||||
Retrieves the HTTP body.
|
||||
!*/
|
||||
|
||||
const std::string& post_url (
|
||||
const std::string& url,
|
||||
const string_to_stringmap& postvars,
|
||||
const string_to_stringmap& filenames = string_to_stringmap()
|
||||
);
|
||||
/*!
|
||||
POST an url to the internet.
|
||||
You can pass the post variables as well as a list of filenames
|
||||
!*/
|
||||
|
||||
const std::string& post_url (
|
||||
const std::string& url,
|
||||
const std::string& postbuffer
|
||||
);
|
||||
/*!
|
||||
POST an url to the internet.
|
||||
In this function you have constructed the POST string yourselves
|
||||
!*/
|
||||
|
||||
const std::string& get_url (
|
||||
const std::string& url
|
||||
);
|
||||
/*!
|
||||
GET an url from the internet.
|
||||
!*/
|
||||
|
||||
bool has_error(
|
||||
) const;
|
||||
/*!
|
||||
Has there happened an error?
|
||||
!*/
|
||||
|
||||
const std::string& get_error(
|
||||
) const;
|
||||
/*!
|
||||
Get the error explanation
|
||||
!*/
|
||||
|
||||
static std::string urlencode(
|
||||
const std::string& in,
|
||||
bool post_encode = false
|
||||
);
|
||||
/*!
|
||||
Convenience function to URLencode a string
|
||||
!*/
|
||||
|
||||
static std::string urldecode(
|
||||
const std::string& in
|
||||
);
|
||||
/*!
|
||||
Convenience function to URLdecode a string
|
||||
!*/
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif // DLIB__BROWSER_ABSTRACT_
|
||||
|
||||
Loading…
Reference in new issue