36 #ifndef _uhal_utilities_files_hpp_ 37 #define _uhal_utilities_files_hpp_ 47 #include <boost/filesystem/path.hpp> 48 #include <boost/filesystem/operations.hpp> 50 #include <boost/asio/error.hpp> 51 #include <boost/asio/io_service.hpp> 52 #include <boost/asio/ip/tcp.hpp> 53 #include <boost/asio/ip/udp.hpp> 54 #include <boost/asio/streambuf.hpp> 55 #include <boost/asio/read.hpp> 56 #include <boost/asio/write.hpp> 58 #include <boost/spirit/home/qi/parse.hpp> 59 #include <boost/spirit/include/qi_char_.hpp> 61 #include <boost/bind/bind.hpp> 67 #include "pugixml.hpp" 69 #include "uhal/log/log.hpp" 72 #include "boost/unordered_map.hpp" 81 UHAL_DEFINE_EXCEPTION_CLASS ( UriListParsingError ,
"Exception class to handle the case where the string is not a comma-delimiter list of URIs." )
83 UHAL_DEFINE_EXCEPTION_CLASS ( NonSupportedUriProtocol , "Exception class to handle the case where a URI contains a non-supported protocol." )
89 UHAL_DEFINE_EXCEPTION_CLASS ( ExpandingShellExpressionFailed , "Exception class to handle the case where expanding a shell expression failed." )
99 void ParseSemicolonDelimitedUriList (
const std::string& aSemicolonDelimitedUriList , std::vector< std::pair<std::string, std::string> >& aUriList );
119 void ShellExpandFilenameExpr (
const std::string& aFilenameExpr ,
const boost::filesystem::path& aParentPath , std::vector< boost::filesystem::path >& aFiles );
137 template <
bool DebugInfo >
144 log (
Info() ,
"Retrieving URL http://" , aURL );
146 catch (
const std::exception& aExc )
152 std::pair<std::string, std::string> lURLPair;
157 boost::spirit::qi::phrase_parse ( aURL.begin() ,
159 + ( boost::spirit::qi::char_ -
"/" ) >> -boost::spirit::qi::lit (
"/" ) >> + ( boost::spirit::qi::char_ ) ,
160 boost::spirit::ascii::space ,
163 catch (
const std::exception& aExc )
169 boost::system::error_code lErrorCode ( boost::asio::error::host_not_found );
171 boost::asio::io_service io_service;
173 boost::asio::ip::tcp::resolver resolver ( io_service );
174 boost::asio::ip::tcp::resolver::query query ( lURLPair.first ,
"http" );
175 boost::asio::ip::tcp::resolver::iterator endpoint_iterator = resolver.resolve ( query );
176 boost::asio::ip::tcp::resolver::iterator end;
178 boost::asio::ip::tcp::socket socket ( io_service );
182 while ( lErrorCode && endpoint_iterator != end )
185 socket.connect ( *endpoint_iterator++, lErrorCode );
190 throw boost::system::system_error ( lErrorCode );
193 catch (
const std::exception& aExc )
200 boost::asio::streambuf request;
201 std::ostream request_stream ( &request );
202 request_stream <<
"GET /" << lURLPair.second <<
" HTTP/1.0\r\n";
203 request_stream <<
"Host: " << lURLPair.first <<
"\r\n";
204 request_stream <<
"Accept: */*\r\n";
205 request_stream <<
"Connection: close\r\n\r\n";
214 throw boost::system::system_error ( lErrorCode );
217 catch (
const std::exception& aExc )
224 static const int mDefaultBufferSize ( 65536 );
225 typedef std::vector<uint8_t> BufferType;
226 BufferType mBuffer ( mDefaultBufferSize , uint8_t ( 0 ) );
227 std::size_t lSize ( 0 );
234 lSize += boost::asio::read ( socket, boost::asio::buffer ( &mBuffer[lSize] , mDefaultBufferSize ) , lErrorCode );
238 if ( lErrorCode == boost::asio::error::eof )
244 throw boost::system::system_error ( lErrorCode );
248 mBuffer.insert ( mBuffer.end() , mDefaultBufferSize , uint8_t ( 0 ) );
251 catch (
const std::exception& aExc )
257 mBuffer.resize ( lSize );
259 grammars::HttpResponseGrammar lGrammar2;
260 boost::spirit::qi::phrase_parse ( mBuffer.begin() , mBuffer.end() , lGrammar2 , boost::spirit::ascii::space , aResponse );
266 log (
Info() ,
"HTTP response parsed as:\n" , aResponse );
268 catch (
const std::exception& aExc )
270 log (
Error() ,
"EXCEPTION: " , aExc.what() ,
" caught in " ,
ThisLocation() ,
" Continuing." );
275 if ( aResponse.
method !=
"HTTP" || aResponse.
status != 200 )
299 template <
typename R ,
typename F ,
typename L>
300 void OpenFileLocal (
const std::string& aFilenameExpr ,
const boost::filesystem::path& aParentPath , boost::_bi::bind_t<R,F,L> aBinder )
302 std::vector< boost::filesystem::path > lFilePaths;
305 for ( std::vector< boost::filesystem::path >::iterator lIt2 = lFilePaths.begin() ; lIt2 != lFilePaths.end() ; ++ lIt2 )
307 std::ifstream lStr ( lIt2->c_str() );
309 if ( !lStr.is_open() )
311 uhal::exception::CannotOpenFile lExc;
312 log ( lExc ,
"Failed to open " , lIt2->c_str() ,
". Continuing with next document for now but be aware!" );
317 lStr.seekg ( 0, std::ios::end );
318 std::vector<uint8_t> lFile ( lStr.tellg() , 0 );
319 lStr.seekg ( 0, std::ios::beg );
320 lStr.read ( (
char* ) & ( lFile[0] ) , lFile.size() );
321 aBinder ( std::string (
"file" ) , *lIt2 , boost::ref ( lFile ) );
334 template <
typename R ,
typename F ,
typename L>
335 void OpenFileHttp (
const std::string& aURL , boost::_bi::bind_t<R,F,L> aBinder )
339 if ( ! uhal::utilities::HttpGet<true> ( aURL , lHttpResponse ) )
341 uhal::exception::CannotOpenFile lExc;
342 log ( lExc ,
"Failed to download file " , aURL ,
". Continuing for now but be aware!" );
346 boost::filesystem::path lFilePath = boost::filesystem::path ( aURL );
347 aBinder ( std::string (
"http" ) , lFilePath , boost::ref ( lHttpResponse.
content ) );
358 template <
typename R ,
typename F ,
typename L>
359 void OpenFile (
const std::string& aProtocol ,
const std::string& aFilenameExpr ,
const boost::filesystem::path& aParentPath , boost::_bi::bind_t<R,F,L> aBinder )
361 if ( aProtocol ==
"file" )
365 else if ( aProtocol ==
"http" )
371 uhal::exception::NonSupportedUriProtocol lExc;
372 log ( lExc ,
"The protocol " ,
Quote ( aProtocol ) ,
" for file " ,
Quote ( aFilenameExpr ) ,
" is not supported." );
373 log ( lExc ,
"The supported protocols are " ,
Quote (
"file://" ) ,
" and " ,
Quote (
"http://" ) );
void ParseSemicolonDelimitedUriList(const std::string &aSemicolonDelimitedUriList, std::vector< std::pair< std::string, std::string > > &aUriList)
Parse a semicolon delimited list of URIs into a vector of protocol/address pairs. ...
int status
the response status
std::string method
the http transport method
#define UHAL_DEFINE_EXCEPTION_CLASS(ClassName, ClassDescription)
void OpenFileLocal(const std::string &aFilenameExpr, const boost::filesystem::path &aParentPath, boost::_bi::bind_t< R, F, L > aBinder)
Given a linux shell expression, open all files which match it and call the callback function on each ...
void OpenFileHttp(const std::string &aURL, boost::_bi::bind_t< R, F, L > aBinder)
Given a URL, retrieve the file and call the callback function on each of them.
void ShellExpandFilenameExpr(const std::string &aFilenameExpr, const boost::filesystem::path &aParentPath, std::vector< boost::filesystem::path > &aFiles)
Perform shell expansion of a linux shell expression ( e.g.
_Quote< T > Quote(const T &aT)
Struct to store an http response received from a server when parsed by boost spirit.
void OpenFile(const std::string &aProtocol, const std::string &aFilenameExpr, const boost::filesystem::path &aParentPath, boost::_bi::bind_t< R, F, L > aBinder)
Given a protocol and either a URL or a linux shell expression, open the file and call the callback fu...
bool HttpGet(const std::string &aURL, HttpResponseType &aResponse)
Retrieve a file by HTTP.
std::vector< uint8_t > content
parsed message content