μHAL (v2.6.5)
Part of the IPbus software repository
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
files.hpp
Go to the documentation of this file.
1 /*
2 ---------------------------------------------------------------------------
3 
4  This file is part of uHAL.
5 
6  uHAL is a hardware access library and programming framework
7  originally developed for upgrades of the Level-1 trigger of the CMS
8  experiment at CERN.
9 
10  uHAL is free software: you can redistribute it and/or modify
11  it under the terms of the GNU General Public License as published by
12  the Free Software Foundation, either version 3 of the License, or
13  (at your option) any later version.
14 
15  uHAL is distributed in the hope that it will be useful,
16  but WITHOUT ANY WARRANTY; without even the implied warranty of
17  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  GNU General Public License for more details.
19 
20  You should have received a copy of the GNU General Public License
21  along with uHAL. If not, see <http://www.gnu.org/licenses/>.
22 
23 
24  Andrew Rose, Imperial College, London
25  email: awr01 <AT> imperial.ac.uk
26 
27  Marc Magrans de Abril, CERN
28  email: marc.magrans.de.abril <AT> cern.ch
29 
30  Tom Williams, Rutherford Appleton Laboratory, Oxfordshire
31  email: tom.williams <AT> cern.ch
32 
33 ---------------------------------------------------------------------------
34 */
35 
36 #ifndef _uhal_utilities_files_hpp_
37 #define _uhal_utilities_files_hpp_
38 
39 #include <vector>
40 #include <string>
41 #include <iostream>
42 #include <fstream>
43 
44 #include <wordexp.h>
45 
46 //#include <boost/filesystem.hpp>
47 #include <boost/filesystem/path.hpp>
48 #include <boost/filesystem/operations.hpp>
49 //#include <boost/asio.hpp>
50 #include <boost/asio/error.hpp>
51 #include <boost/asio/io_service.hpp>
52 #include <boost/asio/ip/tcp.hpp>
53 #include <boost/asio/ip/udp.hpp>
54 #include <boost/asio/streambuf.hpp>
55 #include <boost/asio/read.hpp>
56 #include <boost/asio/write.hpp>
57 //#include <boost/spirit/include/qi.hpp>
58 #include <boost/spirit/home/qi/parse.hpp>
59 #include <boost/spirit/include/qi_char_.hpp>
60 //#include <boost/thread/mutex.hpp>
61 #include <boost/bind/bind.hpp>
62 
66 
67 #include "pugixml.hpp"
68 
69 #include "uhal/log/log.hpp"
70 #include "uhal/log/exception.hpp"
71 
72 #include "boost/unordered_map.hpp"
73 
74 
75 // ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
76 namespace uhal
77 {
78  namespace exception
79  {
81  UHAL_DEFINE_EXCEPTION_CLASS ( UriListParsingError , "Exception class to handle the case where the string is not a comma-delimiter list of URIs." )
83  UHAL_DEFINE_EXCEPTION_CLASS ( NonSupportedUriProtocol , "Exception class to handle the case where a URI contains a non-supported protocol." )
85  UHAL_DEFINE_EXCEPTION_CLASS ( CannotOpenFile , "Exception class to handle the case where a URI can not be opened." )
87  UHAL_DEFINE_EXCEPTION_CLASS ( FileNotFound , "Exception class to handle the case where a URI using the 'file://' protocol can not be expanded." )
89  UHAL_DEFINE_EXCEPTION_CLASS ( ExpandingShellExpressionFailed , "Exception class to handle the case where expanding a shell expression failed." )
90  }
91 
92  namespace utilities
93  {
99  void ParseSemicolonDelimitedUriList ( const std::string& aSemicolonDelimitedUriList , std::vector< std::pair<std::string, std::string> >& aUriList );
100  }
101 }
102 // ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
103 
104 
105 
106 // ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
107 namespace uhal
108 {
109  // boost::mutex gUtilityMutex;
110 
111  namespace utilities
112  {
119  void ShellExpandFilenameExpr ( const std::string& aFilenameExpr , const boost::filesystem::path& aParentPath , std::vector< boost::filesystem::path >& aFiles );
120  }
121 }
122 // ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
123 
124 
125 
126 // ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
127 namespace uhal
128 {
129  namespace utilities
130  {
137  template < bool DebugInfo >
138  bool HttpGet ( const std::string& aURL , HttpResponseType& aResponse )
139  {
140  if ( DebugInfo )
141  {
142  try
143  {
144  log ( Info() , "Retrieving URL http://" , aURL );
145  }
146  catch ( const std::exception& aExc )
147  {
148  log ( Error() , "Exception " , Quote ( aExc.what() ) , " caught at " , ThisLocation() ); // Just debugging so although exception is worrying, it is not critical
149  }
150  }
151 
152  std::pair<std::string, std::string> lURLPair;
153 
154  try
155  {
156  //split at the first slash
157  boost::spirit::qi::phrase_parse ( aURL.begin() ,
158  aURL.end() ,
159  + ( boost::spirit::qi::char_ - "/" ) >> -boost::spirit::qi::lit ( "/" ) >> + ( boost::spirit::qi::char_ ) ,
160  boost::spirit::ascii::space ,
161  lURLPair );
162  }
163  catch ( const std::exception& aExc )
164  {
165  // log ( Error() , "Exception " , Quote ( aExc.what() ) , " caught at " , ThisLocation() );
166  return false;
167  }
168 
169  boost::system::error_code lErrorCode ( boost::asio::error::host_not_found );
170  // The IO service everything will go through
171  boost::asio::io_service io_service;
172  // Get a list of endpoints corresponding to the server name.
173  boost::asio::ip::tcp::resolver resolver ( io_service );
174  boost::asio::ip::tcp::resolver::query query ( lURLPair.first , "http" );
175  boost::asio::ip::tcp::resolver::iterator endpoint_iterator = resolver.resolve ( query );
176  boost::asio::ip::tcp::resolver::iterator end;
177  // Try each endpoint until we successfully establish a connection.
178  boost::asio::ip::tcp::socket socket ( io_service );
179 
180  try
181  {
182  while ( lErrorCode && endpoint_iterator != end )
183  {
184  socket.close();
185  socket.connect ( *endpoint_iterator++, lErrorCode );
186  }
187 
188  if ( lErrorCode )
189  {
190  throw boost::system::system_error ( lErrorCode );
191  }
192  }
193  catch ( const std::exception& aExc )
194  {
195  // log ( Error() , "Exception " , Quote ( aExc.what() ) , " caught at " , ThisLocation() );
196  return false;
197  }
198 
199  // Form the request. We specify the "Connection: close" header so that the server will close the socket after transmitting the response. This will allow us to treat all data up until the EOF as the content.
200  boost::asio::streambuf request;
201  std::ostream request_stream ( &request );
202  request_stream << "GET /" << lURLPair.second << " HTTP/1.0\r\n";
203  request_stream << "Host: " << lURLPair.first << "\r\n";
204  request_stream << "Accept: */*\r\n";
205  request_stream << "Connection: close\r\n\r\n";
206 
207  try
208  {
209  // Send the request...
210  boost::asio::write ( socket , request , lErrorCode );
211 
212  if ( lErrorCode )
213  {
214  throw boost::system::system_error ( lErrorCode );
215  }
216  }
217  catch ( const std::exception& aExc )
218  {
219  // log ( Error() , "Exception " , Quote ( aExc.what() ) , " caught at " , ThisLocation() );
220  return false;
221  }
222 
223  // ... and get the reply. First we need a buffer to write the reply in to...
224  static const int mDefaultBufferSize ( 65536 );
225  typedef std::vector<uint8_t> BufferType;
226  BufferType mBuffer ( mDefaultBufferSize , uint8_t ( 0 ) );
227  std::size_t lSize ( 0 );
228 
229  // Just keep reading and, if we have not reached the EOF, extend the buffer and read some more.
230  try
231  {
232  while ( true )
233  {
234  lSize += boost::asio::read ( socket, boost::asio::buffer ( &mBuffer[lSize] , mDefaultBufferSize ) , lErrorCode );
235 
236  if ( lErrorCode )
237  {
238  if ( lErrorCode == boost::asio::error::eof )
239  {
240  break;
241  }
242  else
243  {
244  throw boost::system::system_error ( lErrorCode );
245  }
246  }
247 
248  mBuffer.insert ( mBuffer.end() , mDefaultBufferSize , uint8_t ( 0 ) );
249  }
250  }
251  catch ( const std::exception& aExc )
252  {
253  // log ( Error() , "Exception " , Quote ( aExc.what() ) , " caught at " , ThisLocation() );
254  return false;
255  }
256 
257  mBuffer.resize ( lSize );
258  //Parse the recieved data into an HttpResponseType object
259  grammars::HttpResponseGrammar lGrammar2;
260  boost::spirit::qi::phrase_parse ( mBuffer.begin() , mBuffer.end() , lGrammar2 , boost::spirit::ascii::space , aResponse );
261 
262  if ( DebugInfo )
263  {
264  try
265  {
266  log ( Info() , "HTTP response parsed as:\n" , aResponse );
267  }
268  catch ( const std::exception& aExc )
269  {
270  log ( Error() , "EXCEPTION: " , aExc.what() , " caught in " , ThisLocation() , " Continuing." );
271  // Just debugging so although exception is worrying, it is not critical
272  }
273  }
274 
275  if ( aResponse.method != "HTTP" || aResponse.status != 200 )
276  {
277  return false;
278  }
279 
280  return true;
281  }
282  }
283 }
284 // ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
285 
286 
287 
288 // ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
289 namespace uhal
290 {
291  namespace utilities
292  {
299  template < typename R , typename F , typename L>
300  void OpenFileLocal ( const std::string& aFilenameExpr , const boost::filesystem::path& aParentPath , boost::_bi::bind_t<R,F,L> aBinder )
301  {
302  std::vector< boost::filesystem::path > lFilePaths;
303  uhal::utilities::ShellExpandFilenameExpr ( aFilenameExpr , aParentPath , lFilePaths );
304 
305  for ( std::vector< boost::filesystem::path >::iterator lIt2 = lFilePaths.begin() ; lIt2 != lFilePaths.end() ; ++ lIt2 )
306  {
307  std::ifstream lStr ( lIt2->c_str() );
308 
309  if ( !lStr.is_open() )
310  {
311  uhal::exception::CannotOpenFile lExc;
312  log ( lExc , "Failed to open " , lIt2->c_str() , ". Continuing with next document for now but be aware!" );
313  throw lExc ;
314  }
315  else
316  {
317  lStr.seekg ( 0, std::ios::end );
318  std::vector<uint8_t> lFile ( lStr.tellg() , 0 );
319  lStr.seekg ( 0, std::ios::beg );
320  lStr.read ( ( char* ) & ( lFile[0] ) , lFile.size() );
321  aBinder ( std::string ( "file" ) , *lIt2 , boost::ref ( lFile ) );
322  }
323 
324  lStr.close();
325  }
326  }
327 
334  template < typename R , typename F , typename L>
335  void OpenFileHttp ( const std::string& aURL , boost::_bi::bind_t<R,F,L> aBinder )
336  {
337  HttpResponseType lHttpResponse;
338 
339  if ( ! uhal::utilities::HttpGet<true> ( aURL , lHttpResponse ) )
340  {
341  uhal::exception::CannotOpenFile lExc;
342  log ( lExc , "Failed to download file " , aURL , ". Continuing for now but be aware!" );
343  throw lExc;
344  }
345 
346  boost::filesystem::path lFilePath = boost::filesystem::path ( aURL );
347  aBinder ( std::string ( "http" ) , lFilePath , boost::ref ( lHttpResponse.content ) );
348  }
349 
358  template < typename R , typename F , typename L>
359  void OpenFile ( const std::string& aProtocol , const std::string& aFilenameExpr , const boost::filesystem::path& aParentPath , boost::_bi::bind_t<R,F,L> aBinder )
360  {
361  if ( aProtocol == "file" )
362  {
363  uhal::utilities::OpenFileLocal ( aFilenameExpr , aParentPath , aBinder );
364  }
365  else if ( aProtocol == "http" )
366  {
367  uhal::utilities::OpenFileHttp ( aFilenameExpr , aBinder );
368  }
369  else
370  {
371  uhal::exception::NonSupportedUriProtocol lExc;
372  log ( lExc , "The protocol " , Quote ( aProtocol ) , " for file " , Quote ( aFilenameExpr ) , " is not supported." );
373  log ( lExc , "The supported protocols are " , Quote ( "file://" ) , " and " , Quote ( "http://" ) );
374  throw lExc;
375  }
376  }
377 
378 
379 
380  }
381 }
382 // ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
383 
384 
385 #endif
void ParseSemicolonDelimitedUriList(const std::string &aSemicolonDelimitedUriList, std::vector< std::pair< std::string, std::string > > &aUriList)
Parse a semicolon delimited list of URIs into a vector of protocol/address pairs. ...
Definition: files.cpp:46
int status
the response status
std::string method
the http transport method
#define UHAL_DEFINE_EXCEPTION_CLASS(ClassName, ClassDescription)
Definition: exception.hpp:64
void OpenFileLocal(const std::string &aFilenameExpr, const boost::filesystem::path &aParentPath, boost::_bi::bind_t< R, F, L > aBinder)
Given a linux shell expression, open all files which match it and call the callback function on each ...
Definition: files.hpp:300
void OpenFileHttp(const std::string &aURL, boost::_bi::bind_t< R, F, L > aBinder)
Given a URL, retrieve the file and call the callback function on each of them.
Definition: files.hpp:335
ErrorLevel Error
Definition: LogLevels.cpp:61
void ShellExpandFilenameExpr(const std::string &aFilenameExpr, const boost::filesystem::path &aParentPath, std::vector< boost::filesystem::path > &aFiles)
Perform shell expansion of a linux shell expression ( e.g.
Definition: files.cpp:69
_Quote< T > Quote(const T &aT)
Struct to store an http response received from a server when parsed by boost spirit.
#define ThisLocation()
void OpenFile(const std::string &aProtocol, const std::string &aFilenameExpr, const boost::filesystem::path &aParentPath, boost::_bi::bind_t< R, F, L > aBinder)
Given a protocol and either a URL or a linux shell expression, open the file and call the callback fu...
Definition: files.hpp:359
InfoLevel Info
Definition: LogLevels.cpp:114
bool HttpGet(const std::string &aURL, HttpResponseType &aResponse)
Retrieve a file by HTTP.
Definition: files.hpp:138
std::vector< uint8_t > content
parsed message content
c write(addr, xx[0])