cwchessboard: tstpgnread.cc Source File

00001 // This program tests what is the best way to read a file
00002 // with as goal to get to the end fast (maximum disk speed).
00003 // That means that we want to read an integer number of
00004 // blocks (likely 4096 bytes) at a time.
00005 
00006 #include "sys.h"
00007 #include <iostream>
00008 #include <fstream>
00009 #include <cstdio>
00010 #include <cstdlib>
00011 #include <stdint.h>
00012 #include <sys/time.h>
00013 #include <sys/types.h>
00014 #include <sys/stat.h>
00015 #include <fcntl.h>
00016 #include <unistd.h>
00017 #include <gio/gio.h>
00018 #include "debug.h"
00019 
00020 char const* filename="/home/carlo/projects/cwchessboard/chessgames/ftp.cis.uab.edu/crafty/enormous.pgn";
00021 char const* warmupfile="/home/carlo/chess/fics2008.pgn";
00022 
00023 //#define REFERENCE_IMPLEMENTATION
00024 //#define BUFFERSIZE_IMPLEMENTATION
00025 //#define FSTREAM_READ_IMPLEMENTATION
00026 //#define FSTREAM_GETLINE_IMPLEMENTATION
00027 //#define GFILE_IMPLEMENTATION
00028 //#define GFILE_ASYNC_IMPLEMENTATION
00029 #define CWCHESS_PGN_IMPLEMENTATION
00030 
00031 #ifdef GFILE_ASYNC_IMPLEMENTATION
00032 #include <gio/gio.h>
00033 #include "PgnDatabase.h"                // Needed for cwchess::pgn::DatabaseSeekable::S_buffer_size.
00034 #endif
00035 #ifdef CWCHESS_PGN_IMPLEMENTATION
00036 #include <giomm/init.h>
00037 #include "PgnDatabase.h"
00038 #endif
00039 #if defined(GFILE_ASYNC_IMPLEMENTATION) || defined(CWCHESS_PGN_IMPLEMENTATION)
00040 std::ostream* global_os;
00041 #endif
00042 
00043 struct timeval before, after;
00044 
00045 inline void start_timer(void)
00046 {
00047   gettimeofday(&before, NULL);
00048 }
00049 
00050 uint64_t stop_timer(void)
00051 {
00052   gettimeofday(&after, NULL);
00053   timersub(&after,& before,& after);
00054   uint64_t t = after.tv_sec;
00055   t* = 1000000;
00056   t += after.tv_usec;
00057   return t;
00058 }
00059 
00060 char buf[4096 * 50];
00061 
00062 #ifdef REFERENCE_IMPLEMENTATION
00063 void benchmark_reference(std::ostream& os, char const* filename)
00064 {
00065   int fd = open(filename, O_RDONLY);  
00066   size_t len = 0;
00067   start_timer();
00068   while(len < 945950820)
00069   {
00070     int s = read(fd, buf, sizeof(buf));
00071     if (s == -1)
00072     {
00073       perror("read");
00074       exit(0);
00075     }
00076     else if (s == 0)
00077       break;
00078     len += s;
00079   }
00080   uint64_t microseconds = stop_timer();
00081   os << "Reference implementation (buffersize " << sizeof(buf) << "): " << microseconds << " microseconds." << std::endl;
00082   close(fd);
00083 }
00084 #endif
00085 
00086 #ifdef BUFFERSIZE_IMPLEMENTATION
00087 void benchmark_buffersize(std::ostream& os, char const* filename)
00088 {
00089   size_t const bufsize = 4096;
00090   int fd = open(filename, O_RDONLY);  
00091   size_t len = 0;
00092   start_timer();
00093   while(len < 945950820)
00094   {
00095     int s = read(fd, buf, bufsize);
00096     if (s == -1)
00097     {
00098       perror("read");
00099       exit(0);
00100     }
00101     else if (s == 0)
00102       break;
00103     len += s;
00104   }
00105   uint64_t microseconds = stop_timer();
00106   os << "read(2) (buffersize " << bufsize << "): " << microseconds << " microseconds." << std::endl;
00107   close(fd);
00108 }
00109 #endif
00110 
00111 #ifdef FSTREAM_READ_IMPLEMENTATION
00112 void benchmark_fstream_read(std::ostream& os, char const* filename)
00113 {
00114   std::ifstream inputfile;
00115   inputfile.open(filename);
00116   start_timer();
00117   size_t len = 0;
00118   while(inputfile)
00119   {
00120     inputfile.read(buf, 4096);
00121     len += inputfile.gcount();
00122   }
00123   uint64_t microseconds = stop_timer();
00124   os << "std::ifstream::read (buffersize 4096): " << microseconds << " microseconds. Size read: " << len << std::endl;
00125   inputfile.close();
00126 }
00127 #endif
00128 
00129 #ifdef FSTREAM_GETLINE_IMPLEMENTATION
00130 void benchmark_fstream_getline(std::ostream& os, char const* filename)
00131 {
00132   std::ifstream inputfile;
00133   inputfile.open(filename);
00134   start_timer();
00135   size_t len = 0;
00136   std::string line;
00137   while(getline(inputfile, line))
00138   {
00139     len += line.length() + 1;
00140   }
00141   uint64_t microseconds = stop_timer();
00142   os << "std::getline(std::ifstream, ...): " << microseconds << " microseconds. Size read: " << len << std::endl;
00143   inputfile.close();
00144 }
00145 #endif
00146 
00147 #ifdef GFILE_IMPLEMENTATION
00148 void benchmark_gfile(std::ostream& os, char const* filename)
00149 {
00150   g_type_init();
00151   GFile* file = g_file_new_for_path(filename);
00152   GCancellable* cancellable = g_cancellable_new();
00153   GError* error = NULL;
00154   GFileInputStream* stream = g_file_read(file, cancellable,& error);
00155   if (stream == NULL)
00156   {
00157     std::cerr << "g_file_read failed: " << error->message << std::endl;
00158     exit(1);
00159   }
00160   start_timer();
00161   size_t len = 0;
00162   gssize res;
00163   do
00164   {
00165     res = g_input_stream_read(G_INPUT_STREAM(stream), buf, 4096, cancellable,& error);
00166     if (res == -1)
00167     {
00168       std::cerr << "g_input_stream_read: " << error->message << std::endl;
00169       exit(1);
00170     }
00171     len += res;
00172   }
00173   while(res > 0);
00174   uint64_t microseconds = stop_timer();
00175   os << "g_input_stream_read (buffersize 4096): " << microseconds << " microseconds. Size read: " << len << std::endl;
00176   g_object_unref(stream);
00177   g_object_unref(cancellable);
00178   g_object_unref(file);
00179 }
00180 #endif
00181 
00182 #ifdef GFILE_ASYNC_IMPLEMENTATION
00183 GMainLoop* gmain_loop;
00184 GCancellable* cancellable;
00185 void async_ready_callback(GObject* source_object, GAsyncResult* async_res, gpointer user_data);
00186 void async_ready_callback2(GObject* source_object, GAsyncResult* async_res, gpointer user_data);
00187 
00188 void benchmark_gfile_async(std::ostream& os, char const* filename)
00189 {
00190   g_type_init();
00191   global_os =& os;
00192   GFile* file = g_file_new_for_path(filename);
00193   cancellable = g_cancellable_new();
00194   g_file_read_async(file, G_PRIORITY_DEFAULT, cancellable, async_ready_callback, file);
00195   gmain_loop = g_main_loop_new(NULL, false);
00196   g_main_loop_run(gmain_loop);
00197   g_object_unref(cancellable);
00198   g_object_unref(file);
00199 }
00200 #endif
00201 
00202 #ifdef CWCHESS_PGN_IMPLEMENTATION
00203 Glib::RefPtr<Glib::MainLoop> main_loop;
00204 Glib::RefPtr<cwchess::pgn::Database> pgn_data_base;
00205 void open_finished(size_t len);
00206 
00207 void benchmark_cwchess_pgn(std::ostream& os, char const* filename)
00208 {
00209   using namespace cwchess;
00210   Gio::init();
00211   global_os =& os;
00212   pgn_data_base = pgn::DatabaseSeekable::open(filename, sigc::ptr_fun(&open_finished));
00213   main_loop = Glib::MainLoop::create(false);
00214   start_timer();
00215   main_loop->run();
00216 }
00217 #endif
00218 
00219 void clear_disk_cache(void)
00220 {
00221   // Free pagecache.
00222   system("sudo sh -c \"sync; echo 1 > /proc/sys/vm/drop_caches\"");
00223   // Warm up (read libraries etc).
00224   start_timer();
00225   uint64_t microseconds = stop_timer();
00226   std::ofstream dump("/dev/null");
00227   //std::ostream& dump(std::cout);
00228 #ifdef REFERENCE_IMPLEMENTATION
00229   benchmark_reference(dump, warmupfile);
00230 #endif
00231 #ifdef BUFFERSIZE_IMPLEMENTATION
00232   benchmark_buffersize(dump, warmupfile);
00233 #endif
00234 #ifdef FSTREAM_READ_IMPLEMENTATION
00235   benchmark_fstream_read(dump, warmupfile);
00236 #endif
00237 #ifdef FSTREAM_GETLINE_IMPLEMENTATION
00238   benchmark_fstream_getline(dump, warmupfile);
00239 #endif
00240 #ifdef GFILE_IMPLEMENTATION
00241   benchmark_gfile(dump, warmupfile);
00242 #endif
00243 #ifdef GFILE_ASYNC_IMPLEMENTATION
00244   benchmark_gfile_async(dump, warmupfile);
00245 #endif
00246 #ifdef CWCHESS_PGN_IMPLEMENTATION
00247   benchmark_cwchess_pgn(dump, warmupfile);
00248 #endif
00249   dump.close();
00250   // Sleep to let other running application catch up too.
00251   sleep(1);
00252 }
00253 
00254 int main()
00255 {
00256   if (!Glib::thread_supported())
00257       Glib::thread_init();
00258   Debug(debug::init());
00259 
00260 #ifdef REFERENCE_IMPLEMENTATION
00261   clear_disk_cache();
00262   benchmark_reference(std::cout, filename);
00263 #endif
00264 
00265 #ifdef BUFFERSIZE_IMPLEMENTATION
00266   clear_disk_cache();
00267   benchmark_buffersize(std::cout, filename);
00268 #endif
00269 
00270 #ifdef FSTREAM_READ_IMPLEMENTATION
00271   clear_disk_cache();
00272   benchmark_fstream_read(std::cout, filename);
00273 #endif
00274 
00275 #ifdef FSTREAM_GETLINE_IMPLEMENTATION
00276   clear_disk_cache();
00277   benchmark_fstream_getline(std::cout, filename);
00278 #endif
00279 
00280 #ifdef GFILE_IMPLEMENTATION
00281   clear_disk_cache();
00282   benchmark_gfile(std::cout, filename);
00283 #endif
00284 
00285 #ifdef GFILE_ASYNC_IMPLEMENTATION
00286   clear_disk_cache();
00287   benchmark_gfile_async(std::cout, filename);
00288 #endif
00289 
00290 #ifdef CWCHESS_PGN_IMPLEMENTATION
00291   clear_disk_cache();
00292   benchmark_cwchess_pgn(std::cout, filename);
00293 #endif
00294 }
00295 
00296 #ifdef GFILE_ASYNC_IMPLEMENTATION
00297 size_t len = 0;
00298 
00299 void async_ready_callback(GObject* source_object, GAsyncResult* async_res, gpointer user_data)
00300 {
00301   GFile* file = G_FILE(user_data);
00302   GError* error = NULL;
00303   GFileInputStream* stream = g_file_read_finish(file, async_res,& error);
00304   if (stream == NULL)
00305   {
00306     std::cerr << "g_file_read_finish failed: " << error->message << std::endl;
00307     exit(1);
00308   }
00309   start_timer();
00310   len = 0;
00311   g_input_stream_read_async(G_INPUT_STREAM(stream), buf, cwchess::pgn::DatabaseSeekable::S_buffer_size,
00312       G_PRIORITY_DEFAULT, cancellable, async_ready_callback2, stream);
00313 }
00314 
00315 void async_ready_callback2(GObject* source_object, GAsyncResult* async_res, gpointer user_data)
00316 {
00317   GInputStream* stream = G_INPUT_STREAM(user_data);
00318   GError* error = NULL;
00319   gssize res = g_input_stream_read_finish(stream, async_res,& error);
00320   if (res == -1)
00321   {
00322     std::cerr << "g_input_stream_read_finish: " << error->message << std::endl;
00323     exit(1);
00324   }
00325   len += res;
00326   if (res > 0)
00327     g_input_stream_read_async(G_INPUT_STREAM(stream), buf, cwchess::pgn::DatabaseSeekable::S_buffer_size,
00328         G_PRIORITY_DEFAULT, cancellable, async_ready_callback2, stream);
00329   else
00330   {
00331     uint64_t microseconds = stop_timer();
00332    * global_os << "g_input_stream_read_async (buffersize " << cwchess::pgn::DatabaseSeekable::S_buffer_size << "): " <<
00333         microseconds << " microseconds. Size read: " << len << std::endl;
00334     g_object_unref(stream);
00335     g_main_loop_quit(gmain_loop);
00336   }
00337 }
00338 #endif
00339 
00340 #ifdef CWCHESS_PGN_IMPLEMENTATION
00341 void open_finished(size_t len)
00342 {
00343   uint64_t microseconds = stop_timer();
00344  * global_os << "cwchess::pgn::DatabaseSeekable (buffersize " << cwchess::pgn::DatabaseSeekable::S_buffer_size << "): " <<
00345       microseconds << " microseconds. Size read: " << len << "; number of lines: " << pgn_data_base->number_of_lines() << "; number of characters: " <<
00346       pgn_data_base->number_of_characters() << std::endl;
00347   main_loop->quit();
00348 }
00349 #endif