General cleanup (#64)

Create pisa library
Declare dependency for gumbo
Fix #60
Fix #22

Author: amallia

CMakeLists.txt

@@ -6,8 +6,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
 configure_file(
-  ${DS2I_SOURCE_DIR}/include/ds2i_config.hpp.in
-  ${DS2I_SOURCE_DIR}/include/ds2i_config.hpp
+  ${DS2I_SOURCE_DIR}/include/pisa/ds2i_config.hpp.in
+  ${DS2I_SOURCE_DIR}/include/pisa/ds2i_config.hpp
   ESCAPE_QUOTES)
 
 if(NOT CMAKE_BUILD_TYPE)
@@ -31,6 +31,7 @@ set_target_properties(gumbo::gumbo PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
     ${CMAKE_CURRENT_SOURCE_DIR}/external/gumbo-parser/src)
 set_property(TARGET gumbo::gumbo APPEND PROPERTY IMPORTED_LOCATION
     ${CMAKE_BINARY_DIR}/gumbo-parser/lib/libgumbo.a)
+add_dependencies( gumbo::gumbo gumbo-external )
 
 # Add code coverage
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/external/CMake-codecov/cmake")
@@ -63,11 +64,29 @@ set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 link_libraries(Threads::Threads)
 
-# add the root directory to include path to make includes absolute
-include_directories(${DS2I_SOURCE_DIR}/external
-		    ${STXXL_INCLUDE_DIRS}
-		    ${DS2I_SOURCE_DIR}/include
-		    )
+
+include_directories(include)
+add_library(pisa INTERFACE)
+target_include_directories(pisa INTERFACE
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include/pisa>
+)
+target_link_libraries(pisa INTERFACE
+    Threads::Threads
+    Boost::boost
+    QMX
+    mio
+    ParallelSTL
+    GSL
+    FastPFor
+    streamvbyte
+    MaskedVByte
+    simdcomp
+    gumbo::gumbo
+    Boost::filesystem
+    Porter2
+    warcpp
+)
+target_include_directories(pisa INTERFACE external)
 
 add_subdirectory(src)
 

benchmarks/CMakeLists.txt

@@ -1,43 +1,19 @@
 add_executable(index_perftest index_perftest.cpp)
 target_link_libraries(index_perftest
-  Boost::boost
-  mio
-  FastPFor
-  streamvbyte
-  MaskedVByte
-  QMX
-  simdcomp
-  ParallelSTL
-  GSL
-  )
+  pisa
+)
 
 add_executable(perftest_interpolative perftest_interpolative.cpp)
 target_link_libraries(perftest_interpolative
-  Boost::boost
-  mio
-  FastPFor
-  QMX
-  simdcomp
-  GSL
-  )
+  pisa
+)
 
 add_executable(selective_queries selective_queries.cpp)
 target_link_libraries(selective_queries
-  Boost::boost
-  mio
-  FastPFor
-  streamvbyte
-  MaskedVByte
-  QMX
-  simdcomp
-  ParallelSTL
-  GSL
-  )
+  pisa
+)
 
 add_executable(scan_perftest scan_perftest.cpp)
 target_link_libraries(scan_perftest
-  Boost::boost
-  mio
-  ParallelSTL
-  GSL
-  )
+  pisa
+)

benchmarks/index_perftest.cpp

@@ -4,9 +4,9 @@
 #include "index_types.hpp"
 #include "util/util.hpp"
 
-using ds2i::logger;
-using ds2i::get_time_usecs;
-using ds2i::do_not_optimize_away;
+using pisa::logger;
+using pisa::get_time_usecs;
+using pisa::do_not_optimize_away;
 
 template <typename IndexType, bool with_freqs>
 void perftest(IndexType const& index, std::string const& type)
@@ -65,7 +65,7 @@ void perftest(IndexType const& index, std::string const& type)
             if (size < min_length) continue;
 
             skip_values.emplace_back(i, std::vector<uint64_t>());
-            for (size_t i = 0; i < std::min(ds2i::ceil_div(size, skip),
+            for (size_t i = 0; i < std::min(pisa::ceil_div(size, skip),
                                             max_calls_per_list); ++i) {
                 reader.move(i * skip);
                 skip_values.back().second.push_back(reader.docid());
@@ -106,7 +106,7 @@ void perftest(const char* index_filename, std::string const& type)
     logger() << "Loading index from " << index_filename << std::endl;
     IndexType index;
     mio::mmap_source m(index_filename);
-    ds2i::mapper::map(index, m, ds2i::mapper::map_flags::warmup);
+    pisa::mapper::map(index, m, pisa::mapper::map_flags::warmup);
 
     perftest<IndexType, false>(index, type);
     perftest<IndexType, true>(index, type);
@@ -115,7 +115,7 @@ void perftest(const char* index_filename, std::string const& type)
 
 int main(int argc, const char** argv) {
 
-    using namespace ds2i;
+    using namespace pisa;
 
     if (argc != 3) {
         std::cerr << "Usage: " << argv[0]

benchmarks/perftest_interpolative.cpp

@@ -6,7 +6,7 @@
 
 int main()
 {
-    using namespace ds2i;
+    using namespace pisa;
     static const size_t size = interpolative_block::block_size;
     static const size_t runs = 1 << 20;
 

benchmarks/scan_perftest.cpp

@@ -5,18 +5,18 @@
 #include "sequence/uniform_partitioned_sequence.hpp"
 #include "util/util.hpp"
 
-using ds2i::logger;
-using ds2i::get_time_usecs;
-using ds2i::do_not_optimize_away;
+using pisa::logger;
+using pisa::get_time_usecs;
+using pisa::do_not_optimize_away;
 
 template <typename BaseSequence>
 void perftest(const char* index_filename)
 {
-    typedef ds2i::sequence_collection<BaseSequence> collection_type;
+    typedef pisa::sequence_collection<BaseSequence> collection_type;
     logger() << "Loading collection from " << index_filename << std::endl;
     collection_type coll;
     mio::mmap_source m(index_filename);
-    ds2i::mapper::map(coll, m, ds2i::mapper::map_flags::warmup);
+    pisa::mapper::map(coll, m, pisa::mapper::map_flags::warmup);
 
     if (true) {
         logger() << "Scanning all the posting lists" << std::endl;
@@ -127,10 +127,10 @@ void perftest(const char* index_filename)
 }
 int main(int argc, const char** argv) {
 
-    using ds2i::compact_elias_fano;
-    using ds2i::indexed_sequence;
-    using ds2i::partitioned_sequence;
-    using ds2i::uniform_partitioned_sequence;
+    using pisa::compact_elias_fano;
+    using pisa::indexed_sequence;
+    using pisa::partitioned_sequence;
+    using pisa::uniform_partitioned_sequence;
 
     if (argc != 3) {
         std::cerr << "Usage: " << argv[0]

benchmarks/selective_queries.cpp

@@ -12,7 +12,7 @@ template <typename IndexType>
 void selective_queries(const char* index_filename,
                        std::string const& type)
 {
-    using namespace ds2i;
+    using namespace pisa;
 
 
     IndexType index;
@@ -59,7 +59,7 @@ void selective_queries(const char* index_filename,
 
 
 int main(int, const char** argv) {
-    using namespace ds2i;
+    using namespace pisa;
 
     std::string type = argv[1];
     const char* index_filename = argv[2];

include/binary_collection.hpp renamed to include/pisa/binary_collection.hpp

@@ -13,7 +13,7 @@
 #include <sys/mman.h>
 #endif
 
-namespace ds2i {
+namespace pisa {
 
     template <typename Source = mio::mmap_source>
     class base_binary_collection {

include/binary_freq_collection.hpp renamed to include/pisa/binary_freq_collection.hpp

@@ -6,7 +6,7 @@
 
 #include "binary_collection.hpp"
 
-namespace ds2i {
+namespace pisa {
 
     class binary_freq_collection {
     public:

include/bit_vector.hpp renamed to include/pisa/bit_vector.hpp

@@ -9,7 +9,7 @@
 
 #include "succinct/mappable_vector.hpp"
 
-namespace ds2i {
+namespace pisa {
 
 namespace detail {
 inline size_t words_for(uint64_t n) { return ceil_div(n, 64); }
@@ -476,4 +476,4 @@ class bit_vector {
     mapper::mappable_vector<uint64_t> m_bits;
 };
 
-} // namespace ds2i
+} // namespace pisa

include/bitvector_collection.hpp renamed to include/pisa/bitvector_collection.hpp

@@ -4,7 +4,7 @@
 
 #include "codec/compact_elias_fano.hpp"
 
-namespace ds2i {
+namespace pisa {
 
     class bitvector_collection {
     public: