minor esthetic fixes; fixes to README

Author: jermp

README.md

@@ -1,9 +1,9 @@
-`tongrams` - Tons of *N*-Grams
-----------
+Tongrams - Tons of *N*-Grams
+==============================
 
 **NEWS: check the language model estimation library [here](https://github.com/jermp/tongrams_estimation)!**
 
-`tongrams` is a C++ library to index and query large language models
+Tongrams is a C++ library to index and query large language models
 in compressed space. It is the result of some research papers [1,2] by Giulio Ermanno Pibiri
 and Rossano Venturini.
 
@@ -99,53 +99,57 @@ their usage.
 
 We now show some examples.
 
-##### Example 1.
+##### Example 1
 The command
 
     ./build_trie ef_trie 5 count --dir ../test_data --out ef_trie.count.bin
 
 builds an Elias-Fano trie
+
 * of order 5;
 * that stores frequency counts;
 * from the *N*-gram counts files contained in the directory `test_data`;
 * with no context-based remapping (default);
 * whose counts ranks are encoded with the indexed codewords (IC) technique (default);
 * that is serialized to the binary file `ef_trie.count.bin`.
 
-##### Example 2.
+##### Example 2
 The command
 
     ./build_trie pef_trie 5 count --dir ../test_data --remapping 1 --ranks PSEF  --out pef_trie.count.out
 
 builds a partitioned Elias-Fano trie
+
 * of order 5;
 * that stores frequency counts;
 * from the *N*-gram counts files contained in the directory `test_data`;
 * with context-based remapping of order 1;
 * whose counts ranks are encoded with prefix sums (PS) + Elias-Fano (EF);
 * that is serialized to the binary file `pef_trie.count.out`.
 
-##### Example 3.
+##### Example 3
 The command
 
     ./build_trie ef_trie 5 prob_backoff --remapping 2 --u -20.0 --p 8 --b 8 --arpa ../test_data/arpa --out ef_trie.prob_backoff.bin
 
 builds an Elias-Fano trie
+
 * of order 5;
 * that stores probabilities and backoffs;
 * with context-based remapping of order 2;
 * with `<unk>` probability of -20.0 and using 8 bits for quantizing probabilities (`--p`) and backoffs (`--b`);
 * from the arpa file named `arpa`;
 * that is serialized to the binary file `ef_trie.prob_backoff.bin`.
 
-##### Example 4.
+##### Example 4
 The command
 
-    ./build_hash 5 8 4 count --dir ../test_data --out hash.bin
+    ./build_hash 5 8 count --dir ../test_data --out hash.bin
 
 builds a MPH-based model
+
 * of order 5;
-* that uses 8 bytes per hash key and 4 bytes per unique count;
+* that uses 8 bytes per hash key;
 * that stores frequency counts;
 * from the *N*-gram counts files contained in the directory `test_data`;
 * that is serialized to the binary file `hash.bin`.
@@ -155,15 +159,15 @@ Tests
 The `test` directory contains the unit tests of some of the fundamental building blocks used by the implemented data structures. As usual, running the executables without any arguments will show the list of their expected input parameters.
 Examples:
 
-    ./compact_vector_test 10000 13
-    ./fast_ef_sequence_test 1000000 128
+    ./test_compact_vector 10000 13
+    ./test_fast_ef_sequence 1000000 128
 
 The directory also contains the unit test for the data structures storing frequency counts, named `check_count_model`, which validates the implementation by checking that each count stored in the data structure is the same as the one provided in the input files from which the data structure was previously built.
 Example:
 
-    ./check_count_model count_data_structure.bin ../test_data
+    ./test_count_model ef_trie.count.bin ../test_data
 
-where `count_data_structure.bin` is the name of the data structure binary file and `test_data` is the name of the folder containing the input *N*-gram counts files.
+where `ef_trie.count.bin` is the name of the data structure binary file (maybe built with the command shown in Example 1) and `test_data` is the name of the folder containing the input *N*-gram counts files.
 
 Benchmarks
 ----------

external/cmd_line_parser

@@ -20,16 +20,16 @@ struct mph_count_lm {
 
         typename Values::builder counts_builder(m_order);
 
-        for (uint8_t order = 1; order <= m_order; ++order) {
+        for (uint8_t ord = 1; ord <= m_order; ++ord) {
             std::string filename;
-            util::input_filename(input_dir, order, filename);
+            util::input_filename(input_dir, ord, filename);
             util::check_filename(filename);
             grams_gzparser gp(filename.c_str());
 
             std::vector<uint64_t> counts;
             counts.reserve(gp.num_lines());
 
-            essentials::logger("Reading " + std::to_string(order) +
+            essentials::logger("Reading " + std::to_string(ord) +
                                "-grams counts");
             for (auto const& l : gp) {
                 counts_builder.eat_value(l.count);
@@ -39,23 +39,23 @@ struct mph_count_lm {
         }
 
         size_t available_ram = sysconf(_SC_PAGESIZE) * sysconf(_SC_PHYS_PAGES);
-        for (uint8_t order = 1; order <= m_order; ++order) {
-            essentials::logger("Building " + std::to_string(order) + "-grams");
+        for (uint8_t ord = 1; ord <= m_order; ++ord) {
+            essentials::logger("Building " + std::to_string(ord) + "-grams");
             grams_counts_pool pool(available_ram * 0.8);
             std::string filename;
-            util::input_filename(input_dir, order, filename);
+            util::input_filename(input_dir, ord, filename);
             pool.load_from<grams_gzparser>(filename.c_str());
 
             auto& pool_index = pool.index();
             uint64_t n = pool_index.size();
             compact_vector::builder counts_ranks_cvb(
-                n, util::ceil_log2(counts_builder.size(order - 1) + 1));
+                n, util::ceil_log2(counts_builder.size(ord - 1) + 1));
 
             std::vector<byte_range> byte_ranges;
             byte_ranges.reserve(n);
             for (auto const& record : pool_index) {
                 byte_ranges.push_back(record.gram);
-                uint64_t rank = counts_builder.rank(order - 1, record.count);
+                uint64_t rank = counts_builder.rank(ord - 1, record.count);
                 counts_ranks_cvb.push_back(rank);
             }
 

include/mph_count_lm.hpp

@@ -37,21 +37,21 @@ struct mph_prob_lm {
                     "specified order exceeds arpa file order");
             }
 
-            for (uint8_t order = 1; order <= m_order; ++order) {
+            for (uint8_t ord = 1; ord <= m_order; ++ord) {
                 std::vector<float> probs;
                 std::vector<float> backoffs;
-                uint64_t n = counts[order - 1];
+                uint64_t n = counts[ord - 1];
                 probs.reserve(n);
                 backoffs.reserve(n);
 
-                ap.read_values(order, n, probs, backoffs);
+                ap.read_values(ord, n, probs, backoffs);
                 assert(probs.size() == n);
 
-                if (order !=
+                if (ord !=
                     1) {  // need to scan unigrams anyway to set arpa offsets
                     probs_builder.build_probs_sequence(probs,
                                                        probs_quantization_bits);
-                    if (order != m_order) {
+                    if (ord != m_order) {
                         backoffs_builder.build_backoffs_sequence(
                             backoffs, backoffs_quantization_bits);
                     }
@@ -66,11 +66,10 @@ struct mph_prob_lm {
 
             size_t available_ram =
                 sysconf(_SC_PAGESIZE) * sysconf(_SC_PHYS_PAGES);
-            for (uint8_t order = 2; order <= m_order; ++order) {
-                essentials::logger("Building " + std::to_string(order) +
+            for (uint8_t ord = 2; ord <= m_order; ++ord) {
+                essentials::logger("Building " + std::to_string(ord) +
                                    "-grams");
-                arpa_iterator it(m_arpa_filename, order,
-                                 arpa_offsets[order - 1]);
+                arpa_iterator it(m_arpa_filename, ord, arpa_offsets[ord - 1]);
                 uint64_t n = it.num_grams();
                 grams_probs_pool pool(n, available_ram * 0.8);
 
@@ -85,18 +84,18 @@ struct mph_prob_lm {
 
                 compact_vector::builder cvb(
                     n, probs_quantization_bits +
-                           (order != m_order ? backoffs_quantization_bits : 0));
+                           (ord != m_order ? backoffs_quantization_bits : 0));
 
                 for (auto const& record : pool_index) {
                     bytes.push_back(record.gram);
                     float prob = record.prob;
                     float backoff = record.backoff;
                     // store interleaved ranks
-                    uint64_t prob_rank = probs_builder.rank(order - 2, prob, 0);
+                    uint64_t prob_rank = probs_builder.rank(ord - 2, prob, 0);
                     uint64_t packed = prob_rank;
-                    if (order != m_order) {
+                    if (ord != m_order) {
                         uint64_t backoff_rank =
-                            backoffs_builder.rank(order - 2, backoff, 1);
+                            backoffs_builder.rank(ord - 2, backoff, 1);
                         packed |= backoff_rank << probs_quantization_bits;
                     }
                     cvb.push_back(packed);

include/mph_prob_lm.hpp

@@ -407,12 +407,12 @@ struct fast_ef_sequence {
             uint64_t nodes = uint64_t(1) << h;
             for (uint64_t i = 0; i < nodes; ++i) {
                 auto r = ranges.pop_front();
-                uint64_t lo = r.first;
-                uint64_t hi = r.second;
-                size_t mid = (lo + hi) >> 1;
+                uint64_t l = r.first;
+                uint64_t h = r.second;
+                size_t mid = (l + h) >> 1;
                 samplings.push_back(it[mid]);
-                range_t left(lo, mid);
-                range_t right(mid + 1, hi);
+                range_t left(l, mid);
+                range_t right(mid + 1, h);
                 ranges.push_back(left);
                 ranges.push_back(right);
             }

include/sequences/fast_ef_sequence.hpp

@@ -57,7 +57,8 @@ int main(int argc, char** argv) {
         ap.read_line();
 
         // skip to specified order
-        for (uint8_t i = 0; i < order - 1; ++i) {
+        assert(order > 0);
+        for (uint32_t i = 0; i != order - 1; ++i) {
             while (ap.read_line())
                 ;
             ap.read_line();