mnemonics: sanity checks for word lists

and a test to go with it

Remember to run the test when changing word lists, or simplewallet
will throw uncaught if that word list is used.

src/mnemonics/language_base.h

@@ -38,6 +38,7 @@
 #include <vector>
 #include <unordered_map>
 #include <string>
+#include "misc_log_ex.h"
 
 /*!
  * \namespace Language
@@ -73,6 +74,10 @@ namespace Language
   class Base
   {
   protected:
+    enum {
+      ALLOW_SHORT_WORDS = 1<<0,
+      ALLOW_DUPLICATE_PREFIXES = 1<<1,
+    };
     const std::vector<std::string> word_list; /*!< A pointer to the array of words */
     std::unordered_map<std::string, uint32_t> word_map; /*!< hash table to find word's index */
     std::unordered_map<std::string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
@@ -81,21 +86,39 @@ namespace Language
     /*!
      * \brief Populates the word maps after the list is ready.
      */
-    void populate_maps()
+    void populate_maps(uint32_t flags = 0)
     {
       int ii;
       std::vector<std::string>::const_iterator it;
+      if (word_list.size () != 1626)
+        throw std::runtime_error("Wrong word list length for " + language_name);
       for (it = word_list.begin(), ii = 0; it != word_list.end(); it++, ii++)
       {
         word_map[*it] = ii;
+        if ((*it).size() < unique_prefix_length)
+        {
+          if (flags & ALLOW_SHORT_WORDS)
+            MWARNING(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
+          else
+            throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
+        }
+        std::string trimmed;
         if (it->length() > unique_prefix_length)
         {
-          trimmed_word_map[utf8prefix(*it, unique_prefix_length)] = ii;
+          trimmed = utf8prefix(*it, unique_prefix_length);
         }
         else
         {
-          trimmed_word_map[*it] = ii;
+          trimmed = *it;
         }
+        if (trimmed_word_map.find(trimmed) != trimmed_word_map.end())
+        {
+          if (flags & ALLOW_DUPLICATE_PREFIXES)
+            MWARNING("Duplicate prefix in " << language_name << " word list: " << trimmed);
+          else
+            throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + trimmed);
+        }
+        trimmed_word_map[trimmed] = ii;
       }
     }
   public:

src/mnemonics/old_english.h

@@ -1680,7 +1680,7 @@ namespace Language
         "weary"
       }), 4)
     {
-      populate_maps();
+      populate_maps(ALLOW_DUPLICATE_PREFIXES | ALLOW_SHORT_WORDS);
     }
   };
 }

src/mnemonics/spanish.h

@@ -1680,7 +1680,7 @@ namespace Language
         "rito"
       }), 4)
     {
-      populate_maps();
+      populate_maps(ALLOW_SHORT_WORDS);
     }
   };
 }

tests/unit_tests/mnemonics.cpp

@@ -38,6 +38,11 @@
 #include "mnemonics/spanish.h"
 #include "mnemonics/portuguese.h"
 #include "mnemonics/japanese.h"
+#include "mnemonics/german.h"
+#include "mnemonics/italian.h"
+#include "mnemonics/russian.h"
+#include "mnemonics/french.h"
+#include "mnemonics/dutch.h"
 #include "mnemonics/old_english.h"
 #include "mnemonics/language_base.h"
 #include "mnemonics/singleton.h"
@@ -133,6 +138,19 @@ namespace
   }
 }
 
+TEST(mnemonics, consistency)
+{
+  try {
+    std::vector<std::string> language_list;
+    crypto::ElectrumWords::get_language_list(language_list);
+  }
+  catch(const std::exception &e)
+  {
+    std::cout << "Error initializing mnemonics: " << e.what() << std::endl;
+    ASSERT_TRUE(false);
+  }
+}
+
 TEST(mnemonics, all_languages)
 {
   srand(time(NULL));
@@ -141,12 +159,23 @@ TEST(mnemonics, all_languages)
     Language::Singleton<Language::Spanish>::instance(),
     Language::Singleton<Language::Portuguese>::instance(),
     Language::Singleton<Language::Japanese>::instance(),
+    Language::Singleton<Language::German>::instance(),
+    Language::Singleton<Language::Italian>::instance(),
+    Language::Singleton<Language::Russian>::instance(),
+    Language::Singleton<Language::French>::instance(),
+    Language::Singleton<Language::Dutch>::instance(),
   });
 
   for (std::vector<Language::Base*>::iterator it = languages.begin(); it != languages.end(); it++)
   {
+    try {
       test_language(*(*it));
     }
+    catch (const std::exception &e) {
+      std::cout << "Error testing " << (*it)->get_language_name() << " language: " << e.what() << std::endl;
+      ASSERT_TRUE(false);
+    }
+  }
 }
 
 TEST(mnemonics, language_detection_with_bad_checksum)