Filter some nonsensical lexemes from the data

1 files changed, 6 insertions, 1 deletions

diff --git a/process-database.hs b/process-database.hs
index 8ce60fd..bfbd4b7 100755
--- a/process-database.hs
+++ b/process-database.hs
@@ -92,7 +92,12 @@ makeTrie prefixlen rows =
       longer = Map.fromListWith (++)
                  [(rReading row !! prefixlen, [row])
                  | row <- rows, length (rReading row) > prefixlen]
-  in Node [(target, rFreq r) | r <- here, target <- dedup [rLexeme r, rOrtho r]]
+  in Node [(target, rFreq r)
+          | r <- here
+          , target <- dedup [rLexeme r, rOrtho r]
+          , not (null target)
+          , target `notElem` ["＊", "0", "#N/A"]  -- 隘 has "＊" as "orthography"
+          , all (`notElem` ['a'..'z']) target]  -- "アイ鱻eye" is not a sensible lexeme
           (Map.assocs (makeTrie (prefixlen + 1) <$> longer))
 
 -- Serialised format of the trie: