Debug and optimise C znc parser

1 files changed, 19 insertions, 20 deletions

diff --git a/src/Index.hs b/src/Index.hs
index 221be78..d9a1379 100644
--- a/src/Index.hs
+++ b/src/Index.hs
@@ -26,7 +26,6 @@ import Control.Monad (forM, forM_, when, guard)
 import Control.Monad.Trans.Class (lift)
 import Control.Monad.Trans.Maybe
 import Data.ByteString qualified as BS
-import Data.ByteString (ByteString)
 import Data.Char (isDigit, chr, ord)
 import Data.Functor ((<&>))
 import Data.IORef
@@ -43,7 +42,6 @@ import Data.Vector.Generic qualified as VG
 import Data.Vector.Generic.Mutable qualified as VGM
 import Data.Vector.Unboxed qualified as VU
 import Data.Vector.Unboxed.Base qualified as VU (Vector(V_2))
-import Data.Vector.Storable qualified as VS
 import Data.Word
 import System.Clock qualified as Clock
 import System.Directory
@@ -59,7 +57,7 @@ import Config (Channel(..), prettyChannel)
 import ImmutGrowVector qualified as IGV
 import Mmap
 import Util
-import ZNC
+import ZNC.Parser
 
 
 -- This module keeps an index both for the full list of events, as well as a
@@ -136,7 +134,7 @@ ciEndDay ci =
 --    simple data structure is fine.
 data Index = Index !FilePath
                    !(Map Channel (IORef ChanIndex))
-                   !(Cache (Channel, YMD) (ByteString, VS.Vector Word32))
+                   !(Cache (Channel, YMD) RawEvents)
 
 type EventID = Text
 
@@ -287,8 +285,9 @@ indexUpdateImport index@(Index _ mp _) chan = do
         dayidx = fromIntegral @Integer @Int (day `diffDays` ciStartDay ci)
 
     loadDay index chan ymd >>= \case
-      Just (bs, lineStarts) ->
-        return (Just (dayidx, Counts (VS.length lineStarts) (countCompressed (map snd (parseLog bs)))))
+      Just raw ->
+        return (Just (dayidx, Counts (rawNumEvents raw)
+                                     (countCompressed (map snd (realiseEvents raw)))))
       Nothing -> return Nothing
 
   atomicPrint $ "Update import for " <> prettyChannel chan <> ": " <> T.show readCounts <> " (len = " <> T.show (IGV.length (ciCountUntil ci)) <> ")"
@@ -375,17 +374,17 @@ indexGetEventsLinear index@(Index _ mp _) chan kind from count = do
                          | otherwise = scan IGV.! dayidx - scan IGV.! (dayidx - 1)
             rangeStart = if day == day1 then off1 else 0
             rangeEnd = if day == day2 then off2 else neventsOnDay
-            range = (rangeStart, Just (rangeEnd - rangeStart))
+            range = (rangeStart, rangeEnd)
             ymd = ymdFromGregorian (toGregorian day)
         in if neventsOnDay > 0
              then loadDay index chan ymd <&> \case
-                    Just (bs, lineStarts) -> case kind of
+                    Just raw -> case kind of
                       CKAll ->
-                        let events = parseLogRange range lineStarts bs
+                        let events = realiseEventsRange raw range
                         in [(YMDHMS ymd hms, genEventID (YMDHMS ymd hms) off, ev)
                            | ((hms, ev), off) <- zip events [rangeStart ..]]
                       CKCompressed ->
-                        let events = parseLog bs
+                        let events = realiseEvents raw
                             events' = take (rangeEnd - rangeStart) $ drop rangeStart $
                                         compressEvents [((hms, off), ev) | ((hms, ev), off) <- zip events [0..]]
                         in [(YMDHMS ymd hms, genEventID (YMDHMS ymd hms) off, ev) | ((hms, off), ev) <- events']
@@ -429,16 +428,16 @@ findEventIDLinear index@(Index _ mp _) chan kind eid = runMaybeT $ do
   ci <- lift $ readIORef (mp Map.! chan)
   guard (ciStartDay ci <= day && day <= ciEndDay ci)
 
-  (bs, lineStarts) <- MaybeT $ loadDay index chan ymd
+  raw <- MaybeT $ loadDay index chan ymd
   let candidates =  -- [(event offset, index in possibly compressed event list)]
         map snd $
         takeWhile ((== hms) . fst) $
         dropWhile ((< hms) . fst) $
         case kind of
-          CKAll -> zip (parseLogTimesOnly lineStarts bs) (zip [0..] [0..])
+          CKAll -> zip (map fst (realiseEvents raw)) (zip [0..] [0..])
           CKCompressed ->
             let compressed = compressEvents [((hms', off), ev)
-                                            | ((hms', ev), off) <- zip (parseLog bs) [0..]]
+                                            | ((hms', ev), off) <- zip (realiseEvents raw) [0..]]
             in [(hms', (off, idx)) | (((hms', off), _ev), idx) <- zip compressed [0..]]
   case candidates of
     [] -> empty
@@ -473,8 +472,8 @@ indexGetEventsDay index@(Index _ mp _) chan kind day = do
   if day < ciStartDay ci || day > ciEndDay ci
     then return (firstlast, [])
     else loadDay index chan ymd <&> \case
-           Just (bs, _lineStarts) ->
-             let events = parseLog bs
+           Just raw ->
+             let events = realiseEvents raw
                  events' = case kind of
                    CKAll ->
                      [(hms, genEventID (YMDHMS ymd hms) off, ev)
@@ -488,17 +487,17 @@ indexGetEventsDay index@(Index _ mp _) chan kind day = do
 
 -- utilities
 
-loadDay :: Index -> Channel -> YMD -> IO (Maybe (ByteString, VS.Vector Word32))
+loadDay :: Index -> Channel -> YMD -> IO (Maybe RawEvents)
 loadDay (Index basedir _ cache) chan@(Channel network channel) ymd = do
   cacheLookup cache (chan, ymd) >>= \case
     Nothing -> do
       mapFile (basedir </> T.unpack network </> T.unpack channel </> toFileName ymd) >>= \case
         Just bs -> do
-          let lineStarts = preparseLog bs
-          cacheAdd cache (chan, ymd) (bs, lineStarts)
-          return (Just (bs, lineStarts))
+          let raw = parseLogRaw bs
+          cacheAdd cache (chan, ymd) raw
+          return (Just raw)
         Nothing -> return Nothing  -- file didn't exist
-    Just (bs, lineStarts) -> return (Just (bs, lineStarts))
+    Just raw -> return (Just raw)
 
 isImportant :: Event -> Bool
 isImportant ReNick{} = True