summaryrefslogtreecommitdiff
path: root/mustache-th/Mustache
diff options
context:
space:
mode:
Diffstat (limited to 'mustache-th/Mustache')
-rw-r--r--mustache-th/Mustache/TH.hs226
-rw-r--r--mustache-th/Mustache/TH/Parser.hs352
2 files changed, 578 insertions, 0 deletions
diff --git a/mustache-th/Mustache/TH.hs b/mustache-th/Mustache/TH.hs
new file mode 100644
index 0000000..929d40a
--- /dev/null
+++ b/mustache-th/Mustache/TH.hs
@@ -0,0 +1,226 @@
+{-|
+Copyright: (c) Tom Smeding 2026
+License: AGPL-3.0-or-later
+-}
+{-# LANGUAGE LinearTypes #-}
+{-# LANGUAGE TemplateHaskellQuotes #-}
+module Mustache.TH (makeRender, Template) where
+
+import Data.ByteString (ByteString)
+import Data.ByteString qualified as BS
+import Data.Char (toUpper)
+import Data.Foldable (toList)
+import Data.List.NonEmpty (NonEmpty(..))
+import Data.List.NonEmpty qualified as NE
+import Data.Map.Strict (Map)
+import Data.Map.Strict qualified as Map
+import Data.Text (Text)
+import Data.Text qualified as T
+import Data.Text.Encoding qualified as TE
+import Data.Text.Builder.Linear.Buffer
+import Data.Vector (Vector)
+import Language.Haskell.TH
+
+import EscapeXML
+
+import Mustache.TH.Parser
+
+
+data DataMap
+ = VQInt
+ | VQBool
+ | VQText
+ | VQUnit -- ()
+ | VQList DataMap -- ^ embedded map is of a list element
+ | VQMaybe DataMap
+ | VQObject [(Text, (Exp -> Exp, DataMap))] -- ^ field name, projection function, submap
+
+
+makeDataMap :: Name -> Q DataMap
+makeDataMap name = makeDataMapTy (ConT name)
+
+makeDataMapTy :: Type -> Q DataMap
+makeDataMapTy (ConT tyname)
+ | tyname == ''Int = return VQInt
+ | tyname == ''Bool = return VQBool
+ | tyname == ''Text = return VQText
+makeDataMapTy (TupleT 0) = return VQUnit
+makeDataMapTy (AppT ListT ty) = VQList <$> makeDataMapTy ty
+makeDataMapTy (AppT (ConT name) ty) | name == ''Maybe = VQMaybe <$> makeDataMapTy ty
+
+makeDataMapTy topty = do
+ case splitApps topty [] of
+ (ConT tyname, tyargs) ->
+ reify tyname >>= \case
+ TyConI (DataD [] _ tyvars _ [RecC conname fields] _) ->
+ let n = length fields
+ in VQObject <$> sequence
+ [do dm <- makeDataMapTy (subst (Map.fromList (zip (map tyvarName tyvars) tyargs)) fieldty)
+ f <- projectField conname i n
+ return (T.pack (nameBase name), (f, dm))
+ | ((name, _bang, fieldty), i) <- zip fields [0..]]
+ _ -> fail $ "Not a suitable data type: " ++ show tyname
+ _ -> fail $ "Not a straightforward type: " ++ show topty
+ where
+ projectField :: Name -> Int -> Int -> Q (Exp -> Exp)
+ projectField conname i n = do
+ varname <- newName ('x' : show (i + 1))
+ let pats = replicate i WildP ++ [VarP varname] ++ replicate (n-i-1) WildP
+ return $ \e ->
+ CaseE e [Match (ConP conname [] pats) (NormalB (VarE varname)) []]
+
+ splitApps :: Type -> [Type] -> (Type, [Type])
+ splitApps (AppT t1 t2) args = splitApps t1 (t2 : args)
+ splitApps t args = (t, args)
+
+ tyvarName :: TyVarBndr flag -> Name
+ tyvarName (PlainTV n _) = n
+ tyvarName (KindedTV n _ _) = n
+
+ subst :: Map Name Type -> Type -> Type
+ subst _ t@ConT{} = t
+ subst _ t@ListT = t
+ subst _ t@TupleT{} = t
+ subst mp (VarT n) | Just t <- Map.lookup n mp = t
+ subst mp (AppT t1 t2) = AppT (subst mp t1) (subst mp t2)
+ subst _ t = error $ "Unsupported type: " ++ show t
+
+makeRender :: String -> Template -> Name -> Q [Dec]
+makeRender tplname (Template nodes) dataname = do
+ dm <- makeDataMap dataname
+ let funname = mkName ("renderPage" ++ toPascalCase tplname)
+ argname <- newName "dat"
+ bufname <- newName "buf"
+ body <- renderNodes ((dm, VarE argname) :| []) (DuplicableExp (VarE bufname)) (toList nodes)
+ let body' = VarE 'runBufferBS `AppE` LamE [VarP bufname] body
+ return
+ [SigD funname (ArrowT `AppT` ConT dataname `AppT` ConT ''ByteString)
+ ,FunD funname
+ [Clause [BangP (VarP argname)] (NormalB body') []]]
+
+data ExpensiveExp
+ = ExpensiveExp Exp
+ | DuplicableExp Exp
+
+expensive :: ExpensiveExp -> Exp
+expensive (ExpensiveExp e) = e
+expensive (DuplicableExp e) = e
+
+makeDuplicable :: ExpensiveExp -> (Exp -> Q Exp) -> Q Exp
+makeDuplicable (ExpensiveExp e) k = do
+ varname <- newName "a"
+ caseE (pure e)
+ [match (varP varname) (normalB (k (VarE varname))) []]
+makeDuplicable (DuplicableExp e) k = k e
+
+-- | This is just '(|>)', but with a @NOINLINE@ pragma. This is because GHC
+-- eagerly inlines '(|>)', and that function contains some conditionals, and as
+-- a result the generated Core for the render function contains a whole forest
+-- of join points. By avoiding inlining this function, the Core becomes a
+-- straight line of @case@s that contains join points only when there is
+-- genuinely control flow going on, i.e. when substituting a @Maybe@ or when
+-- looping over a list.
+{-# NOINLINE noinlineAppText #-}
+noinlineAppText :: Buffer %1-> Text -> Buffer
+noinlineAppText = (|>)
+
+-- | All input expressions must be duplicable.
+renderNodes :: NonEmpty (DataMap, Exp) -> ExpensiveExp -> [Chunk] -> Q (Exp {- Buffer -})
+renderNodes _ buf [] = return (expensive buf)
+renderNodes dats buf (node : nodes) = do
+ let app buf' name e = InfixE (Just buf') (VarE name) (Just e)
+
+ buf' <- case node of
+ Text t ->
+ let bytes = BS.unpack (TE.encodeUtf8 t)
+ in return $ app (expensive buf) '(|>#) (LitE (StringPrimL bytes))
+ Var esc named ->
+ case search named dats of
+ Just (VQInt, e) -> return $ app (expensive buf) '(|>$) e
+ Just (VQText, e) ->
+ return $ app (expensive buf) 'noinlineAppText (if esc then VarE 'escapeXML `AppE` e else e)
+ Just (_, _) -> fail $ "Strange value in variable reference " ++ show named
+ Nothing -> fail $ "Variable not found: " ++ show named
+ Section True named nodes' ->
+ case search named dats of
+ Just (VQBool, e) ->
+ makeDuplicable buf $ \buf' ->
+ condE (pure e)
+ (renderNodes ((VQBool, e) NE.<| dats) (DuplicableExp buf') (toList nodes'))
+ (pure buf')
+ Just (VQText, e) ->
+ makeDuplicable buf $ \buf' ->
+ condE [| T.null $(pure e) |]
+ (pure buf')
+ (renderNodes ((VQText, e) NE.<| dats) (DuplicableExp buf') (toList nodes'))
+ Just (VQList dm, e) -> do
+ bvar <- newName "b"
+ vvar <- newName "v"
+ varE 'foldlIntoBuffer
+ `appE` lamE [varP bvar, varP vvar]
+ (renderNodes ((dm, VarE vvar) NE.<| dats) (DuplicableExp (VarE bvar)) (toList nodes'))
+ `appE` pure (expensive buf)
+ `appE` pure e
+ Just (VQMaybe dm, e) -> do
+ xvar <- newName "_x" -- tends to be unused in {{#sec}} {{.}} {{/sec}} situations, so suppress the warning
+ makeDuplicable buf $ \buf' ->
+ caseE (pure e)
+ [match (conP 'Nothing []) (normalB (pure buf')) []
+ ,match (conP 'Just [varP xvar])
+ (normalB (renderNodes ((dm, VarE xvar) NE.<| dats) (DuplicableExp buf') (toList nodes')))
+ []]
+ Just pair@(VQObject _, _) -> do
+ renderNodes (pair NE.<| dats) buf (toList nodes')
+ Just (VQInt, _) -> fail $ "Int value in section scrutinee " ++ show named
+ Just (VQUnit, _) -> fail $ "() value in section scrutinee " ++ show named
+ Nothing -> fail $ "Section scrutinee not found: " ++ show named
+ Section False named nodes' ->
+ case search named dats of
+ Just (VQBool, e) ->
+ makeDuplicable buf $ \buf' ->
+ condE (pure e)
+ (pure buf')
+ (renderNodes dats (DuplicableExp buf') (toList nodes'))
+ Just (VQText, e) ->
+ makeDuplicable buf $ \buf' ->
+ condE [| T.null $(pure e) |]
+ (renderNodes dats (DuplicableExp buf') (toList nodes'))
+ (pure buf')
+ Just (VQMaybe _, e) ->
+ makeDuplicable buf $ \buf' ->
+ caseE (pure e)
+ [match (conP 'Nothing []) (normalB (renderNodes dats (DuplicableExp buf') (toList nodes'))) []
+ ,match (conP 'Just [wildP]) (normalB (pure buf')) []]
+ Just (VQList _, e) ->
+ makeDuplicable buf $ \buf' ->
+ caseE (pure e)
+ [match (listP []) (normalB (renderNodes dats (DuplicableExp buf') (toList nodes'))) []
+ ,match wildP (normalB (pure buf')) []]
+ Just (VQObject _, _) -> fail $ "Object value in inverted section scrutinee " ++ show named
+ Just (VQInt, _) -> fail $ "Int value in inverted section scrutinee " ++ show named
+ Just (VQUnit, _) -> fail $ "() value in inverted section scrutinee " ++ show named
+ Nothing -> fail $ "Inverted section scrutinee not found: " ++ show named
+ Partial _ _ -> fail $ "Unresolved partial in template: " ++ show node
+
+ renderNodes dats (ExpensiveExp buf') nodes
+
+search :: Vector Text -> NonEmpty (DataMap, Exp) -> Maybe (DataMap, Exp)
+search = search' . toList
+
+search' :: [Text] -> NonEmpty (DataMap, Exp) -> Maybe (DataMap, Exp)
+search' [] (value :| _) = Just value
+search' (key : keys) ((dm, e) :| dats) =
+ case dm of
+ VQObject fields ->
+ case lookup key fields of
+ Just (proj, dm') -> search' keys ((dm', proj e) :| [])
+ Nothing -> search' (key : keys) =<< NE.nonEmpty dats
+ _ ->
+ search' (key : keys) =<< NE.nonEmpty dats
+
+toPascalCase :: String -> String
+toPascalCase "" = ""
+toPascalCase (c:cs) = toUpper c : go cs
+ where go ('-':cs') = toPascalCase cs'
+ go (c':cs') = c' : go cs'
+ go "" = ""
diff --git a/mustache-th/Mustache/TH/Parser.hs b/mustache-th/Mustache/TH/Parser.hs
new file mode 100644
index 0000000..46d3e93
--- /dev/null
+++ b/mustache-th/Mustache/TH/Parser.hs
@@ -0,0 +1,352 @@
+{-|
+Copyright: (c) Tom Smeding 2026
+License: AGPL-3.0-or-later
+-}
+{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE DeriveFoldable #-}
+{-# LANGUAGE DeriveFunctor #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ViewPatterns #-}
+module Mustache.TH.Parser (Template(..), Chunk(..), parseTemplate, inlinePartials) where
+
+import Control.Monad (when, guard)
+import Control.Monad.Trans.Class (lift)
+import Control.Monad.Trans.Except
+import Control.Monad.Trans.State.Strict
+import Control.Monad.Trans.Writer.CPS
+import Data.Char
+import Data.Foldable (toList, foldMap')
+import Data.List (intercalate)
+import Data.Map.Strict (Map)
+import Data.Map.Strict qualified as Map
+import Data.Text (Text)
+import Data.Text qualified as T
+import Data.Vector (Vector)
+import Data.Vector qualified as V
+
+
+data Template = Template (Vector Chunk)
+ deriving (Show)
+
+data Chunk
+ = Text Text
+ | Var Bool (Vector Text)
+ | Section Bool (Vector Text) (Vector Chunk)
+ | Partial (Maybe Int) Text -- ^ indent, if standalone
+ deriving (Show)
+
+-- | The tags that have the "standalone" logic (i.e. if they're the only thing
+-- on a line except for spaces, that line is deleted entirely) have an
+-- additional Int that indicates the column (0-based) at which the tag occurs.
+data FlatChunk
+ = FlatText Text
+ | FlatVar Bool (Vector Text)
+ | StartSection Int Bool (Vector Text)
+ | EndSection Int (Vector Text)
+ | FlatPartial Int (Maybe Int) Text -- ^ tag column, partial indent if standalone
+ | EmptyTag Int
+ -- ^ A tag that produces no output, but does count as standalone for line elimination.
+ deriving (Show)
+
+
+-- |
+-- * If @recurse@ is true, recursively instantiates partials, and thus explodes
+-- when given a loop.
+-- * If @laxMissing@ is true, expands missing partials to the
+-- empty string instead of leaving them unexpanded.
+inlinePartials :: Bool -> Bool -> Map Text Template -> Template -> Template
+inlinePartials recurse laxMissing partials = \(Template chunks) -> Template (V.fromList (subChunks chunks))
+ where
+ subChunks :: Foldable t => t Chunk -> [Chunk]
+ subChunks chs = toList (foldMap' subChunk chs)
+
+ subChunk :: Chunk -> Bag Chunk
+ subChunk ch@Text{} = BOne ch
+ subChunk ch@Var{} = BOne ch
+ subChunk (Section pos var chs) = BOne (Section pos var (V.fromList (subChunks chs)))
+ subChunk original@(Partial mindent name) =
+ case Map.lookup name partials of
+ Just (Template chunks) ->
+ case mindent of
+ Just indent ->
+ let chunks' = snd (indentChunks True indent (toList chunks))
+ in if recurse then BList (subChunks chunks') else BList chunks'
+ Nothing -> BList (toList chunks)
+ Nothing | laxMissing -> mempty
+ | otherwise -> BOne original
+
+ indentChunks :: Bool -> Int -> [Chunk] -> (Bool, [Chunk])
+ indentChunks sol _ [] = (sol, [])
+ indentChunks sol n [Text (text T.:> '\n')] = (True, [Text (indentText sol n text T.:> '\n')])
+ indentChunks sol n (Text text@(_ T.:> '\n') : chs) = (Text (indentText sol n text) :) <$> indentChunks (n == 0) n chs
+ indentChunks sol n (Text text : chs) = (Text (indentText sol n text) :) <$> indentChunks False n chs
+
+ indentChunks sol n (ch@Var{} : chs) = ((if sol then (Text (T.replicate n " ") :) else id) . (ch :)) <$> indentChunks False n chs
+ indentChunks sol n (Section pos var body : chs) =
+ let (sol', body') = indentChunks sol n (toList body)
+ in (Section pos var (V.fromList body') :) <$> indentChunks sol' n chs
+
+ indentChunks sol n (Partial Nothing name : chs) = -- inline partial
+ (if sol then (Text (T.replicate n " ") :) else id) . (Partial Nothing name :) <$> indentChunks False n chs
+ indentChunks True n (Partial (Just indent) name : chs) = -- standalone partial with existing indent
+ (Partial (Just (n + indent)) name :) <$> indentChunks True n chs
+ indentChunks False _ (Partial (Just _) _ : _) = error "indentChunks: standalone partial not at start of line"
+
+ indentText :: Bool -> Int -> Text -> Text
+ indentText _ _ "" = ""
+ indentText sol n text = (if sol then T.replicate n " " else mempty) <> T.replace "\n" ("\n" <> T.replicate n " ") text
+
+
+parseTemplate :: Text -> Either String Template
+parseTemplate text =
+ fmap (Template . V.fromList) $ do
+ flatchunks <-
+ fmap (elimStandalone . toList)
+ . runExcept
+ . execWriterT
+ . flip runStateT (PState (mkCursor text) ("{{", "}}"))
+ $ pTemplate
+ -- traceShowM flatchunks
+ parseSections flatchunks
+
+-- | This also converts the column index in a 'FlatPartial' to the indent of
+-- the partial, if any: if the partial is standalone, the column index is
+-- preserved, otherwise it is set to 0.
+elimStandalone :: [FlatChunk] -> [FlatChunk]
+elimStandalone = \case
+ [] -> []
+
+ (withCol -> Just (0, ch)) : FlatText tafter : chs
+ | Just tafter' <- afterSpacesNewline tafter
+ -> ch : elimStandalone (tafter' ++ chs)
+
+ FlatText tbefore : (withCol -> Just (col, ch)) : FlatText tafter : chs
+ -- beforepre includes its final newline; tafter' also does
+ | (beforepre, beforepost) <- T.splitAt (T.length tbefore - col) tbefore
+ , T.length beforepost == col
+ , T.all isSpace beforepost
+ , Just tafter' <- afterSpacesNewline tafter
+ -> FlatText beforepre : ch : elimStandalone (tafter' ++ chs)
+
+ FlatPartial col Nothing name : chs -> FlatPartial col Nothing name : elimStandalone chs -- not standalone, so retain Nothing
+ FlatPartial _ Just{} _ : _ -> error "elimStandalone: FlatPartial with annotated indent"
+ ch : chs -> ch : elimStandalone chs
+ where
+ withCol FlatText{} = Nothing
+ withCol FlatVar{} = Nothing
+ withCol ch@(StartSection col _ _) = Just (col, ch)
+ withCol ch@(EndSection col _) = Just (col, ch)
+ withCol ch@(EmptyTag col) = Just (col, ch)
+ withCol (FlatPartial col _ name) =
+ -- withCol results are used only if the tag turns out to be actually standalone, so we can set the indent here
+ Just (col, FlatPartial col (Just col) name)
+
+ afterSpacesNewline tafter
+ | (pre, post) <- T.span (/= '\n') tafter
+ , T.all isSpace pre
+ = case post of
+ T.Empty -> Just []
+ '\n' T.:< post' -> Just [FlatText post']
+ _ T.:< _ -> error "impossible"
+ | otherwise
+ = Nothing
+
+parseSections :: [FlatChunk] -> Either String [Chunk]
+parseSections = \chs -> do
+ (chs', rest) <- go [] Nothing chs
+ case rest of
+ [] -> return chs'
+ _ -> error "impossible"
+ where
+ go :: [Chunk] -> Maybe (Vector Text) -> [FlatChunk] -> Either String ([Chunk], [FlatChunk])
+ go _ (Just ctx) [] =
+ Left $ "Unclosed section at end of template: " ++ showName ctx
+ go acc Nothing [] = return (reverse acc, [])
+ go acc mctx (FlatText text : chs) = go (Text text : acc) mctx chs
+ go acc mctx (FlatVar esc var : chs) = go (Var esc var : acc) mctx chs
+ go acc mctx (StartSection _ pos var : chs) = do
+ (body, after) <- go [] (Just var) chs
+ go (Section pos var (V.fromList body) : acc) mctx after
+ go acc (Just ctx) (EndSection _ var : chs)
+ | var == ctx = return (reverse acc, chs)
+ | otherwise = Left $ "Section " ++ showName ctx ++ " ended with {{/" ++ showName var ++ "}}"
+ go _ Nothing (EndSection _ var : _) =
+ Left $ "Found {{/" ++ showName var ++ "}} without opening tag"
+ go acc mctx (EmptyTag _ : chs) = go acc mctx chs
+ go acc mctx (FlatPartial _ indent path : chs) = go (Partial indent path : acc) mctx chs
+
+ showName :: Vector Text -> String
+ showName = intercalate "." . map T.unpack . V.toList
+
+data PState = PState
+ { psCursor :: Cursor
+ , psDelim :: (Text, Text) }
+
+cursor_ :: Lens PState Cursor
+cursor_ = Lens psCursor (\s x -> s { psCursor = x })
+delim_ :: Lens PState (Text, Text)
+delim_ = Lens psDelim (\s x -> s { psDelim = x })
+
+type M = StateT PState (WriterT (Bag FlatChunk) (Except String))
+
+raise :: String -> M a
+raise err = do
+ Cursor _ pos <- gets psCursor
+ lift . lift . throwE $ showCursorPos pos ++ ": " ++ err
+
+output :: FlatChunk -> M ()
+output = lift . tell . BOne
+
+pTemplate :: M ()
+pTemplate = do
+ (delim1, delim2) <- gets psDelim
+
+ -- Consume until the next tag
+ ch1 <- FlatText <$> stateL cursor_ (takeUntil delim1)
+ output ch1
+ ateof <- gets (eof . psCursor)
+ (_, tagcol) <- gets (cursorPos . psCursor)
+
+ -- Consume delim1; if not at eof, takeUntil stopped at a delim1, so this succeeds
+ when (not ateof) $ modifyL cursor_ (consume delim1)
+ -- Ensure we'll be parsing the correct closing delimiter, even with {{{stuff}}}
+ (triple, delim2') <- do
+ cursor <- gets psCursor -- if ateof the takeN check will fail
+ if delim1 == "{{" && delim2 == "}}" && fst (takeN 1 cursor) == "{"
+ then do modifyL cursor_ (consume "{")
+ return (True, "}}}")
+ else return (False, delim2)
+
+ -- Consume the body
+ contents <- stateL cursor_ (takeUntil delim2') -- empty if ateof
+
+ -- Consume delim2'; again, if not at eof, we will have delim2' here
+ when (not ateof) $ modifyL cursor_ (consume delim2')
+ if | ateof ->
+ return ()
+ | '!' T.:< _ <- contents -> do -- comment
+ output (EmptyTag tagcol)
+ pTemplate
+ | '#' T.:< (matchVarRef -> Just var) <- contents -> do
+ output (StartSection tagcol True var)
+ pTemplate
+ | '^' T.:< (matchVarRef -> Just var) <- contents -> do
+ output (StartSection tagcol False var)
+ pTemplate
+ | '/' T.:< (matchVarRef -> Just var) <- contents -> do
+ output (EndSection tagcol var)
+ pTemplate
+ | '>' T.:< (T.strip -> path) <- contents -> do
+ output (FlatPartial tagcol Nothing path)
+ pTemplate
+ | '&' T.:< (matchVarRef -> Just var) <- contents -> do
+ when triple $
+ raise $ "Use of {{{&...}}} which is redundant"
+ output (FlatVar False var)
+ pTemplate
+ | Just newdelims <- matchSetDelims contents -> do
+ modifyL delim_ (const newdelims)
+ output (EmptyTag tagcol)
+ pTemplate
+ | Just var <- matchVarRef contents -> do
+ output (FlatVar (not triple) var)
+ pTemplate
+ | otherwise ->
+ raise $ "Could not parse tag: " ++ T.unpack (delim1 <> contents <> delim2)
+
+matchSetDelims :: Text -> Maybe (Text, Text)
+matchSetDelims ('=' T.:< (text T.:> '=')) =
+ case lexer [TokLex TLMany isSpace
+ ,TokLex TLSome (\c -> not (isSpace c))
+ ,TokLex TLSome isSpace
+ ,TokLex TLSome (\c -> not (isSpace c))
+ ,TokLex TLMany isSpace]
+ text of
+ Just [_, d1, _, d2, _] -> Just (d1, d2)
+ _ -> Nothing
+matchSetDelims _ = Nothing
+
+matchVarRef :: Text -> Maybe (Vector Text)
+matchVarRef text = do
+ let comps = T.splitOn "." (T.strip text)
+ -- {{.}} results in ["", ""] upon splitting on "."
+ let implicit = comps == ["", ""]
+ guard (all isValidName comps || implicit)
+ return (if implicit then mempty else V.fromList comps)
+ where
+ isValidName T.Empty = False
+ isValidName name@(c1 T.:< _) = T.all isAlphaNum name && isAlpha c1
+
+
+data Lens a t = Lens (a -> t) (a -> t -> a)
+
+stateL :: Monad m => Lens s s' -> (s' -> (a, s')) -> StateT s m a
+stateL (Lens get' set) f = state (\s -> let (x, !s') = f (get' s) in (x,) $! set s s')
+
+modifyL :: Monad m => Lens s s' -> (s' -> s') -> StateT s m ()
+modifyL lens f = stateL lens (((),) . f)
+
+
+data TokLex = TokLex TLKind (Char -> Bool)
+data TLKind = TLOne | TLSome | TLMany
+
+-- | Returned list is the same length as the list of predicates.
+lexer :: [TokLex] -> Text -> Maybe [Text]
+lexer = go []
+ where
+ go :: [Text] -> [TokLex] -> Text -> Maybe [Text]
+ go acc [] T.Empty = Just (reverse acc)
+ go _ [] _ = Nothing
+ go acc (TokLex TLOne f : fs) t
+ | c T.:< t' <- t, f c = go (T.pack [c] : acc) fs t'
+ | otherwise = Nothing
+ go acc (TokLex TLSome f : fs) t =
+ let (tok, post) = T.span f t
+ in if T.null tok then Nothing else go (tok : acc) fs post
+ go acc (TokLex TLMany f : fs) t =
+ let (tok, post) = T.span f t
+ in go (tok : acc) fs post
+
+
+-- | tail (line, col); all zero-based
+data Cursor = Cursor { cursorText :: Text, cursorPos :: (Int, Int) }
+ deriving (Show)
+
+mkCursor :: Text -> Cursor
+mkCursor t = Cursor t (0, 0)
+
+takeN :: Int -> Cursor -> (Text, Cursor)
+takeN n (Cursor t pos) =
+ let (pre, post) = T.splitAt n t
+ in (pre, Cursor post (cursorSkipLC pre pos))
+
+-- | Throws if the given text is not found at the cursor
+consume :: Text -> Cursor -> Cursor
+consume text (Cursor t pos) =
+ case T.stripPrefix text t of
+ Just rest -> Cursor rest (cursorSkipLC text pos)
+ Nothing -> error $ "consume: Text " ++ show text ++ " not found at " ++ showCursorPos pos
+
+takeUntil :: Text -> Cursor -> (Text, Cursor)
+takeUntil delim (Cursor t pos) =
+ let (pre, post) = T.breakOn delim t
+ in (pre, Cursor post (cursorSkipLC pre pos))
+
+eof :: Cursor -> Bool
+eof (Cursor t _) = T.null t
+
+cursorSkipLC :: Text -> (Int, Int) -> (Int, Int)
+cursorSkipLC prefix (l, c) =
+ T.foldl' (\(l', c') -> \case '\n' -> (l'+1, 0)
+ _ -> (l', c'+1))
+ (l, c) prefix
+
+showCursorPos :: (Int, Int) -> String
+showCursorPos (l0, c0) = show (l0 + 1) ++ ":" ++ show (c0 + 1)
+
+
+data Bag a = BZero | BOne a | BTwo (Bag a) (Bag a) | BList [a]
+ deriving (Show, Functor, Foldable)
+
+instance Semigroup (Bag a) where (<>) = BTwo
+instance Monoid (Bag a) where mempty = BZero