opensource/pandoc
3
0
Fork 0
mirror of https://github.com/jgm/pandoc.git synced 2026-08-28 17:20:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Collect transforms in new module T.P.Transforms. [API change]

Browse Source 
Bundles all document transformations, i.e., functions that run
predefined filters on Pandoc documents.
This commit is contained in:
Albert Krewinkel
2024-05-09 16:51:09 +02:00
committed by John MacFarlane
parent edf21e6dab
commit 7394bc920c
7 changed files with 179 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pandoc-server/src/Text/Pandoc/Server.hs
+3 -2
View File
@@ -39,12 +39,13 @@ import Text.Collate.Lang (Lang (..), parseLang)
import System.Console.GetOpt
import System.Environment (getProgName)
import qualified Control.Exception as E
import Text.Pandoc.Shared (safeStrRead, headerShift, filterIpynbOutput,
eastAsianLineBreakFilter)
import Text.Pandoc.Shared (safeStrRead)
import Text.Pandoc.App ( IpynbOutput (..), Opt(..), defaultOpts )
import Text.Pandoc.Builder (setMeta)
import Text.Pandoc.Format (parseFlavoredFormat, formatName)
import Text.Pandoc.SelfContained (makeSelfContained)
import Text.Pandoc.Transforms (headerShift, filterIpynbOutput,
eastAsianLineBreakFilter)
import System.Exit
import GHC.Generics (Generic)
import Network.Wai.Middleware.Cors ( cors,
pandoc.cabal
+1
View File
@@ -651,6 +651,7 @@ library
Text.Pandoc.Class.IO,
Text.Pandoc.Citeproc,
Text.Pandoc.Chunks,
Text.Pandoc.Transforms,
Text.Pandoc.Version
other-modules: Text.Pandoc.App.CommandLineOptions,
Text.Pandoc.App.Input,
src/Text/Pandoc/App.hs
+3 -9
View File
@@ -57,6 +57,8 @@ import Text.Pandoc.App.CommandLineOptions (parseOptions, parseOptionsFromArgs,
options, handleOptInfo)
import Text.Pandoc.App.Input (InputParameters (..), readInput)
import Text.Pandoc.App.OutputSettings (OutputSettings (..), optToOutputSettings)
import Text.Pandoc.Transforms (applyTransforms, filterIpynbOutput,
headerShift, eastAsianLineBreakFilter)
import Text.Collate.Lang (Lang (..), parseLang)
import Text.Pandoc.Filter (Filter (JSONFilter, LuaFilter), Environment (..),
applyFilters)
@@ -64,8 +66,7 @@ import qualified Text.Pandoc.Format as Format
import Text.Pandoc.PDF (makePDF)
import Text.Pandoc.Scripting (ScriptingEngine (..), CustomComponents(..))
import Text.Pandoc.SelfContained (makeSelfContained)
import Text.Pandoc.Shared (eastAsianLineBreakFilter,
headerShift, filterIpynbOutput, tshow)
import Text.Pandoc.Shared (tshow)
import Text.Pandoc.URI (isURI)
import Text.Pandoc.Writers.Shared (lookupMetaString)
import Text.Pandoc.Readers.Markdown (yamlToMeta)
@@ -341,8 +342,6 @@ data PandocOutput =
| ZipOutput BL.ByteString
deriving (Show)
type Transform = Pandoc -> Pandoc
-- | Configure the common state
configureCommonState :: PandocMonad m => Maybe FilePath -> Opt -> m ()
configureCommonState datadir opts = do
@@ -412,11 +411,6 @@ isTextFormat s = s `notElem` ["odt","docx","epub2","epub3","epub","pptx"]
adjustMetadata :: (Meta -> Meta) -> Pandoc -> Pandoc
adjustMetadata f (Pandoc meta bs) = Pandoc (f meta) bs
-- Transformations of a Pandoc document post-parsing:
applyTransforms :: Monad m => [Transform] -> Pandoc -> m Pandoc
applyTransforms transforms d = return $ foldr ($) d transforms
writeFnBinary :: FilePath -> BL.ByteString -> IO ()
writeFnBinary "-" = BL.putStr
writeFnBinary f = BL.writeFile (UTF8.encodePath f)
src/Text/Pandoc/App/Input.hs
+5 -54
View File
@@ -17,18 +17,18 @@ module Text.Pandoc.App.Input
import Control.Monad ((>=>))
import Control.Monad.Except (throwError, catchError)
import Data.Text (Text)
import Network.URI (URI (..), parseURI, unEscapeString)
import Network.URI (URI (..), parseURI)
import Text.Pandoc.Transforms (adjustLinksAndIds)
import Text.Pandoc.Class ( PandocMonad, openURL, toTextM
, readFileStrict, readStdinStrict, report)
import Text.Pandoc.Definition (Pandoc (..), Attr, Block (..), Inline (..))
import Text.Pandoc.Definition (Pandoc)
import Text.Pandoc.Error (PandocError (..))
import Text.Pandoc.Logging (LogMessage (..))
import Text.Pandoc.MIME (getCharset, MimeType)
import Text.Pandoc.Options (Extensions, ReaderOptions (..))
import Text.Pandoc.Options (ReaderOptions (..))
import Text.Pandoc.Readers (Reader (..))
import Text.Pandoc.Shared (tabFilter, textToIdentifier)
import Text.Pandoc.Shared (tabFilter)
import Text.Pandoc.URI (uriPathToPath)
import Text.Pandoc.Walk (walk)
import qualified Data.ByteString as BS
import qualified Data.ByteString.Char8 as B8
import qualified Data.ByteString.Lazy as BL
@@ -119,52 +119,3 @@ inputToText convTabs (fp, (bs,mt)) =
inputToLazyByteString :: (FilePath, (BS.ByteString, Maybe MimeType))
-> BL.ByteString
inputToLazyByteString (_, (bs,_)) = BL.fromStrict bs
adjustLinksAndIds :: Extensions -> Text -> [Text] -> Pandoc -> Pandoc
adjustLinksAndIds exts thisfile allfiles
| length allfiles > 1 = walk fixInline . walk fixBlock
| otherwise = id
where
toIdent :: Text -> Text
toIdent = textToIdentifier exts . T.intercalate "__" .
T.split (\c -> c == '/' || c == '\\')
fixBlock :: Block -> Block
fixBlock (CodeBlock attr t) = CodeBlock (fixAttrs attr) t
fixBlock (Header lev attr ils) = Header lev (fixAttrs attr) ils
fixBlock (Table attr cap cols th tbs tf) =
Table (fixAttrs attr) cap cols th tbs tf
fixBlock (Div attr bs) = Div (fixAttrs attr) bs
fixBlock x = x
-- add thisfile as prefix of identifier
fixAttrs :: Attr -> Attr
fixAttrs (i,cs,kvs)
| T.null i = (i,cs,kvs)
| otherwise =
(T.intercalate "__"
(filter (not . T.null) [toIdent thisfile, i]),
cs, kvs)
-- if URL begins with file from allfiles, convert to
-- an internal link with the appropriate identifier
fixURL :: Text -> Text
fixURL u =
let (a,b) = T.break (== '#') $ T.pack . unEscapeString . T.unpack $ u
filepart = if T.null a
then toIdent thisfile
else toIdent a
fragpart = T.dropWhile (== '#') b
in if T.null a || a `elem` allfiles
then "#" <> T.intercalate "__"
(filter (not . T.null) [filepart, fragpart])
else u
fixInline :: Inline -> Inline
fixInline (Code attr t) = Code (fixAttrs attr) t
fixInline (Link attr ils (url,tit)) =
Link (fixAttrs attr) ils (fixURL url,tit)
fixInline (Image attr ils (url,tit)) =
Image (fixAttrs attr) ils (fixURL url,tit)
fixInline (Span attr ils) = Span (fixAttrs attr) ils
fixInline x = x
src/Text/Pandoc/Readers/DocBook.hs
+2 -1
View File
@@ -44,8 +44,9 @@ import Text.Pandoc.Builder
import Text.Pandoc.Class.PandocMonad (PandocMonad, report)
import Text.Pandoc.Options
import Text.Pandoc.Logging (LogMessage(..))
import Text.Pandoc.Shared (safeRead, extractSpaces, headerShift)
import Text.Pandoc.Shared (safeRead, extractSpaces)
import Text.Pandoc.Sources (ToSources(..), sourcesToText)
import Text.Pandoc.Transforms (headerShift)
import Text.TeXMath (readMathML, writeTeX)
import qualified Data.Map as M
import Text.Pandoc.XML.Light
src/Text/Pandoc/Shared.hs
+1 -75
View File
@@ -55,7 +55,6 @@ module Text.Pandoc.Shared (
inlineListToIdentifier,
textToIdentifier,
isHeaderBlock,
headerShift,
stripEmptyParagraphs,
onlySimpleTableCells,
isTightList,
@@ -63,9 +62,7 @@ module Text.Pandoc.Shared (
taskListItemToAscii,
handleTaskListItem,
addMetaField,
eastAsianLineBreakFilter,
htmlSpanLikeElements,
filterIpynbOutput,
formatCode,
-- * TagSoup HTML handling
renderTags',
@@ -93,8 +90,7 @@ import Data.Containers.ListUtils (nubOrd)
import Data.Char (isAlpha, isLower, isSpace, isUpper, toLower, isAlphaNum,
generalCategory, GeneralCategory(NonSpacingMark,
SpacingCombiningMark, EnclosingMark, ConnectorPunctuation))
import Data.List (find, foldl', groupBy, intercalate, intersperse,
union, sortOn)
import Data.List (find, foldl', groupBy, intercalate, intersperse, union)
import qualified Data.Map as M
import Data.Maybe (mapMaybe)
import Data.Monoid (Any (..) )
@@ -113,7 +109,6 @@ import Data.Time
import Text.Pandoc.Asciify (toAsciiText)
import Text.Pandoc.Definition
import Text.Pandoc.Extensions (Extensions, Extension(..), extensionEnabled)
import Text.Pandoc.Generic (bottomUp)
import Text.DocLayout (charWidth)
import Text.Pandoc.Walk
-- for addPandocAttributes:
@@ -600,21 +595,6 @@ isHeaderBlock :: Block -> Bool
isHeaderBlock Header{} = True
isHeaderBlock _ = False
-- | Shift header levels up or down.
headerShift :: Int -> Pandoc -> Pandoc
headerShift n (Pandoc meta (Header m _ ils : bs))
| n < 0
, m + n == 0 = headerShift n $
B.setTitle (B.fromList ils) $ Pandoc meta bs
headerShift n (Pandoc meta bs) = Pandoc meta (walk shift bs)
where
shift :: Block -> Block
shift (Header level attr inner)
| level + n > 0 = Header (level + n) attr inner
| otherwise = Para inner
shift x = x
-- | Remove empty paragraphs.
stripEmptyParagraphs :: Pandoc -> Pandoc
stripEmptyParagraphs = walk go
@@ -692,65 +672,11 @@ addMetaField key val (Meta meta) =
tolist (MetaList ys) = ys
tolist y = [y]
-- | Remove soft breaks between East Asian characters.
eastAsianLineBreakFilter :: Pandoc -> Pandoc
eastAsianLineBreakFilter = bottomUp go
where go (x:SoftBreak:y:zs)
| Just (_, b) <- T.unsnoc $ stringify x
, Just (c, _) <- T.uncons $ stringify y
, charWidth b == 2
, charWidth c == 2
= x:y:zs
| otherwise
= x:SoftBreak:y:zs
go xs
= xs
-- | Set of HTML elements that are represented as Span with a class equal as
-- the element tag itself.
htmlSpanLikeElements :: Set.Set T.Text
htmlSpanLikeElements = Set.fromList ["kbd", "mark", "dfn"]
-- | Process ipynb output cells. If mode is Nothing,
-- remove all output. If mode is Just format, select
-- best output for the format. If format is not ipynb,
-- strip out ANSI escape sequences from CodeBlocks (see #5633).
filterIpynbOutput :: Maybe Format -> Pandoc -> Pandoc
filterIpynbOutput mode = walk go
where go (Div (ident, "output":os, kvs) bs) =
case mode of
Nothing -> Div (ident, "output":os, kvs) []
-- "best" for ipynb includes all formats:
Just fmt
| fmt == Format "ipynb"
-> Div (ident, "output":os, kvs) bs
| otherwise -> Div (ident, "output":os, kvs) $
walk removeANSI $
take 1 $ sortOn rank bs
where
rank (RawBlock (Format "html") _)
| fmt == Format "html" = 1 :: Int
| fmt == Format "markdown" = 3
| otherwise = 4
rank (RawBlock (Format "latex") _)
| fmt == Format "latex" = 1
| fmt == Format "markdown" = 3
| otherwise = 4
rank (RawBlock f _)
| fmt == f = 1
| otherwise = 4
rank (Para [Image{}]) = 2
rank _ = 3
removeANSI (CodeBlock attr code) =
CodeBlock attr (removeANSIEscapes code)
removeANSI x = x
removeANSIEscapes t
| Just cs <- T.stripPrefix "\x1b[" t =
removeANSIEscapes $ T.drop 1 $ T.dropWhile (/='m') cs
| Just (c, cs) <- T.uncons t = T.cons c $ removeANSIEscapes cs
| otherwise = ""
go x = x
-- | Reformat 'Inlines' as code, putting the stringlike parts in 'Code'
-- elements while bringing other inline formatting outside.
-- The idea is that e.g. `[Str "a",Space,Strong [Str "b"]]` should turn
src/Text/Pandoc/Transforms.hs
+164
View File
@@ -0,0 +1,164 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE StrictData #-}
{- |
Module : Text.Pandoc.Transforms
Copyright : © 2006-2024 John MacFarlane
License : GPL-2.0-or-later
Maintainer : John MacFarlane <jgm@berkeley@edu>
Transformation of a Pandoc document post-parsing
-}
module Text.Pandoc.Transforms
( Transform
, applyTransforms
, adjustLinksAndIds
, eastAsianLineBreakFilter
, filterIpynbOutput
, headerShift
) where
import Data.List (sortOn)
import Data.Text (Text)
import Network.URI (unEscapeString)
import Text.DocLayout (charWidth)
import Text.Pandoc.Definition
( Pandoc (..), Attr, Block (..), Format (..), Inline (..) )
import Text.Pandoc.Generic (bottomUp)
import Text.Pandoc.Options (Extensions)
import Text.Pandoc.Shared (stringify, textToIdentifier)
import Text.Pandoc.Walk (walk)
import qualified Data.Text as T
import qualified Text.Pandoc.Builder as B
-- | Transformation of a Pandoc document post-parsing
type Transform = Pandoc -> Pandoc
-- | Apply a list of transforms to a document, in order.
applyTransforms :: Monad m => [Transform] -> Pandoc -> m Pandoc
applyTransforms transforms d = return $ foldr ($) d transforms
-- | Prefixes identifiers with a string derived from the filepath of
-- @thisfile@; fixes links to targets in @allfiles@ accordingly.
adjustLinksAndIds :: Extensions -- ^ defines how IDs are generated
-> Text -- ^ thisfile
-> [Text] -- ^ allfiles
-> Transform
adjustLinksAndIds exts thisfile allfiles
| length allfiles > 1 = walk fixInline . walk fixBlock
| otherwise = id
where
-- fix ids in blocks
fixBlock :: Block -> Block
fixBlock (CodeBlock attr t) = CodeBlock (fixAttrs attr) t
fixBlock (Header lev attr ils) = Header lev (fixAttrs attr) ils
fixBlock (Table attr cap cols th tbs tf) =
Table (fixAttrs attr) cap cols th tbs tf
fixBlock (Div attr bs) = Div (fixAttrs attr) bs
fixBlock x = x
-- fix ids and links in inlines
fixInline :: Inline -> Inline
fixInline (Code attr t) = Code (fixAttrs attr) t
fixInline (Link attr ils (url,tit)) =
Link (fixAttrs attr) ils (fixURL url,tit)
fixInline (Image attr ils (url,tit)) =
Image (fixAttrs attr) ils (fixURL url,tit)
fixInline (Span attr ils) = Span (fixAttrs attr) ils
fixInline x = x
-- add thisfile as prefix of identifier
fixAttrs :: Attr -> Attr
fixAttrs (i,cs,kvs)
| T.null i = (i,cs,kvs)
| otherwise =
(T.intercalate "__"
(filter (not . T.null) [toIdent thisfile, i]),
cs, kvs)
-- turns a filepath into an identifier
toIdent :: Text -> Text
toIdent = textToIdentifier exts . T.intercalate "__" .
T.split (\c -> c == '/' || c == '\\')
-- if URL begins with file from allfiles, convert to
-- an internal link with the appropriate identifier
fixURL :: Text -> Text
fixURL u =
let (a,b) = T.break (== '#') $ T.pack . unEscapeString . T.unpack $ u
filepart = if T.null a
then toIdent thisfile
else toIdent a
fragpart = T.dropWhile (== '#') b
in if T.null a || a `elem` allfiles
then "#" <> T.intercalate "__"
(filter (not . T.null) [filepart, fragpart])
else u
-- | Process ipynb output cells. If mode is Nothing,
-- remove all output. If mode is Just format, select
-- best output for the format. If format is not ipynb,
-- strip out ANSI escape sequences from CodeBlocks (see #5633).
filterIpynbOutput :: Maybe Format -> Pandoc -> Pandoc
filterIpynbOutput mode = walk go
where go (Div (ident, "output":os, kvs) bs) =
case mode of
Nothing -> Div (ident, "output":os, kvs) []
-- "best" for ipynb includes all formats:
Just fmt
| fmt == Format "ipynb"
-> Div (ident, "output":os, kvs) bs
| otherwise -> Div (ident, "output":os, kvs) $
walk removeANSI $
take 1 $ sortOn rank bs
where
rank (RawBlock (Format "html") _)
| fmt == Format "html" = 1 :: Int
| fmt == Format "markdown" = 3
| otherwise = 4
rank (RawBlock (Format "latex") _)
| fmt == Format "latex" = 1
| fmt == Format "markdown" = 3
| otherwise = 4
rank (RawBlock f _)
| fmt == f = 1
| otherwise = 4
rank (Para [Image{}]) = 2
rank _ = 3
removeANSI (CodeBlock attr code) =
CodeBlock attr (removeANSIEscapes code)
removeANSI x = x
removeANSIEscapes t
| Just cs <- T.stripPrefix "\x1b[" t =
removeANSIEscapes $ T.drop 1 $ T.dropWhile (/='m') cs
| Just (c, cs) <- T.uncons t = T.cons c $ removeANSIEscapes cs
| otherwise = ""
go x = x
-- | Remove soft breaks between East Asian characters.
eastAsianLineBreakFilter :: Pandoc -> Pandoc
eastAsianLineBreakFilter = bottomUp go
where go (x:SoftBreak:y:zs)
| Just (_, b) <- T.unsnoc $ stringify x
, Just (c, _) <- T.uncons $ stringify y
, charWidth b == 2
, charWidth c == 2
= x:y:zs
| otherwise
= x:SoftBreak:y:zs
go xs
= xs
-- | Shift header levels up or down.
headerShift :: Int -> Pandoc -> Pandoc
headerShift n (Pandoc meta (Header m _ ils : bs))
| n < 0
, m + n == 0 = headerShift n $
B.setTitle (B.fromList ils) $ Pandoc meta bs
headerShift n (Pandoc meta bs) = Pandoc meta (walk shift bs)
where
shift :: Block -> Block
shift (Header level attr inner)
| level + n > 0 = Header (level + n) attr inner
| otherwise = Para inner
shift x = x