Depend on encoding and begin to implement code pages in RTF reader.

See #9683.  TODO: implement language shifting with `\langN` and
`\deflangN`, taking account of `\fcharsetN`.

pandoc.cabal

@@ -531,7 +531,8 @@ library
                  vector                >= 0.12     && < 0.14,
                  djot                  >= 0.1.1.3  && < 0.2,
                  tls                   >= 2.0.1    && < 2.1,
-                 crypton-x509-system   >= 1.6.7    && < 1.7
+                 crypton-x509-system   >= 1.6.7    && < 1.7,
+                 encoding              >= 0.8.9    && < 0.9
 
   if !os(windows)
     build-depends:  unix >= 2.4 && < 2.9

src/Text/Pandoc/Readers/RTF.hs

@@ -37,7 +37,10 @@ import qualified Data.ByteString.Lazy as BL
 import Data.Digest.Pure.SHA (sha1, showDigest)
 import Data.Maybe (mapMaybe, fromMaybe)
 import Safe (lastMay, initSafe, headDef)
--- import Debug.Trace
+import Data.Encoding (decodeLazyByteStringExplicit)
+import Data.Encoding.CP932 (CP932)
+
+import Debug.Trace
 
 -- TODO:
 -- [ ] more complex table features
@@ -956,22 +959,13 @@ processFontTable = snd . foldl' go (0, mempty)
 
 ansiWords :: Maybe Int -> [Word8] -> [Char]
 ansiWords mbCodePage ws =
-  case mbCodePage of
-    Just 932 -> map cp932ToChar $ cp932Split ws
+  case traceShowId mbCodePage of
+    Just 932 -> case decodeLazyByteStringExplicit (undefined :: CP932)
+                  (BL.pack ws) of
+                  Left _ -> "\xFFFD"
+                  Right cs -> cs
     _ -> map defaultAnsiWordToChar ws
 
-cp932ToChar :: Word16 -> Char
-cp932ToChar i = chr $ fromIntegral i  -- TODO
--- to properly implement this we'll need the full lookup table
--- https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP932.TXT
-
-cp932Split :: [Word8] -> [Word16]
-cp932Split [] = []
-cp932Split (i:j:is)
-  | (i >= 0x81 && i <= 0x9F) || i >= 0xE0
-  = fromIntegral ((i * 0xFF) + j) : cp932Split is
-cp932Split (i:is) = fromIntegral i : cp932Split is
-
 defaultAnsiWordToChar :: Word8 -> Char
 defaultAnsiWordToChar i =
   case i of