Skip to content

Commit

Permalink
Recursive markdown generation (#3268)
Browse files Browse the repository at this point in the history
- Closes #3130

Now the `markdown` command accepts the path of a project folder. In that
case, it will generate the markdown for all files in that project.
Similarly, if no input file or folder are provided, it will process all
the files in the project found from the cwd.
  • Loading branch information
janmasrovira authored Jan 15, 2025
1 parent f0454d6 commit f20492a
Show file tree
Hide file tree
Showing 16 changed files with 365 additions and 160 deletions.
14 changes: 14 additions & 0 deletions app/App.hs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module App where
import CommonOptions
import Data.ByteString qualified as ByteString
import GlobalOptions
import Juvix.Compiler.Backend.Markdown.Error
import Juvix.Compiler.Internal.Translation (InternalTypedResult)
import Juvix.Compiler.Internal.Translation.FromInternal.Analysis.Termination.Checker
import Juvix.Compiler.Pipeline.Loader.PathResolver
Expand Down Expand Up @@ -264,6 +265,16 @@ runPipeline opts input_ =
runPipelineLogger opts input_
. inject

runPipelineRecursive ::
forall r.
(Members '[App, EmbedIO, Logger, TaggedLock] r) =>
Maybe (AppPath File) ->
Sem r (InternalTypedResult, [InternalTypedResult])
runPipelineRecursive input_ = do
args <- askArgs
entry <- getEntryPoint' args input_
runReader defaultPipelineOptions (runPipelineHtmlEither entry) >>= fromRightJuvixError

runPipelineHtml ::
(Members '[App, EmbedIO, Logger, TaggedLock] r) =>
Bool ->
Expand Down Expand Up @@ -318,6 +329,9 @@ getRight = either appError return
runAppError :: forall e r a. (AppError e, Members '[App] r) => Sem (Error e ': r) a -> Sem r a
runAppError = runErrorNoCallStackWith appError

instance AppError MarkdownBackendError where
appError = appError . JuvixError

instance AppError ParserError where
appError = appError . JuvixError

Expand Down
14 changes: 7 additions & 7 deletions app/Commands/Format.hs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module Commands.Format where
import Commands.Base
import Commands.Format.Options
import Data.Text qualified as Text
import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.Scoping.Data.Context
import Juvix.Compiler.Pipeline.Loader.PathResolver.ImportTree.Base
import Juvix.Compiler.Store.Language (ModuleInfo)
import Juvix.Formatter

data FormatNoEditRenderMode
Expand Down Expand Up @@ -49,19 +49,19 @@ targetFromOptions opts = do
-- | Formats the project on the root
formatProject ::
forall r.
(Members '[App, EmbedIO, TaggedLock, Logger, ScopeEff, Files, Output FormattedFileInfo] r) =>
(Members (ScopeEff ': Output FormattedFileInfo ': AppEffects) r) =>
Sem r FormatResult
formatProject = silenceProgressLog . runPipelineOptions . runPipelineSetup $ do
res :: [(ImportNode, PipelineResult ModuleInfo)] <- processProject
res :: [ProcessedNode ScoperResult] <- processProjectUpToScoping
pkgId :: PackageId <- (^. entryPointPackageId) <$> ask
res' :: [(ImportNode, SourceCode)] <- runReader pkgId $ forM res $ \(node, nfo) -> do
src <- formatModuleInfo node nfo
return (node, src)
res' :: [(ImportNode, SourceCode)] <- runReader pkgId $ forM res $ \node -> do
src <- formatModuleInfo node
return (node ^. processedNode, src)
formatRes <- formatProjectSourceCode res'
formatPkgRes <- formatPackageDotJuvix
return (formatRes <> formatPkgRes)

formatPackageDotJuvix :: forall r. (Members '[App, Files, Logger, Output FormattedFileInfo, ScopeEff] r) => Sem r FormatResult
formatPackageDotJuvix :: forall r. (Members (Output FormattedFileInfo ': ScopeEff ': AppEffects) r) => Sem r FormatResult
formatPackageDotJuvix = do
pkgDotJuvix <- askPackageDotJuvixPath
ifM (fileExists' pkgDotJuvix) (format pkgDotJuvix) (return mempty)
Expand Down
41 changes: 20 additions & 21 deletions app/Commands/Html.hs
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,26 @@ runGenOnlySourceHtml HtmlOptions {..} = do
res <- runPipelineNoOptions _htmlInputFile upToScopingEntry
let m = res ^. Scoper.resultModule
outputDir <- fromAppPathDir _htmlOutputDir
liftIO $
Html.genSourceHtml
GenSourceHtmlArgs
{ _genSourceHtmlArgsAssetsDir = _htmlAssetsPrefix,
_genSourceHtmlArgsHtmlKind = Html.HtmlOnly,
_genSourceHtmlArgsOnlyCode = _htmlOnlyCode,
_genSourceHtmlArgsParamBase = "",
_genSourceHtmlArgsUrlPrefix = _htmlUrlPrefix,
_genSourceHtmlArgsIdPrefix = _htmlIdPrefix,
_genSourceHtmlArgsNoPath = _htmlNoPath,
_genSourceHtmlArgsFolderStructure = _htmlFolderStructure,
_genSourceHtmlArgsExt = _htmlExt,
_genSourceHtmlArgsStripPrefix = _htmlStripPrefix,
_genSourceHtmlArgsConcreteOpts = Concrete.defaultOptions,
_genSourceHtmlArgsModule = m,
_genSourceHtmlArgsComments = Scoper.getScoperResultComments res,
_genSourceHtmlArgsOutputDir = outputDir,
_genSourceHtmlArgsNoFooter = _htmlNoFooter,
_genSourceHtmlArgsNonRecursive = _htmlNonRecursive,
_genSourceHtmlArgsTheme = _htmlTheme
}
Html.genSourceHtml
GenSourceHtmlArgs
{ _genSourceHtmlArgsAssetsDir = _htmlAssetsPrefix,
_genSourceHtmlArgsHtmlKind = Html.HtmlOnly,
_genSourceHtmlArgsOnlyCode = _htmlOnlyCode,
_genSourceHtmlArgsParamBase = "",
_genSourceHtmlArgsUrlPrefix = _htmlUrlPrefix,
_genSourceHtmlArgsIdPrefix = _htmlIdPrefix,
_genSourceHtmlArgsNoPath = _htmlNoPath,
_genSourceHtmlArgsFolderStructure = _htmlFolderStructure,
_genSourceHtmlArgsExt = _htmlExt,
_genSourceHtmlArgsStripPrefix = _htmlStripPrefix,
_genSourceHtmlArgsConcreteOpts = Concrete.defaultOptions,
_genSourceHtmlArgsModule = m,
_genSourceHtmlArgsComments = Scoper.getScoperResultComments res,
_genSourceHtmlArgsOutputDir = outputDir,
_genSourceHtmlArgsNoFooter = _htmlNoFooter,
_genSourceHtmlArgsNonRecursive = _htmlNonRecursive,
_genSourceHtmlArgsTheme = _htmlTheme
}

resultToJudocCtx :: InternalTypedResult -> Html.JudocCtx
resultToJudocCtx res =
Expand Down
133 changes: 85 additions & 48 deletions app/Commands/Markdown.hs
Original file line number Diff line number Diff line change
@@ -1,62 +1,99 @@
module Commands.Markdown where
module Commands.Markdown (runCommand) where

import Commands.Base
import Commands.Markdown.Options
import Data.Text.IO qualified as Text
import Juvix.Compiler.Backend.Markdown.Error
import Juvix.Compiler.Backend.Markdown.Translation.FromTyped.Source
import Juvix.Compiler.Backend.Markdown.Translation.FromTyped.Source qualified as MK
import Juvix.Compiler.Concrete.Data.ScopedName qualified as S
import Juvix.Compiler.Concrete.Language qualified as Concrete
import Juvix.Compiler.Concrete.Pretty qualified as Concrete
import Juvix.Compiler.Concrete.Language as Concrete
import Juvix.Compiler.Concrete.Pretty as Concrete
import Juvix.Compiler.Concrete.Print.Base qualified as Concrete
import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.Scoping qualified as Scoper
import Juvix.Compiler.Concrete.Translation.FromParsed.Analysis.Scoping.Data.Context
import Juvix.Data.CodeAnn
import Juvix.Extra.Assets (writeAssets)

runCommand ::
forall r.
(Members AppEffects r) =>
MarkdownOptions ->
Sem r ()
runCommand opts = do
let inputFile = opts ^. markdownInputFile
scopedM <- runPipelineNoOptions inputFile upToScopingEntry
let m = scopedM ^. Scoper.resultModule
outputDir <- fromAppPathDir (opts ^. markdownOutputDir)
let res =
MK.fromJuvixMarkdown'
ProcessJuvixBlocksArgs
{ _processJuvixBlocksArgsConcreteOpts = Concrete.defaultOptions,
_processJuvixBlocksArgsUrlPrefix = opts ^. markdownUrlPrefix,
_processJuvixBlocksArgsIdPrefix =
opts ^. markdownIdPrefix,
_processJuvixBlocksArgsNoPath =
opts ^. markdownNoPath,
_processJuvixBlocksArgsExt =
opts ^. markdownExt,
_processJuvixBlocksArgsStripPrefix =
opts ^. markdownStripPrefix,
_processJuvixBlocksArgsComments = Scoper.getScoperResultComments scopedM,
_processJuvixBlocksArgsModule = m,
_processJuvixBlocksArgsOutputDir = outputDir,
_processJuvixBlocksArgsFolderStructure =
opts ^. markdownFolderStructure
}
case res of
Left err -> exitJuvixError (JuvixError err)
Right md
| opts ^. markdownStdout -> liftIO . putStrLn $ md
runCommand opts = runReader opts $ do
case opts ^. markdownInputFile of
Nothing -> goProject
Just p ->
fromAppPathFileOrDir p >>= \case
Left f -> goSingleFile f
Right {} -> goProject

goSingleFile ::
forall r.
(Members (Reader MarkdownOptions ': AppEffects) r) =>
Path Abs File ->
Sem r ()
goSingleFile f = do
let inputFile =
AppPath
{ _pathPath = preFileFromAbs f,
_pathIsInput = True
}
scopedM :: Scoper.ScoperResult <- runPipelineNoOptions (Just inputFile) upToScopingEntry
goScoperResult scopedM

goProject ::
forall r.
(Members (Reader MarkdownOptions ': AppEffects) r) =>
Sem r ()
goProject = runPipelineOptions . runPipelineSetup $ do
res :: [ProcessedNode ScoperResult] <- processProjectUpToScoping
forM_ res (goScoperResult . (^. processedNodeData))

goScoperResult :: (Members (Reader MarkdownOptions ': AppEffects) r) => Scoper.ScoperResult -> Sem r ()
goScoperResult scopedM = do
opts <- ask
let m :: Module 'Scoped 'ModuleTop = scopedM ^. Scoper.resultModule
if
| isNothing (m ^. moduleMarkdownInfo) ->
logInfo (mkAnsiText @(Doc CodeAnn) ("Skipping" <+> Concrete.docNoCommentsDefault (m ^. modulePath)))
| otherwise -> do
ensureDir outputDir
when (opts ^. markdownWriteAssets) $
liftIO $
writeAssets outputDir

let mdFile :: Path Rel File
mdFile =
relFile
( Concrete.topModulePathToDottedPath
(m ^. Concrete.modulePath . S.nameConcrete)
<.> markdownFileExt
)
absPath :: Path Abs File
absPath = outputDir <//> mdFile

liftIO $ Text.writeFile (toFilePath absPath) md
outputDir <- fromAppPathDir (opts ^. markdownOutputDir)
logProgress (mkAnsiText @(Doc CodeAnn) ("Processing" <+> Concrete.docNoCommentsDefault (m ^. modulePath)))
let args =
ProcessJuvixBlocksArgs
{ _processJuvixBlocksArgsConcreteOpts = Concrete.defaultOptions,
_processJuvixBlocksArgsUrlPrefix = opts ^. markdownUrlPrefix,
_processJuvixBlocksArgsIdPrefix =
opts ^. markdownIdPrefix,
_processJuvixBlocksArgsNoPath =
opts ^. markdownNoPath,
_processJuvixBlocksArgsExt =
opts ^. markdownExt,
_processJuvixBlocksArgsStripPrefix =
opts ^. markdownStripPrefix,
_processJuvixBlocksArgsComments = Scoper.getScoperResultComments scopedM,
_processJuvixBlocksArgsModule = m,
_processJuvixBlocksArgsOutputDir = outputDir,
_processJuvixBlocksArgsFolderStructure =
opts ^. markdownFolderStructure
}

md :: Text <- runAppError @MarkdownBackendError (MK.fromJuvixMarkdown args)
if
| opts ^. markdownStdout -> putStrLn md
| otherwise -> do
ensureDir outputDir
when (opts ^. markdownWriteAssets) $
writeAssets outputDir

let mdFile :: Path Rel File
mdFile =
relFile
( Concrete.topModulePathToDottedPath
(m ^. Concrete.modulePath . S.nameConcrete)
<.> markdownFileExt
)
absPath :: Path Abs File
absPath = outputDir <//> mdFile

writeFileEnsureLn absPath md
13 changes: 11 additions & 2 deletions app/Commands/Markdown/Options.hs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module Commands.Markdown.Options where
import CommonOptions

data MarkdownOptions = MarkdownOptions
{ _markdownInputFile :: Maybe (AppPath File),
{ _markdownInputFile :: Maybe (AppPath FileOrDir),
_markdownOutputDir :: AppPath Dir,
_markdownUrlPrefix :: Text,
_markdownIdPrefix :: Text,
Expand Down Expand Up @@ -33,7 +33,16 @@ parseJuvixMarkdown = do
<> showDefault
<> help "Prefix used for HTML element IDs"
)
_markdownInputFile <- optional (parseInputFile FileExtJuvixMarkdown)
_markdownInputFile <-
optional
( argument
someInputPreFileOrDirOpt
( metavar "JUVIX_MD_FILE_OR_PROJECT"
<> help ("Path to a " <> show FileExtJuvixMarkdown <> " file or to a directory containing a Juvix project.")
<> completer (extCompleter FileExtJuvixMarkdown)
<> action "directory"
)
)
_markdownOutputDir <-
parseGenericOutputDir
( value "markdown"
Expand Down
19 changes: 14 additions & 5 deletions app/CommonOptions.hs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import Juvix.Compiler.Tree.Data.TransformationId.Parser qualified as Tree
import Juvix.Data.Field
import Juvix.Data.Keyword.All qualified as Kw
import Juvix.Prelude
import Juvix.Prelude as Juvix
import Juvix.Prelude.Parsing qualified as P
import Juvix.Prelude.Pretty hiding (group, list)
import Options.Applicative hiding (helpDoc)
Expand Down Expand Up @@ -50,7 +49,7 @@ parseInputFilesMod :: NonEmpty FileExt -> Mod ArgumentFields (Prepath File) -> P
parseInputFilesMod exts' mods = do
let exts = NonEmpty.toList exts'
mvars = intercalate "|" (map toMetavar exts)
dotExts = intercalate ", " (map Prelude.show exts)
dotExts = intercalate ", " (map show exts)
helpMsg = "Path to a " <> dotExts <> " file"
completers = foldMap (completer . extCompleter) exts
_pathPath <-
Expand Down Expand Up @@ -130,7 +129,7 @@ parseNumThreads = do
<> value defaultNumThreads
<> showDefault
<> help "Number of physical threads to run"
<> completer (listCompleter (Juvix.show NumThreadsAuto : [Juvix.show j | j <- [1 .. numCapabilities]]))
<> completer (listCompleter (show NumThreadsAuto : [show j | j <- [1 .. numCapabilities]]))
)

parseProgramInputFile :: Parser (AppPath File)
Expand Down Expand Up @@ -186,6 +185,16 @@ parseGenericOutputDir m = do
somePreDirOpt :: ReadM (Prepath Dir)
somePreDirOpt = mkPrepath <$> str

someInputPreFileOrDirOpt :: ReadM (AppPath FileOrDir)
someInputPreFileOrDirOpt = mkInputAppPath . mkPrepath <$> str
where
mkInputAppPath :: Prepath f -> AppPath f
mkInputAppPath p =
AppPath
{ _pathPath = p,
_pathIsInput = True
}

somePreFileOrDirOpt :: ReadM (Prepath FileOrDir)
somePreFileOrDirOpt = mkPrepath <$> str

Expand Down Expand Up @@ -240,13 +249,13 @@ enumReader :: forall a. (Bounded a, Enum a, Show a) => Proxy a -> ReadM a
enumReader _ = eitherReader $ \val ->
case lookup val assocs of
Just x -> return x
Nothing -> Left ("Invalid value " <> val <> ". Valid values are: " <> (Juvix.show (allElements @a)))
Nothing -> Left ("Invalid value " <> val <> ". Valid values are: " <> (show (allElements @a)))
where
assocs :: [(String, a)]
assocs = [(Prelude.show x, x) | x <- allElements @a]

enumCompleter :: forall a. (Bounded a, Enum a, Show a) => Proxy a -> Completer
enumCompleter _ = listCompleter [Juvix.show e | e <- allElements @a]
enumCompleter _ = listCompleter [show e | e <- allElements @a]

extCompleter :: FileExt -> Completer
extCompleter ext = mkCompleter $ \word -> do
Expand Down
Loading

0 comments on commit f20492a

Please sign in to comment.