Add Clean.Doc, Clean.Parse, Clean.Parse.Comments

src/libraries/OS-Independent/Clean/Doc.dcl

+definition module Clean.Doc
+
+/**
+ * Parsing and storing Clean documentation
+ */
+
+import StdGeneric
+from StdOverloaded import class toString
+
+from Data.Either import :: Either
+from Data.GenDefault import generic gDefault
+from Data.Maybe import :: Maybe
+
+from Clean.Types import :: Type
+
+/**
+ * A wrapper around the {{`String`}} type which makes sure that multi-line
+ * documentation blocks get trimmed w.r.t. whitespace.
+ */
+:: MultiLineString = MultiLine !String
+
+class docDescription            d :: !d -> Maybe Description
+class docComplexity             d :: !d -> Maybe String
+class docParams                 d :: !d -> [ParamDoc]
+class docVars                   d :: !d -> [Description]
+class docResults                d :: !d -> [Description]
+class docType                   d :: !d -> Maybe Type
+class docThrows                 d :: !d -> [Description]
+class docMembers                d :: !d -> [Maybe ClassMemberDoc]
+class docFields                 d :: !d -> Maybe [Maybe Description]
+class docConstructors           d :: !d -> Maybe [Maybe ConstructorDoc]
+class docRepresentation         d :: !d -> Maybe (Maybe Description)
+class docPropertyBootstrap      d :: !d -> Maybe String
+class docPropertyTestWith       d :: !d -> [PropertyVarInstantiation]
+class docPropertyTestGenerators d :: !d -> [PropertyTestGenerator]
+class docProperties             d :: !d -> [Property]
+
+/**
+ * Documentation of a Clean module.
+ */
+:: ModuleDoc =
+	{ description        :: !Maybe Description
+	, property_bootstrap :: !Maybe MultiLineString      //* For generating unit tests with clean-test
+	, property_test_with :: ![PropertyVarInstantiation] //* With which types to test the properties
+	, property_test_generators :: ![PropertyTestGenerator]
+	  //* Functions to generate values of types for which Gast's {{`ggen`}} is not good enough, like {{`Map`}}
+	}
+
+instance docDescription ModuleDoc
+instance docPropertyBootstrap ModuleDoc
+instance docPropertyTestWith ModuleDoc
+instance docPropertyTestGenerators ModuleDoc
+derive gDefault ModuleDoc
+
+/**
+ * Documentation of a Clean function.
+ */
+:: FunctionDoc =
+	{ description        :: !Maybe Description
+	, complexity         :: !Maybe String  //* E.g. "O(n log n)"
+	, params             :: ![ParamDoc]    //* Descriptions of the parameters
+	, vars               :: ![Description] //* Descriptions of the type variables (for generics)
+	, results            :: ![Description] //* Descriptions of the result(s, for tuples)
+	, type               :: !Maybe Type    //* The type (for macros)
+	, throws             :: ![Description] //* The exceptions it may throw (iTasks)
+	, properties         :: ![Property]    //* Properties of this function
+	, property_test_with :: ![PropertyVarInstantiation] //* With which types to test the properties
+	, preconditions      :: ![String]      //* Preconditions for the properties
+	}
+
+instance docDescription FunctionDoc
+instance docComplexity FunctionDoc
+instance docParams FunctionDoc
+instance docVars FunctionDoc
+instance docResults FunctionDoc
+instance docType FunctionDoc
+instance docThrows FunctionDoc
+instance docPropertyTestWith FunctionDoc
+instance docProperties FunctionDoc
+
+/**
+ * Documentation of a function parameter.
+ */
+:: ParamDoc =
+	{ name        :: !Maybe String      //* An optional name for the parameter
+	, description :: !Maybe Description //* An optional description
+	}
+
+instance toString ParamDoc
+instance docDescription ParamDoc
+
+/**
+ * A property of a function.
+ * Typically, the property can be tested with Gast.
+ *
+ * - `ForAll`: the right-hand side (the third argument) holds for all values of
+ *   the arguments (the second argument). The first argument is the name.
+ */
+:: Property
+	= ForAll !String ![(!String,!Type)] !String
+
+/**
+ * When a property type contains type variables, a `PropertyVarInstantiation`
+ * can be used to instantiate those variables when generating test cases.
+ */
+:: PropertyVarInstantiation = PropertyVarInstantiation !(!String, !Type)
+
+/**
+ * A test generator generates values of some type. The first argument of the
+ * constructor is the function type of the generator, for instance
+ * `[(k,v)] -> {{Map}} k v`. The second argument is the implementation, which
+ * should assume the generator is called `gen` (e.g.: `gen elems = ...`).
+ */
+:: PropertyTestGenerator = PropertyTestGenerator !Type !String
+
+derive gDefault FunctionDoc, Property, PropertyVarInstantiation, PropertyTestGenerator
+
+/**
+ * Documentation of a Clean class member.
+ * For an explanation of the fields, see the documentation on {{`FunctionDoc`}}.
+ */
+:: ClassMemberDoc =
+	{ description :: !Maybe Description
+	, complexity  :: !Maybe String
+	, params      :: ![ParamDoc]
+	, results     :: ![Description]
+	, type        :: !Maybe Type
+	, throws      :: ![Description]
+	}
+
+instance docDescription ClassMemberDoc
+instance docComplexity ClassMemberDoc
+instance docParams ClassMemberDoc
+instance docResults ClassMemberDoc
+instance docType ClassMemberDoc
+instance docThrows ClassMemberDoc
+derive gDefault ClassMemberDoc
+
+/**
+ * Documentation of a Clean ADT constructor.
+ * For an explanation of the fields, see the documentation on {{`FunctionDoc`}}.
+ */
+:: ConstructorDoc =
+	{ description :: !Maybe Description
+	, params      :: ![ParamDoc]
+	}
+
+instance docDescription ConstructorDoc
+instance docParams ConstructorDoc
+derive gDefault ConstructorDoc
+
+/**
+ * Documentation of a Clean class.
+ */
+:: ClassDoc =
+	{ description :: !Maybe Description
+	, vars        :: ![Description]          //* The type variables
+	, members     :: ![Maybe ClassMemberDoc] //* Documentation on the members
+	}
+
+instance docDescription ClassDoc
+instance docVars ClassDoc
+instance docMembers ClassDoc
+derive gDefault ClassDoc
+
+/**
+ * Documentation of a Clean type.
+ */
+:: TypeDoc =
+	{ description    :: !Maybe Description
+	, vars           :: ![Description]                //* Type variables
+	, representation :: !Maybe (Maybe Description)    //* For synonym types
+	, fields         :: !Maybe [Maybe Description]    //* For records
+	, constructors   :: !Maybe [Maybe ConstructorDoc] //* For ADTs
+	}
+
+instance docDescription TypeDoc
+instance docVars TypeDoc
+instance docFields TypeDoc
+instance docConstructors TypeDoc
+instance docRepresentation TypeDoc
+derive gDefault TypeDoc
+
+/**
+ * Description of a Clean syntax element
+ */
+:: Description :== String
+
+/**
+ * Parse error for parsing Clean documentation; no documentation could be found
+ */
+:: ParseError
+	= MissingAsterisk !String //* At least one line did not start with a *
+	| MissingField !String    //* A required field was missing
+	| UnknownError !String    //* Another error
+	| InternalNoDataError
+
+/**
+ * Parse warning while parsing Clean documentation; the parser has made a
+ * best-effort result nevertheless
+ */
+:: ParseWarning
+	= UnknownField !String   //* Unknown @-field
+	| IllegalField !String   //* This @-field is not allowed in this docblock
+	| NoDescription          //* The main description is missing
+	| UsedReturn             //* Used @return instead of @result
+	| UnparsableType !String //* Could not parse a @type field as a type
+
+/**
+ * Convert a ConstructorDoc to a FunctionDoc.
+ */
+constructorToFunctionDoc :: !ConstructorDoc -> FunctionDoc
+
+/**
+ * Convert a FunctionDoc to a ClassMemberDoc.
+ */
+functionToClassMemberDoc :: !FunctionDoc -> ClassMemberDoc
+
+/**
+ * Add a class member to an existing class definition
+ *
+ * @param The documentation to add the member to
+ * @param The documentation on the class member
+ * @result The new ClassDoc
+ */
+addClassMemberDoc :: !ClassDoc !(Maybe ClassMemberDoc) -> ClassDoc
+
+/**
+ * Parse a single docstring, removing the asterisk and trimming whitespace.
+ */
+parseSingleLineDoc :: (String -> String)
+
+/**
+ * Parse a documentation block. The magic happens in {{`docBlockToDoc`}}.
+ */
+parseDoc :: !String -> Either ParseError (!d, ![ParseWarning]) | docBlockToDoc{|*|} d
+
+/**
+ * A documentation block.
+ * @representation An order list of key-value pairs. A key can occur multiple
+ *   times. The description has key `description`.
+ */
+:: DocBlock :== [(!String, !String)]
+
+/**
+ * The magic for {{`parseDoc`}}. Usually, a record type like {{`FunctionDoc`}}
+ * will derive a convenient parser. In some cases, it may be necessary to
+ * override the default, such as in the instance for {{`Type`}}, where parsing
+ * of the type happens.
+ * @var The thing to parse
+ */
+generic docBlockToDoc d :: !(Either [String] DocBlock) -> Either ParseError (!d, ![ParseWarning])
+
+derive docBlockToDoc UNIT, PAIR, EITHER, CONS, OBJECT, FIELD of d, RECORD
+derive docBlockToDoc String, [], Maybe, Type
+derive docBlockToDoc ModuleDoc, FunctionDoc, ClassMemberDoc, ClassDoc,
+	ConstructorDoc, TypeDoc
+
+/**
+ * Trace a list of ParseWarnings like StdDebug might do it
+ */
+traceParseWarnings :: ![ParseWarning] !a -> a
+
+/**
+ * Trace a ParseError like StdDebug might do it
+ */
+traceParseError :: !ParseError !a -> a

src/libraries/OS-Independent/Clean/Parse.dcl

+definition module Clean.Parse
+
+/**
+ * A small wrapper around the parser of the Clean compiler.
+ * You will need to have the source of the Clean compiler available in your path.
+ */
+
+from Data.Error import :: MaybeError
+from Data.Maybe import :: Maybe
+from System.FilePath import :: FilePath
+
+from hashtable import :: HashTable
+from Heap import :: Heap
+from syntax import :: Module, :: ParsedDefinition, :: ParsedModule
+
+/**
+ * Parse a Clean module.
+ *
+ * @param The path to the file to parse
+ * @result
+ *   The parsed module and the corresponding hash table.
+ *   When the result is an {{`Error`}}, there is a descriptive error message.
+ */
+readModule :: !FilePath !*World -> *(!MaybeError String (ParsedModule, HashTable), !*World)

src/libraries/OS-Independent/Clean/Parse.icl

+implementation module Clean.Parse
+
+// NOTE: be VERY restrictive with adding imports here, because this may break
+// the module when the compiler changes.
+
+import StdFile
+
+import Data.Error
+import Data.Maybe
+import System.File
+import System.FilePath
+from Text import class Text(endsWith), instance Text String
+
+from hashtable import :: BoxedIdent{boxed_ident}, :: HashTable,
+	:: IdentClass(IC_Module), :: QualifiedIdents(NoQualifiedIdents),
+	putIdentInHashTable, set_hte_mark, newHashTable
+from parse import wantModule
+import syntax
+
+readModule :: !FilePath !*World -> *(!MaybeError String (ParsedModule, HashTable), !*World)
+readModule filename w
+# (s,w) = readFile filename w
+| isError s = (Error (toString (fromError s)), w)
+# modname = getModuleName (fromString (fromOk s))
+# modname = fromMaybe (takeFileName (dropExtension filename)) modname
+# ht = newHashTable newHeap
+# ht = set_hte_mark (if icl 1 0) ht
+# (ok,f,w) = fopen filename FReadText w
+| not ok = (Error ("Couldn't open " +++ filename), w)
+# (mod_id, ht) = putIdentInHashTable modname (IC_Module NoQualifiedIdents) ht
+# ((b1,b2,pm,ht,f),w) = accFiles (wantModule` f "" icl mod_id.boxed_ident NoPos True ht stderr) w
+# (ok,w) = fclose f w
+| not ok = (Error ("Couldn't close " +++ filename), w)
+= (Ok (pm, ht), w)
+where
+	icl = endsWith "icl" filename
+
+	wantModule` :: !*File !{#Char} !Bool !Ident !Position !Bool !*HashTable !*File !*Files
+		-> ((!Bool,!Bool,!ParsedModule, !*HashTable, !*File), !*Files)
+	wantModule` f s b1 i p b2 ht io fs
+	# (b1,b2,pm,ht,f,fs) = wantModule f s b1 i p b2 ht io fs
+	= ((b1,b2,pm,ht,f),fs)
+
+// A reasonably accurate simple scanner to get the module name from the file
+
+getModuleName :: ![Char] -> Maybe String
+getModuleName ['definition':c:cs]     | isSpace c = justModule cs
+getModuleName ['implementation':c:cs] | isSpace c = justModule cs
+getModuleName ['system':c:cs]         | isSpace c = justModule cs
+getModuleName [c:cs]                  | isSpace c = getModuleName cs
+getModuleName ['//':cs]                           = getModuleName (dropWhile ((<>) '\n') cs)
+getModuleName ['/*':cs]                           = getModuleName (skipMultiLineComment cs)
+getModuleName cs                                  = justModule cs
+
+justModule :: ![Char] -> Maybe String
+justModule ['module':c:cs] | isSpace c = justModuleName cs
+justModule [c:cs]          | isSpace c = justModule cs
+justModule ['//':cs]                   = justModule (dropWhile ((<>) '\n') cs)
+justModule ['/*':cs]                   = justModule (skipMultiLineComment cs)
+justModule _                           = Nothing
+
+justModuleName :: ![Char] -> Maybe String
+justModuleName cs
+# (_,cs) = span isSpace cs
+# (name,_) = span (\c -> c <> '/' && c <> ';' && not (isSpace c)) cs
+= case name of
+	[] -> Nothing
+	_  -> Just (toString name)
+
+skipMultiLineComment :: ![Char] -> [Char]
+skipMultiLineComment ['*/':cs] = cs
+skipMultiLineComment ['/*':cs] = skipMultiLineComment (skipMultiLineComment cs)
+skipMultiLineComment [c:cs] = skipMultiLineComment cs
+skipMultiLineComment [] = []

src/libraries/OS-Independent/Clean/Parse/Comments.dcl

+definition module Clean.Parse.Comments
+
+/**
+ * This module can combine the AST of the Clean compiler (which can be parsed
+ * using {{`Clean.Parse`}} with comments scanned by {{`Clean.ScanComments`}}.
+ */
+
+from StdFile import class FileSystem
+
+from Data.Error import :: MaybeError
+from Data.Maybe import :: Maybe
+from System.File import :: FileError
+from System.FilePath import :: FilePath
+
+from syntax import :: Ident, :: Module, :: ParsedDefinition, :: ParsedModule
+
+:: CleanComment =
+	{ line      :: !Int
+	, column    :: !Int
+	, level     :: !Maybe Int
+	, content   :: !String
+	, multiline :: !Bool
+	}
+
+scanComments :: !FilePath !*env -> *(!MaybeError FileError [CleanComment], !*env) | FileSystem env
+scanCommentsFile :: !*File -> *(!MaybeError FileError [CleanComment], !*File)
+
+:: CollectedComments
+
+emptyCollectedComments :: CollectedComments
+getComment :: !Ident !CollectedComments -> Maybe String
+collectComments :: ![CleanComment] !ParsedModule -> CollectedComments

tests/linux64/test.icl

@@ -12,6 +12,9 @@ import qualified StdMaybe
 import qualified StdLibMisc
 
 // Main libraries
+import qualified Clean.Doc
+//import qualified Clean.Parse // requires Clean compiler
+//import qualified Clean.Parse.Comments // requires Clean compiler
 import qualified Clean.Types
 //import qualified Clean.Types.CoclTransform // requires Clean compiler
 import qualified Clean.Types.Parse