Module and Use expressions (#245)

* Module and Use expressions

* UseFromExpr

* ImportFromExpr

* PubExpr

* Export, declares

* Collecting all the needed info WIP

* Got all the needed data

* Tests fixed

* HeaderSem

* HeaderSem wip

* Everything except `export`/`declares` should be working

* Compile bug fixed

* Fix readme: cli/assembly

* Handle declares, exports

* Compile only exports in AquaRes

* Call services imported from modules

* Import consts, types, services from modules

* Resolve arrows from modules

* Bugfix

.gitignore

@@ -1,4 +1,8 @@
 .idea
 .bsp
+.metals
+.vscode
+.bloop
+metals.sbt
 target
 project/target

.scalafmt.conf

@@ -58,4 +58,5 @@ rewrite {
   rules = [
     SortImports
   ]
-}
+}
+#runner.dialect = scala3

README.md

@@ -16,7 +16,7 @@ Please refer to [Aqua Book](https://doc.fluence.dev/aqua-book/) to learn how to
 
 ## Compiler CLI
 
-To build the Aqua compiler, clone the repo & run `sbt assembly`,
+To build the Aqua compiler, clone the repo & run `sbt cli/assembly`,
 or simply download the latest JAR file from the [releases](https://github.com/fluencelabs/aqua/releases) page.
 
 It requires `java` to run Aqua compiler from the command line:

build.sbt

@@ -17,7 +17,7 @@ val declineV = "2.1.0"
 name := "aqua-hll"
 
 val commons = Seq(
-  baseAquaVersion := "0.1.13",
+  baseAquaVersion := "0.1.14",
   version         := baseAquaVersion.value + "-" + sys.env.getOrElse("BUILD_NUMBER", "SNAPSHOT"),
   scalaVersion    := dottyVersion,
   libraryDependencies ++= Seq(

cli/src/main/scala/aqua/ErrorRendering.scala

@@ -5,7 +5,7 @@ import aqua.files.FileModuleId
 import aqua.io.AquaFileError
 import aqua.parser.lift.FileSpan
 import aqua.parser.{BlockIndentError, FuncReturnError, LexerError}
-import aqua.semantics.{RulesViolated, WrongAST}
+import aqua.semantics.{HeaderError, RulesViolated, WrongAST}
 import cats.Show
 
 object ErrorRendering {
@@ -61,6 +61,11 @@ object ErrorRendering {
             .focus(2)
             .map(_.toConsoleStr(message, Console.CYAN))
             .getOrElse("(Dup error, but offset is beyond the script)") + "\n"
+        case HeaderError(token, message) =>
+          token.unit._1
+            .focus(2)
+            .map(_.toConsoleStr(message, Console.CYAN))
+            .getOrElse("(Dup error, but offset is beyond the script)") + "\n"
         case WrongAST(ast) =>
           s"Semantic error"
 

cli/src/main/scala/aqua/files/FileModuleId.scala

@@ -1,11 +1,18 @@
 package aqua.files
 
 import fs2.io.file.Path
+import cats.Order
 
 case class FileModuleId private (file: Path)
 
 object FileModuleId {
 
+  implicit object FileModuleIdOrder extends Order[FileModuleId] {
+
+    override def compare(x: FileModuleId, y: FileModuleId): Int =
+      x.file.toString.compareTo(y.file.toString)
+  }
+
   def apply(file: Path): FileModuleId =
     new FileModuleId(file.absolute.normalize)
 }

compiler/src/main/scala/aqua/compiler/AquaCompiler.scala

@@ -6,47 +6,79 @@ import aqua.model.AquaContext
 import aqua.model.transform.TransformConfig
 import aqua.model.transform.res.AquaRes
 import aqua.parser.lift.LiftParser
+import aqua.parser.Ast
 import aqua.semantics.Semantics
+import aqua.semantics.header.HeaderSem
 import cats.data.Validated.{validNec, Invalid, Valid}
-import cats.data.{Chain, NonEmptyChain, Validated, ValidatedNec}
+import cats.data.{Chain, NonEmptyChain, NonEmptyMap, Validated, ValidatedNec}
 import cats.syntax.applicative.*
 import cats.syntax.flatMap.*
 import cats.syntax.functor.*
 import cats.syntax.traverse.*
-import cats.{Comonad, Monad}
+import cats.syntax.monoid.*
+import cats.{Comonad, Monad, Monoid, Order}
 import scribe.Logging
 
 object AquaCompiler extends Logging {
 
-  def compile[F[_]: Monad, E, I, S[_]: Comonad](
+  def compile[F[_]: Monad, E, I: Order, S[_]: Comonad](
     sources: AquaSources[F, E, I],
     liftI: (I, String) => LiftParser[S],
     backend: Backend,
     config: TransformConfig
   ): F[ValidatedNec[AquaError[I, E, S], Chain[AquaCompiled[I]]]] = {
     import config.aquaContextMonoid
     type Err = AquaError[I, E, S]
+    type Ctx = NonEmptyMap[I, AquaContext]
+    type ValidatedCtx = ValidatedNec[Err, Ctx]
+
     new AquaParser[F, E, I, S](sources, liftI)
-      .resolve[ValidatedNec[Err, AquaContext]] { ast => context =>
-        context.andThen { ctx =>
-          Semantics
-            .process(ast, ctx)
-            .leftMap(_.map[Err](CompileError(_)))
-        }
-      }
-      .map {
-        case Valid(modules) =>
-          Linker.link[I, AquaError[I, E, S], ValidatedNec[Err, AquaContext]](
-            modules,
-            cycle => CycleError[I, E, S](cycle.map(_.id))
-          ) match {
-            case Valid(filesWithContext) =>
+      .resolve[ValidatedCtx](mod =>
+        context =>
+          // Context with prepared imports
+          context.andThen(ctx =>
+            // To manage imports, exports run HeaderSem
+            HeaderSem
+              .sem(
+                mod.imports.view
+                  .mapValues(ctx(_))
+                  .collect { case (fn, Some(fc)) => fn -> fc }
+                  .toMap,
+                mod.body.head
+              )
+              .andThen { headerSem =>
+                // Analyze the body, with prepared initial context
+                Semantics
+                  .process(
+                    mod.body,
+                    headerSem.initCtx
+                  )
+                  // Handle exports, declares – finalize the resulting context
+                  .andThen(headerSem.finCtx)
+                  .map(rc => NonEmptyMap.one(mod.id, rc))
+              }
+              // The whole chain returns a semantics error finally
+              .leftMap(_.map[Err](CompileError(_)))
+          )
+      )
+      .map(
+        _.andThen(modules =>
+          Linker
+            .link[I, AquaError[I, E, S], ValidatedCtx](
+              modules,
+              cycle => CycleError[I, E, S](cycle.map(_.id)),
+              // By default, provide an empty context for this module's id
+              i => validNec(NonEmptyMap.one(i, Monoid.empty[AquaContext]))
+            )
+            .andThen { filesWithContext =>
               filesWithContext
                 .foldLeft[ValidatedNec[Err, Chain[AquaProcessed[I]]]](
                   validNec(Chain.nil)
                 ) {
                   case (acc, (i, Valid(context))) =>
-                    acc combine validNec(Chain.one(AquaProcessed(i, context)))
+                    acc combine validNec(
+                      Chain.fromSeq(context.toNel.toList.map { case (i, c) => AquaProcessed(i, c) })
+                    )
                   case (acc, (_, Invalid(errs))) =>
                     acc combine Invalid(errs)
                 }
@@ -56,13 +88,12 @@ object AquaCompiler extends Logging {
                     AquaCompiled(ap.id, compiled)
                   }
                 )
-            case i @ Invalid(_) => i
-          }
-        case i @ Invalid(_) => i
-      }
+            }
+        )
+      )
   }
 
-  def compileTo[F[_]: Monad, E, I, S[_]: Comonad, T](
+  def compileTo[F[_]: Monad, E, I: Order, S[_]: Comonad, T](
     sources: AquaSources[F, E, I],
     liftI: (I, String) => LiftParser[S],
     backend: Backend,

compiler/src/main/scala/aqua/compiler/AquaParser.scala

@@ -2,7 +2,7 @@ package aqua.compiler
 
 import aqua.linker.{AquaModule, Modules}
 import aqua.parser.Ast
-import aqua.parser.head.ImportExpr
+import aqua.parser.head.{FilenameExpr, ImportExpr}
 import aqua.parser.lift.LiftParser
 import cats.data.{Chain, NonEmptyChain, Validated, ValidatedNec}
 import cats.syntax.applicative.*
@@ -32,31 +32,45 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
       )
 
   // Resolve imports (not parse, just resolve) of the given file
-  def resolveImports(id: I, ast: Ast[S]): F[ValidatedNec[Err, Map[I, Err]]] =
+  def resolveImports(id: I, ast: Body): F[ValidatedNec[Err, AquaModule[I, Err, Body]]] =
     ast.head.tailForced
       .map(_.head)
-      .collect { case ImportExpr(filename) =>
+      .collect { case fe: FilenameExpr[F] =>
         sources
-          .resolveImport(id, filename.value.drop(1).dropRight(1))
+          .resolveImport(id, fe.fileValue)
           .map(
             _.bimap(
-              _.map(ResolveImportsErr(id, filename, _)),
-              importId => Chain.one[(I, Err)](importId -> ImportErr(filename))
+              _.map[Err](ResolveImportsErr(id, fe.filename, _)),
+              importId =>
+                Chain.one[(I, (String, Err))](importId -> (fe.fileValue, ImportErr(fe.filename)))
             )
           )
       }
       .traverse(identity)
       .map(
-        _.foldLeft(Validated.validNec[Err, Chain[(I, Err)]](Chain.nil))(_ combine _)
-          .map(_.toList.toMap)
+        _.foldLeft(Validated.validNec[Err, Chain[(I, (String, Err))]](Chain.nil))(_ combine _).map {
+          collected =>
+            AquaModule[I, Err, Body](
+              id,
+              // How filenames correspond to the resolved IDs
+              collected.map { case (i, (fn, _)) =>
+                fn -> i
+              }.toList.toMap[String, I],
+              // Resolved IDs to errors that point to the import in source code
+              collected.map { case (i, (_, err)) =>
+                i -> err
+              }.toList.toMap[I, Err],
+              ast
+            )
+        }
       )
 
   // Parse sources, convert to modules
   def sourceModules: F[ValidatedNec[Err, Modules[I, Err, Body]]] =
     parseSources.flatMap {
       case Validated.Valid(srcs) =>
         srcs.traverse { case (id, ast) =>
-          resolveImports(id, ast).map(_.map(AquaModule(id, _, ast)).map(Chain.one))
+          resolveImports(id, ast).map(_.map(Chain.one))
         }.map(
           _.foldLeft(Validated.validNec[Err, Chain[AquaModule[I, Err, Body]]](Chain.empty))(
             _ combine _
@@ -66,7 +80,7 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
         Validated.invalid[NonEmptyChain[Err], Chain[AquaModule[I, Err, Body]]](errs).pure[F]
     }.map(_.map(_.foldLeft(Modules[I, Err, Body]())(_.add(_, toExport = true))))
 
-  def loadModule(imp: I): F[ValidatedNec[Err, AquaModule[I, Err, Ast[S]]]] =
+  def loadModule(imp: I): F[ValidatedNec[Err, AquaModule[I, Err, Body]]] =
     sources
       .load(imp)
       .map(_.leftMap(_.map[Err](SourcesErr(_))).andThen { src =>
@@ -75,7 +89,7 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
       })
       .flatMap {
         case Validated.Valid(ast) =>
-          resolveImports(imp, ast).map(_.map(AquaModule(imp, _, ast)))
+          resolveImports(imp, ast)
         case Validated.Invalid(errs) =>
           Validated.invalid[NonEmptyChain[Err], AquaModule[I, Err, Ast[S]]](errs).pure[F]
       }
@@ -106,7 +120,9 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
       case err => err.pure[F]
     }
 
-  def resolve[T](transpile: Ast[S] => T => T): F[ValidatedNec[Err, Modules[I, Err, T => T]]] =
-    resolveSources.map(_.map(_.map(transpile)))
+  def resolve[T](
+    transpile: AquaModule[I, Err, Body] => T => T
+  ): F[ValidatedNec[Err, Modules[I, Err, T => T]]] =
+    resolveSources.map(_.map(_.mapModuleToBody(transpile)))
 
 }

compiler/src/main/scala/aqua/compiler/AquaProcessed.scala

@@ -3,5 +3,5 @@ package aqua.compiler
 import aqua.model.AquaContext
 
 case class AquaProcessed[I](id: I, context: AquaContext) {
-  def hasOutput: Boolean = context.funcs.nonEmpty
+  def hasOutput: Boolean = context.funcs.nonEmpty || context.services.nonEmpty
 }

linker/src/main/scala/aqua/linker/AquaModule.scala

@@ -1,8 +1,10 @@
 package aqua.linker
 
-case class AquaModule[I, E, T](id: I, dependsOn: Map[I, E], body: T) {
+case class AquaModule[I, E, T](id: I, imports: Map[String, I], dependsOn: Map[I, E], body: T) {
   def map[TT](f: T => TT): AquaModule[I, E, TT] = copy(body = f(body))
 
+  def mapWithId[TT](f: (I, T) => TT): AquaModule[I, E, TT] = copy(body = f(id, body))
+
   def mapErr[EE](f: E => EE): AquaModule[I, EE, T] =
     copy(dependsOn = dependsOn.view.mapValues(f).toMap)
 }

linker/src/main/scala/aqua/linker/Linker.scala

@@ -2,7 +2,7 @@ package aqua.linker
 
 import cats.data.{NonEmptyChain, Validated, ValidatedNec}
 import cats.kernel.{Monoid, Semigroup}
-import cats.syntax.monoid._
+import cats.syntax.semigroup._
 import scribe.Logging
 
 import scala.annotation.tailrec
@@ -50,17 +50,18 @@ object Linker extends Logging {
         }
     }
 
-  def link[I, E, T: Monoid](
+  def link[I, E, T: Semigroup](
     modules: Modules[I, E, T => T],
-    cycleError: List[AquaModule[I, E, T => T]] => E
+    cycleError: List[AquaModule[I, E, T => T]] => E,
+    empty: I => T
   ): ValidatedNec[E, Map[I, T]] =
     if (modules.dependsOn.nonEmpty) Validated.invalid(modules.dependsOn.values.reduce(_ ++ _))
     else {
       val result = iter(modules.loaded.values.toList, Map.empty[I, T => T], cycleError)
 
       Validated.fromEither(
         result
-          .map(_.view.filterKeys(modules.exports).mapValues(_.apply(Monoid[T].empty)).toMap)
+          .map(_.collect { case (i, f) if modules.exports(i) => i -> f(empty(i)) })
           .left
           .map(NonEmptyChain.one)
       )

linker/src/main/scala/aqua/linker/Modules.scala

@@ -28,6 +28,9 @@ case class Modules[I, E, T](
   def map[TT](f: T => TT): Modules[I, E, TT] =
     copy(loaded = loaded.view.mapValues(_.map(f)).toMap)
 
+  def mapModuleToBody[TT](f: AquaModule[I, E, T] => TT): Modules[I, E, TT] =
+    copy(loaded = loaded.view.mapValues(v => v.map(_ => f(v))).toMap)
+
   def mapErr[EE](f: E => EE): Modules[I, EE, T] =
     copy(
       loaded = loaded.view.mapValues(_.mapErr(f)).toMap,

linker/src/test/scala/aqua/linker/LinkerSpec.scala

@@ -15,6 +15,7 @@ class LinkerSpec extends AnyFlatSpec with Matchers {
         .add(
           AquaModule[String, String, String => String](
             "mod1",
+            Map.empty,
             Map("mod2" -> "unresolved mod2 in mod1"),
             _ ++ " | mod1"
           ),
@@ -24,17 +25,19 @@ class LinkerSpec extends AnyFlatSpec with Matchers {
 
     Linker.link[String, String, String](
       withMod1,
-      cycle => cycle.map(_.id).mkString(" -> ")
+      cycle => cycle.map(_.id).mkString(" -> "),
+      _ => ""
     ) should be(Validated.invalidNec("unresolved mod2 in mod1"))
 
     val withMod2 =
-      withMod1.add(AquaModule("mod2", Map.empty, _ ++ " | mod2"))
+      withMod1.add(AquaModule("mod2", Map.empty, Map.empty, _ ++ " | mod2"))
 
     withMod2.isResolved should be(true)
 
     Linker.link[String, String, String](
       withMod2,
-      cycle => cycle.map(_.id + "?").mkString(" -> ")
+      cycle => cycle.map(_.id + "?").mkString(" -> "),
+      _ => ""
     ) should be(Validated.validNec(Map("mod1" -> " | mod2 | mod1")))
   }