diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b190bec..2e53aab8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,3 +83,6 @@ jobs: - name: Integration test run: scala-cli --power test integration-tests + + - name: Test besom-cfg + run: just test-besom-cfg diff --git a/Justfile b/Justfile index a7c84072..0cc0efbf 100644 --- a/Justfile +++ b/Justfile @@ -1,6 +1,7 @@ # Big idea behind using a Justfile is so that we can have modules like in sbt. besom-version := `cat version.txt` +besom-cfg-version := `cat besom-cfg/version.txt` is-snapshot := if "{{besom-version}}" =~ '.*-SNAPSHOT' { "true" } else { "false" } language-plugin-output-dir := justfile_directory() + "/.out/language-plugin" @@ -35,16 +36,16 @@ default: #################### # Cleans everything -clean-all: clean-json clean-sdk clean-auto clean-out clean-compiler-plugin clean-codegen clean-scripts clean-test-integration clean-test-templates clean-test-examples clean-test-markdown +clean-all: clean-json clean-sdk clean-auto clean-out clean-compiler-plugin clean-codegen clean-scripts clean-test-integration clean-cfg clean-test-templates clean-test-examples clean-test-markdown # Compiles everything compile-all: compile-json compile-sdk compile-auto compile-codegen compile-scripts compile-compiler-plugin build-language-plugin # Tests everything -test-all: test-json test-sdk test-auto test-codegen test-scripts test-integration test-templates test-examples test-markdown +test-all: test-json test-sdk test-auto test-codegen test-scripts test-integration test-cfg test-templates test-examples test-markdown # Publishes everything locally -publish-local-all: publish-local-json publish-local-sdk publish-local-auto publish-local-codegen publish-local-scripts install-language-plugin +publish-local-all: publish-local-json publish-local-sdk publish-local-auto publish-local-cfg publish-local-codegen publish-local-scripts install-language-plugin # Publishes everything to Maven publish-maven-all: publish-maven-json publish-maven-sdk publish-maven-auto publish-maven-codegen publish-maven-scripts @@ -278,6 +279,63 @@ publish-language-plugins-all: package-language-plugins-all just publish-language-plugin windows arm64 just publish-language-plugin windows amd64 +#################### +# Besom CFG +#################### + +# Compiles besom-cfg lib module +compile-cfg-lib: publish-local-json publish-local-core + scala-cli --power compile besom-cfg/lib --suppress-experimental-feature-warning + +# Compiles besom-cfg k8s module +compile-cfg-k8s: publish-local-cfg-lib + just cli packages local kubernetes:4.10.0 + scala-cli --power compile besom-cfg/k8s --suppress-experimental-feature-warning + +# Compiles all besom-cfg modules +compile-cfg: compile-cfg-lib compile-cfg-k8s + +# Publishes locally besom-cfg lib module +publish-local-cfg-lib: + scala-cli --power publish local besom-cfg/lib --project-version {{besom-cfg-version}} --suppress-experimental-feature-warning + +# Publishes locally besom-cfg k8s module +publish-local-cfg-k8s: compile-cfg-k8s + scala-cli --power publish local besom-cfg/k8s --project-version {{besom-cfg-version}} --suppress-experimental-feature-warning + +# Publishes locally all besom-cfg modules +publish-local-cfg: publish-local-cfg-lib publish-local-cfg-k8s + +# Publishes besom-cfg lib module to Maven +publish-maven-cfg-lib: + scala-cli --power publish besom-cfg/lib --project-version {{besom-cfg-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning + +# Publishes besom-cfg k8s module to Maven +publish-maven-cfg-k8s: + scala-cli --power publish besom-cfg/k8s --project-version {{besom-cfg-version}} {{publish-maven-auth-options}} --suppress-experimental-feature-warning + +# Tests besom-cfg lib module +test-cfg-lib: compile-cfg-lib + scala-cli --power test besom-cfg/lib --suppress-experimental-feature-warning + +# Tests besom-cfg k8s module +test-cfg-k8s: publish-local-cfg-lib compile-cfg-k8s + scala-cli --power test besom-cfg/k8s --suppress-experimental-feature-warning + +# Runs all tests of besom-cfg +test-cfg: test-cfg-lib test-cfg-k8s + +# Cleans besom-cfg-lib build +clean-cfg-lib: + scala-cli clean besom-cfg/lib + +# Cleans besom-cfg-k8s build +clean-cfg-k8s: + scala-cli clean besom-cfg/k8s + +# Cleans all besom-cfg builds +clean-cfg: clean-cfg-lib clean-cfg-k8s + #################### # Codegen #################### diff --git a/besom-cfg/k8s/.scalafmt.conf b/besom-cfg/k8s/.scalafmt.conf new file mode 100644 index 00000000..f4d4b655 --- /dev/null +++ b/besom-cfg/k8s/.scalafmt.conf @@ -0,0 +1,11 @@ +version = 3.5.2 +runner.dialect = scala3 +project.git = true +align = most +align.openParenCallSite = false +align.openParenDefnSite = false +align.tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="] +indent.defnSite = 2 +maxColumn = 140 + +rewrite.scala3.insertEndMarkerMinLines = 40 \ No newline at end of file diff --git a/besom-cfg/k8s/project.scala b/besom-cfg/k8s/project.scala new file mode 100644 index 00000000..905fab62 --- /dev/null +++ b/besom-cfg/k8s/project.scala @@ -0,0 +1,23 @@ +//> using scala 3.3.3 + +//> using dep com.lihaoyi::os-lib::0.9.3 +//> using dep org.virtuslab::besom-cfg:0.2.0-SNAPSHOT +//> using dep org.virtuslab::besom-kubernetes::4.10.0-core.0.4-SNAPSHOT +//> using dep com.lihaoyi::fansi::0.2.14 +//> using dep com.lihaoyi::fastparse:3.0.2 + +//> using test.resourceDir ./src/test/resources + +//> using test.dep com.lihaoyi::pprint:0.6.6 +//> using test.dep org.scalameta::munit:1.0.0-M11 + +//> using publish.name "besom-cfg-k8s" +//> using publish.organization "org.virtuslab" +//> using publish.url "https://github.com/VirtusLab/besom" +//> using publish.vcs "github:VirtusLab/besom" +//> using publish.license "Apache-2.0" +//> using publish.repository "central" +//> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy" +//> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ" +//> using publish.developer "KacperFKorban|Kacper Korban|https://github.com/KacperFKorban" +//> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak" diff --git a/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala b/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala new file mode 100644 index 00000000..a99f7863 --- /dev/null +++ b/besom-cfg/k8s/src/main/scala/ConfiguredContainerArgs.scala @@ -0,0 +1,172 @@ +package besom.cfg.k8s + +import besom.cfg.internal.* +import besom.types.{Input, Context, Output} +import besom.cfg.* +import besom.json.* +import besom.cfg.containers.* +import besom.api.kubernetes.core.v1.inputs.* + +import scala.util.* +import scala.quoted.* +import besom.cfg.k8s.syntax.* + +// this is besom-cfg-kubernetes entrypoint + +object syntax: + extension (s: Struct) + def foldedToEnvVarArgs(using Context): Output[List[EnvVarArgs]] = + s.foldToEnv.map(_.map { case (k, v) => EnvVarArgs(name = k, value = v) }) + +object ConfiguredContainerArgs: + + private val NL = System.lineSeparator() + + inline def apply[C <: Struct]( + name: String, + image: String, + configuration: C, + args: Input.Optional[List[Input[String]]] = None, + command: Input.Optional[List[Input[String]]] = None, + env: Input.Optional[List[Input[EnvVarArgs]]] = None, + envFrom: Input.Optional[List[Input[EnvFromSourceArgs]]] = None, + imagePullPolicy: Input.Optional[String] = None, + lifecycle: Input.Optional[LifecycleArgs] = None, + livenessProbe: Input.Optional[ProbeArgs] = None, + ports: Input.Optional[List[Input[ContainerPortArgs]]] = None, + readinessProbe: Input.Optional[ProbeArgs] = None, + resizePolicy: Input.Optional[List[Input[ContainerResizePolicyArgs]]] = None, + resources: Input.Optional[ResourceRequirementsArgs] = None, + restartPolicy: Input.Optional[String] = None, + securityContext: Input.Optional[SecurityContextArgs] = None, + startupProbe: Input.Optional[ProbeArgs] = None, + stdin: Input.Optional[Boolean] = None, + stdinOnce: Input.Optional[Boolean] = None, + terminationMessagePath: Input.Optional[String] = None, + terminationMessagePolicy: Input.Optional[String] = None, + tty: Input.Optional[Boolean] = None, + volumeDevices: Input.Optional[List[Input[VolumeDeviceArgs]]] = None, + volumeMounts: Input.Optional[List[Input[VolumeMountArgs]]] = None, + workingDir: Input.Optional[String] = None + )(using ctx: Context) = ${ + applyImpl( + 'name, + 'image, + 'configuration, + 'args, + 'command, + 'env, + 'envFrom, + 'imagePullPolicy, + 'lifecycle, + 'livenessProbe, + 'ports, + 'readinessProbe, + 'resizePolicy, + 'resources, + 'restartPolicy, + 'securityContext, + 'startupProbe, + 'stdin, + 'stdinOnce, + 'terminationMessagePath, + 'terminationMessagePolicy, + 'tty, + 'volumeDevices, + 'volumeMounts, + 'workingDir, + 'ctx + ) + } + + def applyImpl[C <: Struct: Type]( + name: Expr[String], + image: Expr[String], + configuration: Expr[C], + args: Expr[Input.Optional[List[Input[String]]]], + command: Expr[Input.Optional[List[Input[String]]]], + env: Expr[Input.Optional[List[Input[EnvVarArgs]]]], + envFrom: Expr[Input.Optional[List[Input[EnvFromSourceArgs]]]], + imagePullPolicy: Expr[Input.Optional[String]], + lifecycle: Expr[Input.Optional[LifecycleArgs]], + livenessProbe: Expr[Input.Optional[ProbeArgs]], + ports: Expr[Input.Optional[List[Input[ContainerPortArgs]]]], + readinessProbe: Expr[Input.Optional[ProbeArgs]], + resizePolicy: Expr[Input.Optional[List[Input[ContainerResizePolicyArgs]]]], + resources: Expr[Input.Optional[ResourceRequirementsArgs]], + restartPolicy: Expr[Input.Optional[String]], + securityContext: Expr[Input.Optional[SecurityContextArgs]], + startupProbe: Expr[Input.Optional[ProbeArgs]], + stdin: Expr[Input.Optional[Boolean]], + stdinOnce: Expr[Input.Optional[Boolean]], + terminationMessagePath: Expr[Input.Optional[String]], + terminationMessagePolicy: Expr[Input.Optional[String]], + tty: Expr[Input.Optional[Boolean]], + volumeDevices: Expr[Input.Optional[List[Input[VolumeDeviceArgs]]]], + volumeMounts: Expr[Input.Optional[List[Input[VolumeMountArgs]]]], + workingDir: Expr[Input.Optional[String]], + context: Expr[Context] + )(using Quotes): Expr[ContainerArgs] = + import quotes.reflect.* + + val contName = name.value match + case None => report.errorAndAbort("Container name has to be a literal!", name) + case Some(value) => value + + val dockerImage = image.value match + case None => report.errorAndAbort("Image name has to be a literal!", image) + case Some(value) => value + + val schema = getDockerImageMetadata(dockerImage) match + case Left(throwable) => report.errorAndAbort(s"Failed to get metadata for image $dockerImage:$NL${pprint(throwable)}", image) + case Right(schema) => schema + + Diff.performDiff(schema, configuration) match + case Left(prettyDiff) => // TODO maybe strip all the ansi codes if in CI? + report.errorAndAbort( + s"Configuration provided for container $contName ($dockerImage) is invalid:$NL$NL$prettyDiff", + configuration + ) + + case Right(()) => + val envExpr = '{ + val envOutput = ${ env }.asOptionOutput()(using ${ context }) + val conf = ${ configuration } + val configurationAsEnvVarArgs = conf.foldedToEnvVarArgs(using $context) + envOutput.zip(configurationAsEnvVarArgs).map { + case (Some(envVarArgsList), envVarArgsListFromConf) => envVarArgsList ++ envVarArgsListFromConf + case (None, envVarArgsListFromConf) => envVarArgsListFromConf + } + } + + '{ + ContainerArgs( + args = $args, + command = $command, + env = $envExpr, + envFrom = $envFrom, + image = $image, + imagePullPolicy = $imagePullPolicy, + lifecycle = $lifecycle, + livenessProbe = $livenessProbe, + name = ${ Expr(contName) }, + ports = $ports, + readinessProbe = $readinessProbe, + resizePolicy = $resizePolicy, + resources = $resources, + restartPolicy = $restartPolicy, + securityContext = $securityContext, + startupProbe = $startupProbe, + stdin = $stdin, + stdinOnce = $stdinOnce, + terminationMessagePath = $terminationMessagePath, + terminationMessagePolicy = $terminationMessagePolicy, + tty = $tty, + volumeDevices = $volumeDevices, + volumeMounts = $volumeMounts, + workingDir = $workingDir + )(using $context) + } + end match + end applyImpl +end ConfiguredContainerArgs diff --git a/besom-cfg/k8s/src/main/scala/containers.scala b/besom-cfg/k8s/src/main/scala/containers.scala new file mode 100644 index 00000000..57d5da68 --- /dev/null +++ b/besom-cfg/k8s/src/main/scala/containers.scala @@ -0,0 +1,63 @@ +package besom.cfg.containers + +// this should be a separate package, base for all container integrations + +import besom.cfg.internal.Schema +import besom.json.* +import scala.util.Try + +val cacheDir = sys.props.get("java.io.tmpdir").getOrElse("/tmp") + +def sanitizeImageName(image: String): String = + image + .replace("/", "_") + .replace(":", "_") + +def fetchFromCache(image: String): Option[String] = + if image.endsWith(":latest") then None + else + val sanitized = sanitizeImageName(image) + os.makeDir.all(os.Path(s"$cacheDir/besom-cfg")) + Try(os.read(os.Path(s"$cacheDir/besom-cfg/$sanitized"))).toOption + +def saveToCache(image: String, content: String): Unit = + if !image.endsWith(":latest") then + val sanitized = sanitizeImageName(image) + os.makeDir.all(os.Path(s"$cacheDir/besom-cfg")) + os.write.over(os.Path(s"$cacheDir/besom-cfg/$sanitized"), content) + +def resolveMetadataFromImage(image: String): String = + lazy val sbtNativePackagerFormatCall = + os + .proc("docker", "run", "--rm", "--entrypoint", "java", image, "-cp", "/opt/docker/lib/*", "besom.cfg.SummonConfiguration") + .call(check = false) + + lazy val customDockerFormatCall = + os + .proc("docker", "run", "--rm", "--entrypoint", "java", image, "-cp", "/app/main", "besom.cfg.SummonConfiguration") + .call(check = false) + + if sbtNativePackagerFormatCall.exitCode == 0 then sbtNativePackagerFormatCall.out.text().trim() + else if customDockerFormatCall.exitCode == 0 then customDockerFormatCall.out.text().trim() + else throw RuntimeException(s"Failed to get configuration from $image") + +def getDockerImageMetadata(image: String): Either[Throwable, Schema] = + Try { + // 1. cache result per image in /tmp DONE + // 2. verify the version of the library used, fail macro if we are older than it + // 3. parse the json to correct structure DONE + // next: + // - support different image setups, autodetect which one is used somehow? somewhat DONE + // - cp argument should be configurable + val json = fetchFromCache(image) match { + case Some(cachedJson) => cachedJson + case None => + val json = resolveMetadataFromImage(image) + + saveToCache(image, json) + + json + } + + summon[JsonFormat[Schema]].read(json.parseJson) + }.toEither diff --git a/besom-cfg/k8s/src/main/scala/diffs.scala b/besom-cfg/k8s/src/main/scala/diffs.scala new file mode 100644 index 00000000..8583c483 --- /dev/null +++ b/besom-cfg/k8s/src/main/scala/diffs.scala @@ -0,0 +1,341 @@ +package besom.cfg + +import scala.collection.immutable.{ListMap, Set} +import besom.cfg.internal.* +import scala.quoted.* +import besom.types.Output + +enum Val: + case Str(value: String) + case Map(value: ListMap[String, Val]) + case List(value: Val.Map) + +type ValMap = ListMap[String, Val] + +object ValMap: + inline def empty: ValMap = ListMap.empty + +private val NL = System.lineSeparator() + +// TODO this is to be deprecated and replaced completely with Tpe.parseType +// TODO however, diff was written based on ValMap and Tpe is too precise (allows unions, Outputs) +// TODO we want to diff without outputs so current setup where we strip outputs from Tpe and then +// TODO convert to ValMap kinda works, but it's not ideal +def parseType(tpe: String): ValMap = + val lines = tpe.split(NL).toVector.drop(1).dropRight(1) // drop the first and last line: { } + def loop(lines: Vector[String], parent: ValMap): (Vector[String], ValMap) = + lines match + case Vector() => (Vector.empty, parent) + case Vector(line, rest @ _*) => + val (key, value) = line.split(":").map(_.trim).toList match + case key :: value :: Nil => (key, value) + case "}" :: Nil => ("", "}") + case "}]" :: Nil => ("", "}]") + case _ => throw Exception(s"Invalid line: $line") + + value match + case "List[{" => + val (rest2, nested) = loop(rest.toVector, ValMap.empty) + loop(rest2, parent + (key -> Val.List(Val.Map(nested)))) + case "{" => + val (rest2, nested) = loop(rest.toVector, ValMap.empty) + loop(rest2, parent + (key -> Val.Map(nested))) + case "}" => + (rest.toVector, parent) + case "}]" => + (rest.toVector, parent) + case _ => + loop(rest.toVector, parent + (key -> Val.Str(value))) + + val (rest, valmap) = loop(lines, ValMap.empty) + assert(rest.isEmpty) + valmap + +def prettifyTypeString(tpe: String): String = + tpe + .replace("Predef.", "") + .replace("scala.collection.immutable.", "") + .replace("besom.internal.", "") + .replace("besom.cfg.Struct ", "") + .replace("besom.cfg.Struct", "{}") // empty Structs handled here + .replace("scala.", "") + .replace("java.lang.", "") + .replace("val ", "") + +def applyColorToTextOnly(str: fansi.Str, f: fansi.Str => fansi.Str): fansi.Str = + val indent = str.toString.takeWhile(_.isWhitespace) + val text = str.toString.dropWhile(_.isWhitespace) + val colored = f(fansi.Str(text)) + fansi.Str(s"$indent$colored") + +// rebuild the type string but mark actual differences with red, green and yellow colors +def diff(obtained: ValMap, expected: ValMap): fansi.Str = + import fansi.Str + import fansi.Color.{Green, Yellow, Red} + + def diffInternal(obtained: ValMap, expected: ValMap, indent: String = " "): Vector[fansi.Str] = + val keys = (expected.keys ++ obtained.keys).toVector.distinct + keys.map { key => + val obtainedValue = obtained.get(key) + val expectedValue = expected.get(key) + + (obtainedValue, expectedValue) match + // same key and type on both sides, plain string + case (Some(Val.Str(obtained)), Some(Val.Str(expected))) if obtained == expected => + Str(s"$indent$key: $obtained") + + // same key, different type (simple type) + case (Some(Val.Str(obtained)), Some(Val.Str(expected))) => + val indentedKeyWithoutColor = Str(s"$indent$key: ") + val redObtained = Red(s"got $obtained") + val yelloExpected = Yellow(s", expected $expected") + + indentedKeyWithoutColor ++ redObtained ++ yelloExpected + + case (Some(Val.List(Val.Map(obtained))), Some(Val.List(Val.Map(expected)))) => + val nestedListDiff = diffInternal(obtained, expected, indent + " ") + val nestedListWrappedInBraces = nestedListDiff.mkString(s"List[{$NL", NL, s"$NL$indent}]") + + Str(s"$indent$key: ") ++ nestedListWrappedInBraces + + // same key, difference in a nested struct + case (Some(Val.Map(obtained)), Some(Val.Map(expected))) => + val nestedStructDiff = diffInternal(obtained, expected, indent + " ") + val nestedStructWrappedInBraces = nestedStructDiff.mkString(s"{$NL", NL, s"$NL$indent}") + + Str(s"$indent$key: $nestedStructWrappedInBraces") + + // present in infra type, not present in application type, unnecessary addition (struct) + case (Some(Val.Map(obtained)), None) => + val nestedStructDiff = diffInternal(obtained, ValMap.empty, indent + " ") + // inject Green in newlines because compiler strips it :( + val nestedStructWrappedInBraces = + nestedStructDiff + .map(applyColorToTextOnly(_, Green.apply)) + .mkString(s"${Green("{")}$NL", NL, s"$NL$indent${Green("}")}") + + Green(s"$indent$key: ") ++ nestedStructWrappedInBraces + + // present in infra type, missing in application type, unnecessary addition (simple type) + case (Some(Val.Str(obtained)), None) => + Green(s"$indent$key: $obtained") + + // present in infra type, missing in application type, unnecessary addition (list of structs) + case (Some(Val.List(Val.Map(obtained))), None) => + val nestedListDiff = diffInternal(obtained, ValMap.empty, indent + " ") + val nestedListWrappedInBraces = nestedListDiff.mkString(s"${Green("List[{")}$NL", NL, s"$NL$indent${Green("}]")}") + + Green(s"$indent$key: ") ++ nestedListWrappedInBraces + + // present in application type, missing in infra type (simple type) + case (None, Some(Val.Str(expected))) => + Red(s"$indent$key: $expected") + + // present in application type, missing in infra type (list of structs) + case (None, Some(Val.List(Val.Map(expected)))) => + val nestedStructDiff = diffInternal(ValMap.empty, expected, indent + " ") + // inject Red in newlines because compiler strips it :( + val nestedStructWrappedInBraces = + nestedStructDiff + .map(applyColorToTextOnly(_, Red.apply)) + .mkString(s"${Red("List[{")}$NL", NL, s"$NL$indent${Red("}]")}") + + Red(s"$indent$key: ") ++ nestedStructWrappedInBraces + + // present in application type, missing in infra type (struct) + case (None, Some(Val.Map(expected))) => + val nestedStructDiff = diffInternal(ValMap.empty, expected, indent + " ") + // inject Red in newlines because compiler strips it :( + val nestedStructWrappedInBraces = + nestedStructDiff + .map(applyColorToTextOnly(_, Red.apply)) + .mkString(s"${Red("{")}$NL", NL, s"$NL$indent${Red("}")}") + + Red(s"$indent$key: ") ++ nestedStructWrappedInBraces + + // obtained simple type, expected struct + case (Some(Val.Str(obtained)), Some(Val.Map(expected))) => + val indentedKeyWithoutColor = Str(s"$indent$key: ") + val redObtained = Red(s"got $obtained") + val nestedStructDiff = diffInternal(ValMap.empty, expected, indent + " ") + // inject Yellow in newlines because compiler strips it :( + val nestedStructWrappedInBraces = + nestedStructDiff + .map(applyColorToTextOnly(_, Yellow.apply)) + .mkString(s"${Yellow("{")}$NL", NL, s"$NL$indent${Yellow("}")}") + + val yelloExpected = Yellow(s", expected ") ++ nestedStructWrappedInBraces + + indentedKeyWithoutColor ++ redObtained ++ yelloExpected + + // obtained struct, expected list of structs + case (Some(Val.Map(obtained)), Some(Val.List(Val.Map(expected)))) => + val indentedKeyWithoutColor = Str(s"$indent$key: ") + val nestedObtainedStructDiff = diffInternal(obtained, ValMap.empty, indent + " ") + val nestedObtainedStructWrappedInBraces = + nestedObtainedStructDiff + .map(applyColorToTextOnly(_, Red.apply)) + .mkString(s"${Red("got {")}$NL", NL, s"$NL$indent${Red("}")}") + + val nestedExpectedListOfStructsDiff = diffInternal(ValMap.empty, expected, indent + " ") + val nestedExpectedListOfStructsWrappedInBraces = + nestedExpectedListOfStructsDiff + .map(applyColorToTextOnly(_, Yellow.apply)) + .mkString(s"${Yellow(", expected List[{")}$NL", NL, s"$NL$indent${Yellow("}]")}") + + indentedKeyWithoutColor ++ nestedObtainedStructWrappedInBraces ++ nestedExpectedListOfStructsWrappedInBraces + + // obtained simple type, expected list of structs + case (Some(Val.Str(obtained)), Some(Val.List(Val.Map(expected)))) => + val indentedKeyWithoutColor = Str(s"$indent$key: ") + val redObtained = Red(s"got $obtained") + val nestedStructDiff = diffInternal(ValMap.empty, expected, indent + " ") + val nestedStructWrappedInBraces = + nestedStructDiff + .map(applyColorToTextOnly(_, Yellow.apply)) + .mkString(s"${Yellow("{")}$NL", NL, s"$NL$indent${Yellow("}]")}") + + val yelloExpected = Yellow(s", expected List[") ++ nestedStructWrappedInBraces + + indentedKeyWithoutColor ++ redObtained ++ yelloExpected + + // obtained list of structs, expected simple type + case (Some(Val.List(Val.Map(obtained))), Some(Val.Str(expected))) => + val nestedListDiff = diffInternal(ValMap.empty, obtained, indent + " ") + val nestedListWrappedInBraces = nestedListDiff.mkString(s"${Red("got List[{")}$NL", NL, s"$NL$indent${Red("}]")}") + + val yelloExpected = Yellow(s", expected $expected") + + Str(s"$indent$key: ") ++ nestedListWrappedInBraces ++ yelloExpected + + // obtained list of structs, expected struct + case (Some(Val.List(Val.Map(obtained))), Some(Val.Map(expected))) => + val indentedKeyWithoutColor = Str(s"$indent$key: ") + val nestedObtainedStructDiff = diffInternal(obtained, ValMap.empty, indent + " ") + val nestedObtainedStructWrappedInBraces = + nestedObtainedStructDiff + .map(applyColorToTextOnly(_, Red.apply)) + .mkString(s"${Red("got List[{")}$NL", NL, s"$NL$indent${Red("}]")}") + + val nestedExpectedListOfStructsDiff = diffInternal(ValMap.empty, expected, indent + " ") + val nestedExpectedListOfStructsWrappedInBraces = + nestedExpectedListOfStructsDiff + .map(applyColorToTextOnly(_, Yellow.apply)) + .mkString(s"${Yellow(", expected {")}$NL", NL, s"$NL$indent${Yellow("}")}") + + indentedKeyWithoutColor ++ nestedObtainedStructWrappedInBraces ++ nestedExpectedListOfStructsWrappedInBraces + + // obtained struct, expected simple type + case (Some(Val.Map(obtained)), Some(Val.Str(expected))) => + val indentedKeyWithoutColor = Str(s"$indent$key: ") + val nestedStructDiff = diffInternal(obtained, ValMap.empty, indent + " ") + val nestedStructWrappedInBraces = + nestedStructDiff + .map(applyColorToTextOnly(_, Red.apply)) + .mkString(s"${Red("got {")}$NL", NL, s"$NL$indent${Red("}")}") + + val yelloExpected = Yellow(s", expected $expected") + + indentedKeyWithoutColor ++ nestedStructWrappedInBraces ++ yelloExpected + // impossible state + case (None, None) => throw Exception(s"Invalid state: $key is missing on both sides") + end match + } + end diffInternal + + s"{$NL" + diffInternal(obtained, expected).mkString(NL) + s"$NL}" +end diff + +object Diff: + def performDiff[C <: Struct: Type](schema: Schema, configuration: Expr[C])(using Quotes): Either[fansi.Str, Unit] = + import quotes.reflect.* + + // TODO this generates really f****d up types like: + // shouldBeListOfStructsButItsAStruct: List[{ + // x: Int | Output[Int] + // y: Double | Output[Double] + // } | Output[{ + // x: Int | Output[Int] + // y: Double | Output[Double] + // }]] | List[Output[{ + // x: Int | Output[Int] + // y: Double | Output[Double] + // } | Output[{ + // x: Int | Output[Int] + // y: Double | Output[Double] + // }]]] + def fieldTypeToTypeRepr(fieldType: FieldType): TypeRepr = + fieldType match + case FieldType.Int => TypeRepr.of[scala.Int | Output[scala.Int]] + case FieldType.Long => TypeRepr.of[scala.Long | Output[scala.Long]] + case FieldType.String => TypeRepr.of[String | Output[String]] + case FieldType.Boolean => TypeRepr.of[scala.Boolean | Output[scala.Boolean]] + case FieldType.Float => TypeRepr.of[scala.Float | Output[scala.Float]] + case FieldType.Double => TypeRepr.of[scala.Double | Output[scala.Double]] + case FieldType.Optional(inner) => // TODO this is borked + fieldTypeToTypeRepr(inner).asType match + case '[t] => TypeRepr.of[t | Null] + case FieldType.Array(inner) => + fieldTypeToTypeRepr(inner).asType match + case '[t] => TypeRepr.of[List[t] | List[Output[t]]] + case FieldType.Struct(fields: _*) => + val refinements = fields.map { (name, fieldType) => + val typeRepr = fieldTypeToTypeRepr(fieldType) + name -> typeRepr + }.toList + + MetaUtils.refineType(TypeRepr.of[Struct], refinements).asType match + case '[t] => TypeRepr.of[t | Output[t]] + + def dealiasAll(tpe: TypeRepr): TypeRepr = + tpe match + case AppliedType(tycon, args) => AppliedType(dealiasAll(tycon), args.map(dealiasAll(_))) + case _ => tpe.dealias + + def schemaToTypeRepr(schema: Schema): TypeRepr = + val refinements = schema.fields.map { case Field(name, fieldType) => + val typeRepr = fieldTypeToTypeRepr(fieldType) + name -> typeRepr + }.toList + + MetaUtils.refineType(TypeRepr.of[Struct], refinements) + + val expectedConfigTypeRepr = schemaToTypeRepr(schema) + val obtainedTypeRepr = TypeRepr.of[C] + + val dealiasedExpectedConfigTypeRepr = dealiasAll(expectedConfigTypeRepr) + val dealiasedObtainedTypeRepr = dealiasAll(obtainedTypeRepr) + + val expectedTypeString = dealiasedExpectedConfigTypeRepr.show + val obtainedTypeString = dealiasedObtainedTypeRepr.show + + val expectedConfigType = expectedConfigTypeRepr.asType + + expectedConfigType match + case '[t] => + type Expected = t + + configuration match + case '{ $c: Expected } => + Right(()) + case '{ $c: cType } => + val prettyExpected = prettifyTypeString(expectedTypeString) + val expected = Tpe.parseType(prettyExpected).map(_.stripOutputs) match + case Left(ex) => throw ex + case Right(st @ Tpe.Struct(_)) => st.toValMap + case Right(_) => ??? // TODO should not happen, top levels are always structs + + val prettyObtained = prettifyTypeString(obtainedTypeString) + val obtained = Tpe.parseType(prettyObtained).map(_.stripOutputs) match + case Left(ex) => throw ex + case Right(st @ Tpe.Struct(_)) => st.toValMap + case Right(_) => ??? // TODO should not happen, top levels are always structs + + val prettyDiff = diff( + obtained = obtained, + expected = expected + ) + + Left(prettyDiff) + end performDiff +end Diff diff --git a/besom-cfg/k8s/src/main/scala/typeparser.scala b/besom-cfg/k8s/src/main/scala/typeparser.scala new file mode 100644 index 00000000..2711662b --- /dev/null +++ b/besom-cfg/k8s/src/main/scala/typeparser.scala @@ -0,0 +1,60 @@ +package besom.cfg + +sealed trait Tpe: + def stripOutputs: Tpe = this match + case Tpe.Output(inner) => inner.stripOutputs + case Tpe.Struct(fields) => Tpe.Struct(fields.map((k, v) => (k, v.stripOutputs))) + case Tpe.Lst(inner) => Tpe.Lst(inner.stripOutputs) + case Tpe.Union(types) => + val unionTypes = types.map(_.stripOutputs).distinct + if unionTypes.length == 1 then unionTypes.head + else Tpe.Union(unionTypes) + case Tpe.Simple(name) => Tpe.Simple(name) + +object Tpe: + case class Simple(name: String) extends Tpe + case class Lst(inner: Tpe) extends Tpe + case class Union(types: List[Tpe]) extends Tpe + case class Output(inner: Tpe) extends Tpe + case class Struct(fields: List[(String, Tpe)]) extends Tpe: + def keys: List[String] = fields.map(_._1) + def get(key: String): Option[Tpe] = fields.find(_._1 == key).map(_._2) + + // any list that isn't list of structs is a Val.Str with internal type interpolated + // any struct is a Val.Map + // any list of structs is a Val.List of Val.Map + + // we assume this is after stripping outputs + def toValMap: ValMap = + fields.foldLeft(ValMap.empty) { case (acc, (k, v)) => + acc.updated( + k, + v match + case Tpe.Simple(name) => Val.Str(name) + case Tpe.Lst(inner: Tpe.Struct) => Val.List(Val.Map(inner.toValMap)) + case Tpe.Lst(inner: Tpe.Simple) => Val.Str(s"List[${inner.name}]") + case Tpe.Union(inner :: Tpe.Simple("Null") :: Nil) => throw Exception("Options are not supported yet") + case Tpe.Lst(_) => ??? // should not happen + case Tpe.Union(types) => ??? // should not happen, there are no instances of ConfiguredType for unions + case Tpe.Output(inner) => ??? // should not happen + case s: Tpe.Struct => Val.Map(s.toValMap) + ) + } + + import fastparse._, MultiLineWhitespace._ + + def ident[$: P]: P[String] = P(CharIn("a-zA-Z_") ~ CharsWhileIn("a-zA-Z0-9_", 0)).! + def simpleType[$: P]: P[Tpe] = P((ident ~ !("[" | ".")).map(Tpe.Simple.apply)) + def structType[$: P]: P[Tpe] = P("{" ~ field.rep(0) ~ "}").map(_.toList).map(Tpe.Struct.apply) + def field[$: P]: P[(String, Tpe)] = P(ident ~ ":" ~ anyType) + def anyType[$: P]: P[Tpe] = P(unionType | outputType | structType | lstType | simpleType) + def nonUnionAnyType[$: P]: P[Tpe] = P(outputType | structType | lstType | simpleType) + def unionType[$: P]: P[Tpe] = P(nonUnionAnyType.rep(2, "|")).map(_.toList).map(Tpe.Union.apply) + def lstType[$: P]: P[Tpe] = P("List[" ~ anyType ~ "]").map(Tpe.Lst.apply) + def outputType[$: P]: P[Tpe] = P("Output[" ~ anyType ~ "]").map(Tpe.Output.apply) + + def parseType(input: String): Either[Exception, Tpe] = + parse(input, anyType(_)) match + case Parsed.Success(value, _) => Right(value) + case f: Parsed.Failure => Left(Exception(f.trace().longMsg)) +end Tpe diff --git a/besom-cfg/k8s/src/test/resources/expected-diff.txt b/besom-cfg/k8s/src/test/resources/expected-diff.txt new file mode 100644 index 00000000..7399f4f4 --- /dev/null +++ b/besom-cfg/k8s/src/test/resources/expected-diff.txt @@ -0,0 +1,55 @@ +{ + shouldBeIntAndItsOK: Int + shouldBeStringButItsNot: got Double, expected String + shouldBeBoolButItsStruct: got { + x: String + }, expected Boolean + shouldBeStructButItsNot: got String, expected { + a: Int + b: String + } + oneFieldInsideIsWrongAndOneIsUnnecessary: { + x: Int + y: got Int, expected Double + z: Int + } + wholeStructMissing: { + q: Int + w: String + } + shouldBeListOfInts: got List[Double], expected List[Int] + shouldBeListOfStructs: got String, expected List[{ + a: Int + b: String + }] + shouldBeListOfStructsButItsAStruct: got { + a: Int + b: Int + }, expected List[{ + x: Int + y: Double + }] + shouldBeAStructButItsAListOfStructs: got List[{ + one: String + }], expected { + x: Int + y: Double + } + shouldBeAStringButItsAListOfStructs: got List[{ + one: Int + }], expected String + shouldBeAListOfStructsButItsAString: got String, expected List[{ + a: Int + b: String + }] + thisOneIsUnnecessary: String + unnecessaryListOfStructs: List[{ + a: Int + b: Int + }] + unnecessaryStruct: { + one: String + two: String + three: String + } +} \ No newline at end of file diff --git a/besom-cfg/k8s/src/test/scala/DummyContext.scala b/besom-cfg/k8s/src/test/scala/DummyContext.scala new file mode 100644 index 00000000..f695db3c --- /dev/null +++ b/besom-cfg/k8s/src/test/scala/DummyContext.scala @@ -0,0 +1,58 @@ +package besom.internal + +import pulumirpc.resource.* +import pulumirpc.provider.CallRequest +import pulumirpc.provider.CallResponse +import pulumirpc.provider.InvokeResponse +import pulumirpc.engine.* +import besom.NonEmptyString +import besom.internal.logging.BesomLogger + +object DummyContext: + val dummyRunInfo = RunInfo(Some("test-organization"), "test-project", "test-stack", true, 4, false, "dummy", "dummy") + val dummyFeatureSupport = FeatureSupport(true, true, true, true) + val dummyMonitor = new Monitor: + def call(callRequest: CallRequest): Result[CallResponse] = + Result.fail(Exception("Not implemented")) + def invoke(invokeRequest: ResourceInvokeRequest): Result[InvokeResponse] = + Result.fail(Exception("Not implemented")) + def readResource(readResourceRequest: ReadResourceRequest): Result[ReadResourceResponse] = + Result.fail(Exception("Not implemented")) + def registerResource(registerResourceRequest: RegisterResourceRequest): Result[RegisterResourceResponse] = + Result.fail(Exception(s"Not implemented\n${pprint.apply(registerResourceRequest)}")) + def registerResourceOutputs(registerResourceOutputsRequest: RegisterResourceOutputsRequest): Result[Unit] = + Result.fail(Exception("Not implemented")) + def supportsFeature(supportsFeatureRequest: SupportsFeatureRequest): Result[SupportsFeatureResponse] = + Result.fail(Exception("Not implemented")) + def close(): Result[Unit] = Result.fail(Exception("Not implemented")) + + val dummyEngine = new Engine: + def getRootResource(getRootResource: GetRootResourceRequest): Result[GetRootResourceResponse] = + Result.fail(Exception("Not implemented")) + def setRootResource(setRootResource: SetRootResourceRequest): Result[SetRootResourceResponse] = + Result.fail(Exception("Not implemented")) + def log(logRequest: LogRequest): Result[Unit] = + Result.fail(Exception("Not implemented")) + def close(): Result[Unit] = + Result.fail(Exception("Not implemented")) + + def apply( + runInfo: RunInfo = dummyRunInfo, + featureSupport: FeatureSupport = dummyFeatureSupport, + monitor: Monitor = dummyMonitor, + engine: Engine = dummyEngine, + configMap: Map[NonEmptyString, String] = Map.empty, + configSecretKeys: Set[NonEmptyString] = Set.empty + ): Result[Context] = + for + taskTracker <- TaskTracker() + stackPromise <- Promise[StackResource]() + logger <- BesomLogger.local() + memo <- Memo() + config <- Config(runInfo.project, isProjectName = true, configMap = configMap, configSecretKeys = configSecretKeys) + resources <- Resources() + given Context = Context.create(runInfo, featureSupport, config, logger, monitor, engine, taskTracker, resources, memo, stackPromise) + _ <- stackPromise.fulfill(StackResource()(using ComponentBase(Output(besom.types.URN.empty)))) + yield summon[Context] + +end DummyContext diff --git a/besom-cfg/k8s/src/test/scala/ErrorsSupport.scala b/besom-cfg/k8s/src/test/scala/ErrorsSupport.scala new file mode 100644 index 00000000..c7bcf30d --- /dev/null +++ b/besom-cfg/k8s/src/test/scala/ErrorsSupport.scala @@ -0,0 +1,26 @@ +package besom.cfg + +import besom.json.* +import besom.cfg.internal.* +import scala.quoted.* + +object ErrorsSupport: + inline def apply[C <: Struct](schemaStr: String, configuration: C): Either[fansi.Str, Unit] = ${ + callPerformDiff('schemaStr, 'configuration) + } + + def callPerformDiff[C <: Struct: Type](schemaStr: Expr[String], configuration: Expr[C])(using Quotes): Expr[Either[fansi.Str, Unit]] = + import quotes.reflect.* + + given ToExpr[fansi.Str] with + def apply(str: fansi.Str)(using Quotes): Expr[fansi.Str] = '{ fansi.Str(${ Expr(str.toString) }) } + + val schema = schemaStr.value match + case None => report.errorAndAbort("Schema has to be a literal!", schemaStr) + case Some(schemaJson) => summon[JsonFormat[Schema]].read(schemaJson.parseJson) + + val eitherDiffOrUnit = Diff.performDiff(schema, configuration) + + eitherDiffOrUnit match + case Left(diff) => '{ Left(${ Expr(diff) }) } + case Right(()) => '{ Right(()) } diff --git a/besom-cfg/k8s/src/test/scala/ErrorsTest.scala b/besom-cfg/k8s/src/test/scala/ErrorsTest.scala new file mode 100644 index 00000000..3e053908 --- /dev/null +++ b/besom-cfg/k8s/src/test/scala/ErrorsTest.scala @@ -0,0 +1,219 @@ +package besom.cfg + +import besom.json.* +import besom.cfg.internal.* +import besom.types.{Context, Output} +import besom.internal.DummyContext +import besom.internal.RunOutput.* + +class ErrorsTest extends munit.FunSuite: + + given Context = DummyContext().unsafeRunSync() + + test("should fail on wrong type") { + val struct = Struct( + shouldBeIntAndItsOK = 1, + shouldBeStringButItsNot = 23.0, + shouldBeBoolButItsStruct = Struct( + x = "y" + ), + thisOneIsUnnecessary = "not needed", + shouldBeStructButItsNot = "it's a string", + shouldBeListOfInts = List(1.23, 2, 3), // but it's a list of doubles + oneFieldInsideIsWrongAndOneIsUnnecessary = Struct( + x = 10, + y = 10, + z = 10 + ), + shouldBeListOfStructs = "but it isn't", + shouldBeListOfStructsButItsAStruct = Struct( + a = 1, + b = 2 + ), + shouldBeAStructButItsAListOfStructs = List( + Struct( + one = "no" + ) + ), + shouldBeAStringButItsAListOfStructs = List( + Struct( + one = 1 + ) + ), + shouldBeAListOfStructsButItsAString = "oops", + unnecessaryListOfStructs = List( + Struct( + a = 1, + b = 2 + ) + ), + unnecessaryStruct = Struct( + one = "no", + two = "not", + three = "needed" + ) + ) + + ErrorsSupport( + """{ + "schema": [ + { + "details": { + "type": "int" + }, + "name": "shouldBeIntAndItsOK" + }, + { + "details": { + "type": "string" + }, + "name": "shouldBeStringButItsNot" + }, + { + "details": { + "type": "boolean" + }, + "name": "shouldBeBoolButItsStruct" + }, + { + "details": { + "fields": { + "a": { + "type": "int" + }, + "b": { + "type": "string" + } + }, + "type": "struct" + }, + "name": "shouldBeStructButItsNot" + }, + { + "details": { + "fields": { + "x": { + "type": "int" + }, + "y": { + "type": "double" + } + }, + "type": "struct" + }, + "name": "oneFieldInsideIsWrongAndOneIsUnnecessary" + }, + { + "details": { + "fields": { + "q": { + "type": "int" + }, + "w": { + "type": "string" + } + }, + "type": "struct" + }, + "name": "wholeStructMissing" + }, + { + "details": { + "inner": { + "type": "int" + }, + "type": "array" + }, + "name": "shouldBeListOfInts" + }, + { + "details": { + "inner": { + "fields": { + "a": { + "type": "int" + }, + "b": { + "type": "string" + } + }, + "type": "struct" + }, + "type": "array" + }, + "name": "shouldBeListOfStructs" + }, + { + "details": { + "inner": { + "fields": { + "x": { + "type": "int" + }, + "y": { + "type": "double" + } + }, + "type": "struct" + }, + "type": "array" + }, + "name": "shouldBeListOfStructsButItsAStruct" + }, + { + "details": { + "fields": { + "x": { + "type": "int" + }, + "y": { + "type": "double" + } + }, + "type": "struct" + }, + "name": "shouldBeAStructButItsAListOfStructs" + }, + { + "details": { + "type": "string" + }, + "name": "shouldBeAStringButItsAListOfStructs" + }, + { + "details": { + "inner": { + "fields": { + "a": { + "type": "int" + }, + "b": { + "type": "string" + } + }, + "type": "struct" + }, + "type": "array" + }, + "name": "shouldBeAListOfStructsButItsAString" + } + ], + "version": "0.1.0" +}""", + struct + ) match + case Left(diff) => + val renderedDiff = diff.toString + + assertEquals(renderedDiff, expected) + case Right(()) => fail("should fail") + end match + + } + + private val expected: String = + val src = scala.io.Source.fromResource("expected-diff.txt") + try src.mkString + finally src.close() + +end ErrorsTest diff --git a/besom-cfg/k8s/src/test/scala/RunOutput.scala b/besom-cfg/k8s/src/test/scala/RunOutput.scala new file mode 100644 index 00000000..372de734 --- /dev/null +++ b/besom-cfg/k8s/src/test/scala/RunOutput.scala @@ -0,0 +1,13 @@ +package besom.internal + +import scala.concurrent.Future + +trait RunOutput[F[+_]]: + def run[A](output: Output[A]): Option[A] + +object RunOutput: + given runOutputForFuture(using RunResult[Future]): RunOutput[Future] = new RunOutput[Future]: + def run[A](output: Output[A]): Option[A] = RunResult.unsafeRunSync(output.getData)().getValue + + extension [F[+_], A](output: Output[A])(using ro: RunOutput[F]) def unsafeRunSync(): Option[A] = ro.run(output) + extension [F[+_], A](result: Result[A])(using rr: RunResult[F]) def unsafeRunSync(): A = rr.run(result) diff --git a/besom-cfg/k8s/src/test/scala/RunResult.scala b/besom-cfg/k8s/src/test/scala/RunResult.scala new file mode 100644 index 00000000..e2776687 --- /dev/null +++ b/besom-cfg/k8s/src/test/scala/RunResult.scala @@ -0,0 +1,21 @@ +package besom.internal + +import scala.concurrent._, duration._ + +trait RunResult[F[+_]]: + def run[A](result: Result[A]): A + +object RunResult: + given RunResult[Future] = new RunResult[Future]: + given ExecutionContext = ExecutionContext.fromExecutorService( + null, // FJP does seem to swallow fatals + (t: Throwable) => + // TODO this has to contain a link to github issue tracker to allow user to easily create a bug report, this is EXTREMELY IMPORTANT + scribe.error("Uncaught fatal error in Future Runtime", t) + t.printStackTrace() + sys.exit(1) + ) + given Runtime[Future] = FutureRuntime() + def run[A](result: Result[A]): A = Await.result(result.run, Duration.Inf) + + extension [F[+_], A](result: Result[A])(using rr: RunResult[F]) def unsafeRunSync(): A = rr.run(result) diff --git a/besom-cfg/k8s/src/test/scala/StructSerdeTest.scala b/besom-cfg/k8s/src/test/scala/StructSerdeTest.scala new file mode 100644 index 00000000..b4ab580a --- /dev/null +++ b/besom-cfg/k8s/src/test/scala/StructSerdeTest.scala @@ -0,0 +1,127 @@ +package besom.cfg + +import besom.cfg.k8s.* +import besom.cfg.* +import besom.internal.DummyContext +import besom.internal.RunOutput.* + +import besom.types.{Context, Output} +import besom.api.kubernetes.core.v1.inputs.EnvVarArgs + +class StructSerdeTest extends munit.FunSuite: + // ConfiguredContainerArgs( + // name = "my-app", + // image = "my-app:0.1", + // Struct( + // shouldBeIntAndItsOK = 1, + // shouldBeStringButItsNot = 23.0, + // shouldBeBoolButItsStruct = Struct( + // x = "y" + // ), + // thisOneIsUnnecessary = "not needed", + // shouldBeStructButItsNot = "it's a string", + // shouldBeListOfInts = List(1.23, 2, 3), // but it's a list of doubles + // oneFieldInsideIsWrongAndOneIsUnnecessary = Struct( + // x = 10, + // y = 10, + // z = 10 + // ), + // shouldBeListOfStructs = "but it isn't", + // shouldBeListOfStructsButItsAStruct = Struct( + // a = 1, + // b = 2 + // ), + // shouldBeAStructButItsAListOfStructs = List( + // Struct( + // one = "no" + // ) + // ), + // shouldBeAStringButItsAListOfStructs = List( + // Struct( + // one = 1 + // ) + // ), + // shouldBeAListOfStructsButItsAString = "oops", + // unnecessaryListOfStructs = List( + // Struct( + // a = 1, + // b = 2 + // ) + // ), + // unnecessaryStruct = Struct( + // one = "no", + // two = "not", + // three = "needed" + // ) + // ) + // ) + + given Context = DummyContext().unsafeRunSync() + + test("serialization of Structs to kubernetes EnvVarArgs") { + + // values can be: simple types, Output, Struct, List and their combinations without Lists of Lists + val s = Struct( + name = "test", // simple type + int = Output(23), // Output[simple] + s = Struct( // Struct + d = 23, + e = "test" + ), + l = List(1.2, 2.3, 3.4), // List[simple] + ls = List( // List[Struct] + Struct( + f1 = Output(List(Struct(deep = "q"))), // List[Struct->Output[List[Struct]]] + f2 = "w" + ) + ), + lo = List(Output("a"), Output("b"), Output("c")), // List[Output[simple]] + ol = Output(List("x", "y", "z")), // Output[List[simple]] + os = Output( // Output[Struct] + Struct( + oh = "yeah", + it = "works!" + ) + ) + ) + + val expected = List( + "name" -> "test", + "int" -> "23", + "s.d" -> "23", + "s.e" -> "test", + "l.0" -> "1.2", + "l.1" -> "2.3", + "l.2" -> "3.4", + "ls.0.f1.0.deep" -> "q", + "ls.0.f2" -> "w", + "lo.0" -> "a", + "lo.1" -> "b", + "lo.2" -> "c", + "ol.0" -> "x", + "ol.1" -> "y", + "ol.2" -> "z", + "os.oh" -> "yeah", + "os.it" -> "works!" + ) + + val asEnv = s.foldToEnv + + asEnv.unsafeRunSync().get.zip(expected).foreach((a, b) => assertEquals(a, b)) + + import besom.cfg.k8s.syntax.* + val asEnvVarArgs = s.foldedToEnvVarArgs + + asEnvVarArgs + .unsafeRunSync() + .get + .map { case EnvVarArgs(name, value, _) => + val k = name.unsafeRunSync().get + val v = value.unsafeRunSync().get.get + + k -> v + } + .zip(expected) + .foreach((a, b) => assertEquals(a, b)) + } +end StructSerdeTest diff --git a/besom-cfg/k8s/src/test/scala/TypeParserTest.scala b/besom-cfg/k8s/src/test/scala/TypeParserTest.scala new file mode 100644 index 00000000..8dea5c6e --- /dev/null +++ b/besom-cfg/k8s/src/test/scala/TypeParserTest.scala @@ -0,0 +1,239 @@ +package besom.cfg + +import fastparse._, MultiLineWhitespace._ + +class TypeParserTest extends munit.FunSuite: + + import Tpe.* + + test("should parse simple type") { + parse("a: Int", field(_)) match + case Parsed.Success(value, _) => assertEquals(value, ("a", Tpe.Simple("Int"))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse union of simple types") { + parse("Int | String", anyType(_)) match + case Parsed.Success(value, _) => assertEquals(value, Tpe.Union(List(Tpe.Simple("Int"), Tpe.Simple("String")))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse union of simple type and Output type") { + parse("Int | Output[Int]", anyType(_)) match + case Parsed.Success(value, _) => assertEquals(value, Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int"))))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse list of simple type") { + parse("List[Int]", anyType(_)) match + case Parsed.Success(value, _) => assertEquals(value, Tpe.Lst(Tpe.Simple("Int"))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse an empty struct") { + parse("{}", structType(_)) match + case Parsed.Success(value, _) => assertEquals(value, Tpe.Struct(List.empty)) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse a simple struct type") { + parse( + """{ + | a: Int + |}""".stripMargin, + structType(_) + ) match + case Parsed.Success(value, _) => + assertEquals(value, Tpe.Struct(List(("a", Tpe.Simple("Int"))))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse a struct type with union") { + parse( + """{ + | a: Int | String + |}""".stripMargin, + structType(_) + ) match + case Parsed.Success(value, _) => + assertEquals(value, Tpe.Struct(List(("a", Tpe.Union(List(Tpe.Simple("Int"), Tpe.Simple("String"))))))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse an output type of struct") { + parse( + """Output[{ + | a: Int + |}]""".stripMargin, + outputType(_) + ) match + case Parsed.Success(value, _) => + assertEquals(value, Tpe.Output(Tpe.Struct(List(("a", Tpe.Simple("Int")))))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse an union of simple type and output of simple type") { + parse( + """List[Int | Output[Int]] | List[Output[Int | Output[Int]]]""", + anyType(_) + ) match + case Parsed.Success(value, _) => + assertEquals( + value, + Tpe.Union( + List( + Tpe.Lst(Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int"))))), + Tpe.Lst(Tpe.Output(Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int")))))) + ) + ) + ) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse an union of struct and output of struct") { + parse( + """shouldBeStructButItsNot: { + | a: Int | Output[Int] + | b: String | Output[String] + |} | Output[{ + | a: Int | Output[Int] + | b: String | Output[String] + |}]""".stripMargin, + field(_) + ) match + case Parsed.Success(value, _) => + assertEquals( + value, + ( + "shouldBeStructButItsNot", + Tpe.Union( + List( + Tpe.Struct( + List( + ("a", Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int"))))), + ("b", Tpe.Union(List(Tpe.Simple("String"), Tpe.Output(Tpe.Simple("String"))))) + ) + ), + Tpe.Output( + Tpe.Struct( + List( + ("a", Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int"))))), + ("b", Tpe.Union(List(Tpe.Simple("String"), Tpe.Output(Tpe.Simple("String"))))) + ) + ) + ) + ) + ) + ) + ) + case f: Parsed.Failure => fail(f.trace().longAggregateMsg) + } + + test("should parse simple field") { + parse("shouldBeIntAndItsOK: Int | Output[Int]", field(_)) match + case Parsed.Success(value, _) => + assertEquals(value, ("shouldBeIntAndItsOK", Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int")))))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse union field") { + parse("shouldBeStringButItsNot: String | Output[String]", field(_)) match + case Parsed.Success(value, _) => + assertEquals(value, ("shouldBeStringButItsNot", Tpe.Union(List(Tpe.Simple("String"), Tpe.Output(Tpe.Simple("String")))))) + case f: Parsed.Failure => fail(f.trace().longMsg) + } + + test("should parse very complex unions") { + val union = """shouldBeListOfStructs: List[{ + a: Int | Output[Int] + b: String | Output[String] + } | Output[{ + a: Int | Output[Int] + b: String | Output[String] + }]] | List[Output[{ + a: Int | Output[Int] + b: String | Output[String] + } | Output[{ + a: Int | Output[Int] + b: String | Output[String] + }]]]""" + parse(union, field(_)) match + case Parsed.Success(value, _) => + assertEquals( + value, + ( + "shouldBeListOfStructs", + Tpe.Union( + List( + Tpe.Lst( + Tpe.Union( + List( + Tpe.Struct( + List( + ("a", Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int"))))), + ("b", Tpe.Union(List(Tpe.Simple("String"), Tpe.Output(Tpe.Simple("String"))))) + ) + ), + Tpe.Output( + Tpe.Struct( + List( + ("a", Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int"))))), + ("b", Tpe.Union(List(Tpe.Simple("String"), Tpe.Output(Tpe.Simple("String"))))) + ) + ) + ) + ) + ) + ), + Tpe.Lst( + Tpe.Output( + Tpe.Union( + List( + Tpe.Struct( + List( + ("a", Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int"))))), + ("b", Tpe.Union(List(Tpe.Simple("String"), Tpe.Output(Tpe.Simple("String"))))) + ) + ), + Tpe.Output( + Tpe.Struct( + List( + ("a", Tpe.Union(List(Tpe.Simple("Int"), Tpe.Output(Tpe.Simple("Int"))))), + ("b", Tpe.Union(List(Tpe.Simple("String"), Tpe.Output(Tpe.Simple("String"))))) + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) + case f: Parsed.Failure => fail(f.trace().longMsg) + end match + } + + test("should strip outputs from types correctly") { + val union = """shouldBeListOfStructs: List[{ + a: Int | Output[Int] + b: String | Output[String] + } | Output[{ + a: Int | Output[Int] + b: String | Output[String] + }]] | List[Output[{ + a: Int | Output[Int] + b: String | Output[String] + } | Output[{ + a: Int | Output[Int] + b: String | Output[String] + }]]]""" + + parse(union, field(_)) match + case Parsed.Success((_, tpe), _) => + assertEquals(tpe.stripOutputs, Tpe.Lst(Tpe.Struct(List("a" -> Tpe.Simple("Int"), "b" -> Tpe.Simple("String"))))) + + case f: Parsed.Failure => fail(f.trace().longMsg) + } +end TypeParserTest diff --git a/besom-cfg/lib/.scalafmt.conf b/besom-cfg/lib/.scalafmt.conf new file mode 100644 index 00000000..f4d4b655 --- /dev/null +++ b/besom-cfg/lib/.scalafmt.conf @@ -0,0 +1,11 @@ +version = 3.5.2 +runner.dialect = scala3 +project.git = true +align = most +align.openParenCallSite = false +align.openParenDefnSite = false +align.tokens = [{code = "=>", owner = "Case"}, "<-", "%", "%%", "="] +indent.defnSite = 2 +maxColumn = 140 + +rewrite.scala3.insertEndMarkerMinLines = 40 \ No newline at end of file diff --git a/besom-cfg/lib/project.scala b/besom-cfg/lib/project.scala new file mode 100644 index 00000000..dd77dc08 --- /dev/null +++ b/besom-cfg/lib/project.scala @@ -0,0 +1,18 @@ +//> using scala 3.3.3 + +//> using dep org.virtuslab::besom-core::0.4.0-SNAPSHOT +//> using dep org.virtuslab::besom-json::0.4.0-SNAPSHOT +//> using dep io.github.classgraph:classgraph:4.8.172 + +//> using test.dep org.scalameta::munit::1.0.0-M11 + +//> using publish.name "besom-cfg" +//> using publish.organization "org.virtuslab" +//> using publish.url "https://github.com/VirtusLab/besom" +//> using publish.vcs "github:VirtusLab/besom" +//> using publish.license "Apache-2.0" +//> using publish.repository "central" +//> using publish.developer "lbialy|Łukasz Biały|https://github.com/lbialy" +//> using publish.developer "prolativ|Michał Pałka|https://github.com/prolativ" +//> using publish.developer "KacperFKorban|Kacper Korban|https://github.com/KacperFKorban" +//> using publish.developer "pawelprazak|Paweł Prażak|https://github.com/pawelprazak" diff --git a/besom-cfg/lib/src/main/scala/Configured.scala b/besom-cfg/lib/src/main/scala/Configured.scala new file mode 100644 index 00000000..196662e2 --- /dev/null +++ b/besom-cfg/lib/src/main/scala/Configured.scala @@ -0,0 +1,100 @@ +package besom.cfg + +import scala.quoted.* +import besom.json.* +import besom.cfg.internal.* + +// trait Constraint[A]: +// def validate(a: A): Boolean +// def &(other: Constraint[A]): Constraint[A] = +// (a: A) => validate(a) && other.validate(a) +// def |(other: Constraint[A]): Constraint[A] = +// (a: A) => validate(a) || other.validate(a) + +// object Positive extends Constraint[Int]: +// def validate(a: Int): Boolean = a > 0 + +// object Negative extends Constraint[Int]: +// def validate(a: Int): Boolean = a < 0 + +// object NonZero extends Constraint[Int]: +// def validate(a: Int): Boolean = a != 0 + +// object NonEmpty extends Constraint[String]: +// def validate(a: String): Boolean = a.nonEmpty + +// object NonBlank extends Constraint[String]: +// def validate(a: String): Boolean = a.trim.nonEmpty + +// trait FromEnv[A]: +// def fromEnv(parentKey: String, selected: Map[String, String]): A + +// object StringFromEnv extends FromEnv[String]: +// def fromEnv(parentKey: String, selected: Map[String, String]): String = +// selected.getOrElse(parentKey, throw new Exception(s"Key $parentKey not found in env")) + +// given ListFromEnv[A](using FromEnv[A]): FromEnv[List[A]] with +// def fromEnv(parentKey: String, selected: Map[String, String]): List[A] = +// val prefix = s"$parentKey." +// val subselected = selected.filter(_._1.startsWith(prefix)) +// subselected.keys +// .map { k => +// val index = k.stripPrefix(prefix) +// val value = summon[FromEnv[A]].fromEnv(k, selected) +// index.toInt -> value +// } +// .toList +// .sortBy(_._1) +// .map(_._2) + +trait Configured[A]: + def schema: Schema + def newInstanceFromEnv(env: Map[String, String] = sys.env): A + +object Configured: + val Version = "0.1.0" + + inline def derived[A <: Product]: Configured[A] = ${ derivedImpl[A] } + + def derivedImpl[A <: Product: Type](using ctx: Quotes): Expr[Configured[A]] = + import ctx.reflect.* + + val tpe = TypeRepr.of[A] + val fields = tpe.typeSymbol.caseFields.map { case sym => + val name = Expr(sym.name) + val ftpe: Expr[ConfiguredType[_]] = + tpe.memberType(sym).dealias.asType match + case '[t] => + Expr.summon[ConfiguredType[t]].getOrElse { + report.error( + s"Cannot find ConfiguredType for type ${tpe.memberType(sym).dealias.show}" + ) + throw new Exception("Cannot find ConfiguredType") + } + case _ => + report.error("Unsupported type") + throw new Exception("Unsupported type") + + '{ Field(${ name }, ${ ftpe }.toFieldType) } + } + + val fromEnvExpr = Expr.summon[FromEnv[A]].getOrElse { + report.error(s"Cannot find FromEnv for type ${tpe.show}") + throw new Exception("Cannot find FromEnv") + } + + val schemaExpr = '{ Schema(${ Expr.ofList(fields) }.toList, ${ Expr(Version) }) } + + '{ + new Configured[A] { + def schema = $schemaExpr + def newInstanceFromEnv(env: Map[String, String] = sys.env): A = + $fromEnvExpr.decode(env, "").getOrElse { + throw new Exception("Failed to decode") + } + } + } +end Configured + +def resolveConfiguration[A](using c: Configured[A]): A = + c.newInstanceFromEnv() diff --git a/besom-cfg/lib/src/main/scala/FromEnv.scala b/besom-cfg/lib/src/main/scala/FromEnv.scala new file mode 100644 index 00000000..b43a87d4 --- /dev/null +++ b/besom-cfg/lib/src/main/scala/FromEnv.scala @@ -0,0 +1,87 @@ +package besom.cfg + +// TODO do not use Option[T], use something with a proper error channel and missing value channel +// TODO rationale: if a value is provided but it's not valid (e.g. empty string for an Int), we want to know +// TODO if a value is missing, but the type is optional in configuration, that's fine +trait FromEnv[A]: + def decode(env: Map[String, String], path: String): Option[A] + +object FromEnv: + + import scala.deriving.* + import scala.compiletime.{erasedValue, summonInline} + + inline def summonAllInstances[T <: Tuple]: List[FromEnv[?]] = + inline erasedValue[T] match + case _: (t *: ts) => summonInline[FromEnv[t]] :: summonAllInstances[ts] + case _: EmptyTuple => Nil + + inline def summonLabels[T <: Tuple]: List[String] = + inline erasedValue[T] match + case _: EmptyTuple => Nil + case _: (t *: ts) => + summonInline[ValueOf[t]].value.asInstanceOf[String] :: summonLabels[ts] + + given [A: FromEnv]: FromEnv[Option[A]] with + def decode(env: Map[String, String], path: String): Option[Option[A]] = + Some(summon[FromEnv[A]].decode(env, path)) + + given FromEnv[Int] with + def decode(env: Map[String, String], path: String): Option[Int] = + env.get(path).flatMap(s => scala.util.Try(s.toInt).toOption) + + given FromEnv[Long] with + def decode(env: Map[String, String], path: String): Option[Long] = + env.get(path).flatMap(s => scala.util.Try(s.toLong).toOption) + + given FromEnv[String] with + def decode(env: Map[String, String], path: String): Option[String] = + env.get(path) + + given FromEnv[Double] with + def decode(env: Map[String, String], path: String): Option[Double] = + env.get(path).flatMap(s => scala.util.Try(s.toDouble).toOption) + + given FromEnv[Float] with + def decode(env: Map[String, String], path: String): Option[Float] = + env.get(path).flatMap(s => scala.util.Try(s.toFloat).toOption) + + given FromEnv[Boolean] with + def decode(env: Map[String, String], path: String): Option[Boolean] = + env.get(path).flatMap(s => scala.util.Try(s.toBoolean).toOption) + + given [A: FromEnv]: FromEnv[List[A]] with + def decode(env: Map[String, String], path: String): Option[List[A]] = + Iterator.from(0).map(i => summon[FromEnv[A]].decode(env, s"$path.$i")).takeWhile(_.isDefined).toList.sequence + + given [A: FromEnv]: FromEnv[Vector[A]] with + def decode(env: Map[String, String], path: String): Option[Vector[A]] = + Iterator.from(0).map(i => summon[FromEnv[A]].decode(env, s"$path.$i")).takeWhile(_.isDefined).toVector.sequence + + inline given derived[A](using m: Mirror.ProductOf[A]): FromEnv[A] = new FromEnv[A]: + def decode(env: Map[String, String], path: String): Option[A] = + val elemDecoders = summonAllInstances[m.MirroredElemTypes] + val labels = summonLabels[m.MirroredElemLabels] + + val elemValues = elemDecoders.zip(labels).map { case (decoder, label) => + // handle top-level gracefully (empty path) + decoder.asInstanceOf[FromEnv[Any]].decode(env, if path.isBlank() then label else s"$path.$label") + } + + if elemValues.forall(_.isDefined) then Some(m.fromProduct(Tuple.fromArray(elemValues.flatten.toArray))) + else None + + // Helper to sequence a List[Option[A]] into Option[List[A]] + extension [A](xs: List[Option[A]]) + def sequence: Option[List[A]] = xs.foldRight(Option(List.empty[A])) { + case (Some(a), Some(acc)) => Some(a :: acc) + case _ => None + } + + // Helper to sequence a Vector[Option[A]] into Option[Vector[A]] + extension [A](xs: Vector[Option[A]]) + def sequence: Option[Vector[A]] = xs.foldLeft(Option(Vector.empty[A])) { + case (Some(acc), Some(a)) => Some(acc :+ a) + case _ => None + } +end FromEnv diff --git a/besom-cfg/lib/src/main/scala/Struct.scala b/besom-cfg/lib/src/main/scala/Struct.scala new file mode 100644 index 00000000..0a24cb5e --- /dev/null +++ b/besom-cfg/lib/src/main/scala/Struct.scala @@ -0,0 +1,119 @@ +package besom.cfg + +import scala.language.dynamics +import scala.quoted.* +import scala.collection.immutable.ListMap +import besom.cfg.internal.MetaUtils +import besom.types.{Output, Context} +import scala.util.chaining.* + +// possible types under `Any`: +// simple types: Int, Long, Float, Double, String, Boolean +// complex types: List[A], Struct +// any of the above can be wrapped in Output +class Struct private (val _values: ListMap[String, Any]) extends Selectable: + def selectDynamic(name: String) = _values(name) + + private[cfg] def fold[B]( + onStruct: Map[String, Output[B]] => Output[B], + onList: List[B] => Output[B], + onValue: Any => Output[B] + )(using Context): Output[B] = + val onOutput: Output[_] => Output[B] = _.flatMap { + case s: Struct => s.fold(onStruct, onList, onValue) + case a => onValue(a) + } + + val transformList: List[_] => Output[B] = l => { // v may be simple | Struct | Output, we don't support nested lists + val outputOfVec = l.foldLeft(Output(Vector.empty[B])) { case (acc, v) => + acc.flatMap { accVec => + val transformedV = v match + case s: Struct => s.fold(onStruct, onList, onValue) + case o: Output[_] => onOutput(o) + case a => onValue(a) + + transformedV.map(accVec :+ _) + } + } + + outputOfVec.map(_.toList).flatMap(onList) + } + + _values.view + .mapValues { + case s: Struct => s.fold[B](onStruct, onList, onValue) + case i: List[_] => transformList(i) + case o: Output[_] => // handle String -> Output[simple | Struct | List] + o.flatMap { + case s: Struct => s.fold[B](onStruct, onList, onValue) + case l: List[_] => transformList(l) + case a => onValue(a) + } + case a => onValue(a) + } + .to(ListMap) + .pipe(onStruct) + end fold +end Struct + +object Struct extends Dynamic: + def make(values: ListMap[String, Any]) = new Struct(values) + + inline def applyDynamic(apply: "apply")(): Struct = make(ListMap.empty) + + transparent inline def applyDynamicNamed(apply: "apply")(inline args: (String, Any)*): Struct = + ${ applyDynamicImpl('args) } + + def applyDynamicImpl(args: Expr[Seq[(String, Any)]])(using Quotes): Expr[Struct] = + import quotes.reflect.* + + type StructSubtype[T <: Struct] = T + + args match + case Varargs(argExprs) => + val refinementTypes = argExprs.toList.map { case '{ ($key: String, $value: v) } => + (key.valueOrAbort, TypeRepr.of[v]) + } + val exprs = argExprs.map { case '{ ($key: String, $value: v) } => + '{ ($key, $value) } + } + val argsExpr = Expr.ofSeq(exprs) + + MetaUtils.refineType(TypeRepr.of[Struct], refinementTypes).asType match + case '[StructSubtype[t]] => + '{ Struct.make(${ argsExpr }.to(ListMap)).asInstanceOf[t] } + + case _ => + report.errorAndAbort( + "Expected explicit varargs sequence. " + + "Notation `args*` is not supported.", + args + ) + + extension (s: Struct) + def foldToEnv(using Context): Output[List[(String, String)]] = s.fold[List[(String, String)]]( + onStruct = { mapB => + mapB.foldLeft(Output(List.empty[(String, String)])) { case (acc, (k, v)) => + acc.flatMap { accList => + v.map { vList => + accList ++ vList.map { case (k2, v2) => + // println(s"struct, serializing '$k' '$k2' to ${if k2.isBlank() then s"$k -> $v2" else s"$k.$k2 -> $v2"}") + if k2.isBlank then k -> v2 else s"$k.$k2" -> v2 + } + } + } + } + }, + onList = { list => + Output(list.zipWithIndex.flatMap { (lst, idx) => + lst.map { case (k, v) => + // println(s"list: serializing $k, $v to $k$idx -> $v") + if k.isBlank() then s"$k$idx" -> v else s"$idx.$k" -> v + } + }) + }, + onValue = a => + // println(s"serializing $a to List(\"\" -> $a)") + Output(List("" -> a.toString)) + ) +end Struct diff --git a/besom-cfg/lib/src/main/scala/SummonConfiguration.scala b/besom-cfg/lib/src/main/scala/SummonConfiguration.scala new file mode 100644 index 00000000..c4474d11 --- /dev/null +++ b/besom-cfg/lib/src/main/scala/SummonConfiguration.scala @@ -0,0 +1,78 @@ +package besom.cfg + +import io.github.classgraph.ClassGraph +import scala.util.Using +import scala.jdk.CollectionConverters.* +import besom.json.JsonWriter +import besom.cfg.internal.* + +object SummonConfiguration: + + private val toSkip = Set( + "org.scala-lang", + "io.github.classgraph", + "org.virtuslab.besom-cfg", // self + "org.virtuslab.besom-json" + ) + + def main(args: Array[String]): Unit = + def classPath = new ClassGraph() + .filterClasspathElements(path => toSkip.forall(segment => !path.contains(segment))) + .enableClassInfo() + .scan() + + Using(classPath) { scanResult => + val classes = scanResult + .getClassesImplementing(classOf[Configured[_]]) + .loadClasses() + .asScala + .toSet + + if classes.size > 1 then + throw Exception( + "Multiple Configured instances found! Only one per application is allowed." + ) + + if classes.isEmpty then + throw Exception( + "No Configured instances found! Exactly one is required." + ) + + val clazz = classes.head + + val maybeNoArgConst = clazz + .getDeclaredConstructors() + .filter(_.getParameterCount() == 0) + .headOption + + val maybeSingleArgConst = clazz + .getDeclaredConstructors() + .filter(_.getParameterCount() == 1) + .headOption + + val instance = maybeNoArgConst + .map(_.newInstance().asInstanceOf[Configured[_]]) + .getOrElse { + // this works with the assumption that user used `derives` to create the instance + // and therefore the class is placed in the companion object + maybeSingleArgConst + .map { ctor => + val moduleClass = ctor.getParameterTypes().head + val moduleField = + moduleClass.getDeclaredFields().find(_.getName == "MODULE$").get + + val outer = moduleField.get(null) + + ctor.newInstance(outer).asInstanceOf[Configured[_]] + } + .getOrElse { + throw Exception( + "No compatible constructor found for Configured instance!" + ) + } + } + + println(summon[JsonWriter[Schema]].write(instance.schema)) + }.get + end main +end SummonConfiguration diff --git a/besom-cfg/lib/src/main/scala/internal/ConfiguredType.scala b/besom-cfg/lib/src/main/scala/internal/ConfiguredType.scala new file mode 100644 index 00000000..b410d6bb --- /dev/null +++ b/besom-cfg/lib/src/main/scala/internal/ConfiguredType.scala @@ -0,0 +1,55 @@ +package besom.cfg.internal + +import scala.deriving.* +import scala.compiletime.* + +trait ConfiguredType[A]: + def toFieldType: FieldType + +object ConfiguredType: + inline def summonLabels[T <: Tuple]: List[String] = + inline erasedValue[T] match + case _: EmptyTuple => Nil + case _: (t *: ts) => + summonInline[ValueOf[t]].value.asInstanceOf[String] :: summonLabels[ts] + + inline def summonAllInstances[A <: Tuple]: List[ConfiguredType[_]] = + inline erasedValue[A] match + case _: EmptyTuple => Nil + case _: (t *: ts) => summonInline[ConfiguredType[t]] :: summonAllInstances[ts] + + given ConfiguredType[Int] with + def toFieldType = FieldType.Int + + given ConfiguredType[Long] with + def toFieldType = FieldType.Long + + given ConfiguredType[Float] with + def toFieldType = FieldType.Float + + given ConfiguredType[Double] with + def toFieldType = FieldType.Double + + given ConfiguredType[String] with + def toFieldType = FieldType.String + + given ConfiguredType[Boolean] with + def toFieldType = FieldType.Boolean + + given [A: ConfiguredType]: ConfiguredType[List[A]] with + def toFieldType = FieldType.Array(summon[ConfiguredType[A]].toFieldType) + + // support for Option is not yet implemented completely + // given [A: ConfiguredType]: ConfiguredType[Option[A]] with + // def toFieldType = FieldType.Optional(summon[ConfiguredType[A]].toFieldType) + + def buildConfiguredTypeFor[A](instances: => List[ConfiguredType[_]], labels: => List[String]): ConfiguredType[A] = + new ConfiguredType[A]: + def toFieldType = FieldType.Struct(labels.zip(instances.map(_.toFieldType)): _*) + + inline given derived[A <: Product](using m: Mirror.ProductOf[A]): ConfiguredType[A] = + lazy val elemInstances = summonAllInstances[m.MirroredElemTypes] + lazy val elemLabels = summonLabels[m.MirroredElemLabels] + buildConfiguredTypeFor[A](elemInstances, elemLabels) + +end ConfiguredType diff --git a/besom-cfg/lib/src/main/scala/internal/MetaUtils.scala b/besom-cfg/lib/src/main/scala/internal/MetaUtils.scala new file mode 100644 index 00000000..63fe7169 --- /dev/null +++ b/besom-cfg/lib/src/main/scala/internal/MetaUtils.scala @@ -0,0 +1,14 @@ +package besom.cfg.internal + +import scala.quoted.* + +object MetaUtils: + def refineType(using + Quotes + )(base: quotes.reflect.TypeRepr, refinements: List[(String, quotes.reflect.TypeRepr)]): quotes.reflect.TypeRepr = + import quotes.reflect.* + refinements match + case Nil => base + case (name, info) :: refinementsTail => + val newBase = Refinement(base, name, info) + refineType(newBase, refinementsTail) diff --git a/besom-cfg/lib/src/main/scala/internal/Schema.scala b/besom-cfg/lib/src/main/scala/internal/Schema.scala new file mode 100644 index 00000000..1256e34d --- /dev/null +++ b/besom-cfg/lib/src/main/scala/internal/Schema.scala @@ -0,0 +1,149 @@ +package besom.cfg.internal + +import scala.quoted.* +import besom.json.* + +enum FieldType: + case Int, Long, Float, Double, String, Boolean + case Array(inner: FieldType) + case Struct(fields: (String, FieldType)*) + case Optional(inner: FieldType) + +object FieldType: + given ToExpr[FieldType] with + def apply(fieldType: FieldType)(using Quotes): Expr[FieldType] = + import quotes.reflect.* + fieldType match + case FieldType.Int => '{ FieldType.Int } + case FieldType.Long => '{ FieldType.Long } + case FieldType.Float => '{ FieldType.Float } + case FieldType.Double => '{ FieldType.Double } + case FieldType.String => '{ FieldType.String } + case FieldType.Boolean => '{ FieldType.Boolean } + case FieldType.Array(inner) => '{ FieldType.Array(${ Expr(inner) }) } + case FieldType.Struct(fields: _*) => '{ FieldType.Struct(${ Expr(fields) }: _*) } + case FieldType.Optional(inner) => '{ FieldType.Optional(${ Expr(inner) }) } + + given FromExpr[FieldType] with + def unapply(expr: Expr[FieldType])(using Quotes): Option[FieldType] = + import quotes.reflect.* + expr match + case '{ FieldType.Int } => Some(FieldType.Int) + case '{ FieldType.Long } => Some(FieldType.Long) + case '{ FieldType.Float } => Some(FieldType.Float) + case '{ FieldType.Double } => Some(FieldType.Double) + case '{ FieldType.String } => Some(FieldType.String) + case '{ FieldType.Boolean } => Some(FieldType.Boolean) + case '{ FieldType.Array($inner) } => Some(FieldType.Array(inner.valueOrAbort)) + case '{ FieldType.Struct($fields: _*) } => Some(FieldType.Struct(fields.valueOrAbort: _*)) + case '{ FieldType.Optional($inner) } => Some(FieldType.Optional(inner.valueOrAbort)) + case _ => println("didn't match in FieldType"); None + + given JsonFormat[FieldType] with + def write(fieldType: FieldType): JsValue = fieldType match + case FieldType.Int => JsObject("type" -> JsString("int")) + case FieldType.Long => JsObject("type" -> JsString("long")) + case FieldType.Float => JsObject("type" -> JsString("float")) + case FieldType.Double => JsObject("type" -> JsString("double")) + case FieldType.String => JsObject("type" -> JsString("string")) + case FieldType.Boolean => JsObject("type" -> JsString("boolean")) + case FieldType.Array(inner) => JsObject("type" -> JsString("array"), "inner" -> write(inner)) + case FieldType.Struct(fields: _*) => + JsObject( + "type" -> JsString("struct"), + "fields" -> JsObject(fields.map { case (k, v) => k -> write(v) }.toMap) + ) + case FieldType.Optional(inner) => JsObject("type" -> JsString("optional"), "inner" -> write(inner)) + + def read(json: JsValue): FieldType = json match + case JsObject(fields) => + fields.get("type") match + case Some(JsString("int")) => FieldType.Int + case Some(JsString("long")) => FieldType.Long + case Some(JsString("float")) => FieldType.Float + case Some(JsString("double")) => FieldType.Double + case Some(JsString("string")) => FieldType.String + case Some(JsString("boolean")) => FieldType.Boolean + case Some(JsString("array")) => + fields.get("inner") match + case Some(inner) => FieldType.Array(read(inner)) + case _ => throw new Exception("Invalid JSON: array.inner must be present") + case Some(JsString("struct")) => + fields.get("fields") match + case Some(JsObject(innerFields)) => + val structFields = innerFields.map { case (k, v) => k -> read(v) } + FieldType.Struct(structFields.toVector: _*) + case None => throw new Exception("Invalid JSON: struct.fields must be present") + case _ => throw new Exception("Invalid JSON: struct.fields must be an object") + case Some(JsString("optional")) => + fields.get("inner") match + case Some(inner) => FieldType.Optional(read(inner)) + case _ => throw new Exception("Invalid JSON: optional.inner must be present") + case Some(what) => + throw new Exception(s"Invalid JSON: unknown type $what") + case None => throw new Exception("Invalid JSON: type must present") + case _ => throw new Exception("Invalid JSON: expected object") + end given +end FieldType + +case class Field(name: String, `type`: FieldType) +object Field: + given ToExpr[Field] with + def apply(field: Field)(using Quotes): Expr[Field] = + import quotes.reflect.* + '{ Field(${ Expr(field.name) }, ${ Expr(field.`type`) }) } + + given FromExpr[Field] with + def unapply(expr: Expr[Field])(using Quotes): Option[Field] = + import quotes.reflect.* + expr match + case '{ Field($name, $fieldType) } => Some(Field(name.valueOrAbort, fieldType.valueOrAbort)) + case _ => println("didn't match in Field"); None + + given fieldGiven(using fieldTypeWriter: JsonFormat[FieldType]): JsonFormat[Field] with + def write(field: Field): JsValue = + JsObject("name" -> JsString(field.name), "details" -> fieldTypeWriter.write(field.`type`)) + def read(json: JsValue): Field = + json match + case JsObject(fields) => + val name = fields.get("name") match + case Some(JsString(name)) => name + case _ => throw new Exception("Invalid JSON: field.name must be present") + val details = fields.get("details") match + case Some(details) => fieldTypeWriter.read(details) + case _ => throw new Exception("Invalid JSON: field.details must be present") + Field(name, details) + case _ => throw new Exception("Invalid JSON: expected object") + +case class Schema(fields: List[Field], version: String) +object Schema: + given ToExpr[Schema] with + def apply(schema: Schema)(using Quotes): Expr[Schema] = + import quotes.reflect.* + '{ Schema(${ Expr(schema.fields) }, ${ Expr(schema.version) }) } + + given FromExpr[Schema] with + def unapply(expr: Expr[Schema])(using Quotes): Option[Schema] = + import quotes.reflect.* + expr match + case '{ Schema($fields, $version) } => Some(Schema(fields.valueOrAbort, version.valueOrAbort)) + case _ => println("didn't match in Schema"); None + + given schemaGiven(using fieldWriter: JsonFormat[Field]): JsonFormat[Schema] with + def write(schema: Schema): JsValue = + JsObject( + "version" -> JsString(schema.version), + "schema" -> JsArray(schema.fields.map(field => fieldWriter.write(field)): _*) + ) + + def read(json: JsValue): Schema = + json match + case JsObject(fields) => + val version = fields.get("version") match + case Some(JsString(version)) => version + case _ => throw new Exception("Invalid JSON: schema.version must be present") + val schema = fields.get("schema") match + case Some(JsArray(fields)) => fields.map(fieldWriter.read).toList + case _ => throw new Exception("Invalid JSON: schema.schema must be present") + Schema(schema, version) + case _ => throw new Exception("Invalid JSON: expected object") diff --git a/besom-cfg/lib/src/test/scala/ConfiguredTest.scala b/besom-cfg/lib/src/test/scala/ConfiguredTest.scala new file mode 100644 index 00000000..d39b084f --- /dev/null +++ b/besom-cfg/lib/src/test/scala/ConfiguredTest.scala @@ -0,0 +1,83 @@ +package besom.cfg + +case class Test1(los: List[String]) derives Configured + +case class Test2(name: String, int: Int, struct: First, list: List[Double]) derives Configured +case class First(d: Int, e: String) + +case class Test3( + name: String, + int: Int, + s: First, + l: List[Double], + ls: List[Second], + lo: List[String], + ol: List[String], + os: Third +) derives Configured +case class Second(f1: List[Fourth], f2: String) +case class Third(oh: String, it: String) +case class Fourth(deep: String) + +class ConfiguredTest extends munit.FunSuite: + + test("very simple case class") { + val env = Map("los.0" -> "test", "los.1" -> "test2") + + summon[Configured[Test1]].newInstanceFromEnv(env) match + case Test1(los) => + assertEquals(los, List("test", "test2")) + } + + test("can read a simple case class from environment variables") { + val env = Map( + "name" -> "test", + "int" -> "23", + "struct.d" -> "23", + "struct.e" -> "test", + "list.0" -> "1.2", + "list.1" -> "2.3", + "list.2" -> "3.4" + ) + + summon[Configured[Test2]].newInstanceFromEnv(env) match + case Test2(name, int, s, l) => + assertEquals(name, "test") + assertEquals(int, 23) + assertEquals(s, First(23, "test")) + assertEquals(l, List(1.2, 2.3, 3.4)) + } + + test("can read a complex case class from environment variables") { + val env = Map( + "name" -> "test", + "int" -> "23", + "s.d" -> "23", + "s.e" -> "test", + "l.0" -> "1.2", + "l.1" -> "2.3", + "l.2" -> "3.4", + "ls.0.f1.0.deep" -> "q", + "ls.0.f2" -> "w", + "lo.0" -> "a", + "lo.1" -> "b", + "lo.2" -> "c", + "ol.0" -> "x", + "ol.1" -> "y", + "ol.2" -> "z", + "os.oh" -> "yeah", + "os.it" -> "works!" + ) + + summon[Configured[Test3]].newInstanceFromEnv(env) match + case Test3(name, int, s, l, ls, lo, ol, os) => + assertEquals(name, "test") + assertEquals(int, 23) + assertEquals(s, First(23, "test")) + assertEquals(l, List(1.2, 2.3, 3.4)) + assertEquals(ls, List(Second(List(Fourth("q")), "w"))) + assertEquals(lo, List("a", "b", "c")) + assertEquals(ol, List("x", "y", "z")) + assertEquals(os, Third("yeah", "works!")) + } +end ConfiguredTest diff --git a/besom-cfg/version.txt b/besom-cfg/version.txt new file mode 100644 index 00000000..9b17816c --- /dev/null +++ b/besom-cfg/version.txt @@ -0,0 +1 @@ +0.2.0-SNAPSHOT \ No newline at end of file diff --git a/scripts/Packages.scala b/scripts/Packages.scala index 9c833514..2042793c 100644 --- a/scripts/Packages.scala +++ b/scripts/Packages.scala @@ -36,8 +36,8 @@ object Packages: case "local" :: tail => publishLocalSelected(generateSelected(packagesDir, tail), tail) case "maven-all" :: Nil => publishMavenAll(generateAll(packagesDir)) case "maven" :: tail => publishMavenSelected(generateSelected(packagesDir, tail), tail) - case "metadata-all" :: Nil => downloadPackagesMetadataAndSchema(packagesDir, selected = Nil) - case "metadata" :: tail => downloadPackagesMetadataAndSchema(packagesDir, selected = tail) + case "metadata-all" :: Nil => readOrDownloadPackagesMetadataAndSchema(packagesDir, selected = Nil) + case "metadata" :: tail => readOrDownloadPackagesMetadataAndSchema(packagesDir, selected = tail) case "generate-all" :: Nil => generateAll(packagesDir) case "generate" :: tail => generateSelected(packagesDir, tail) case "publish-local-all" :: Nil => publishLocalAll(generatedFile) @@ -156,15 +156,18 @@ object Packages: "fortios" // method collision - https://github.com/VirtusLab/besom/issues/458 ) - def generateAll(targetPath: os.Path)(using Config, Flags): os.Path = { + def generateAll(targetPath: os.Path)(using config: Config, flags: Flags): os.Path = { val metadata = readOrFetchPackagesMetadata(targetPath, Nil) .filter { - case m if (codegenProblemPackages ++ compileProblemPackages).contains(m.name) => + case (m, false) if !flags.force => + println(s"Skipping unchanged package generation: ${m.name}") + false + case (m, _) if (codegenProblemPackages ++ compileProblemPackages).contains(m.name) => println(s"Skipping problematic package generation: ${m.name}") false case _ => true } - upsertGeneratedFile(generate(metadata)) + upsertGeneratedFile(generate(metadata.map(_._1))) } def publishLocalAll(sourceFile: os.Path)(using Config, Flags): os.Path = { @@ -193,6 +196,13 @@ object Packages: def generateSelected(targetPath: os.Path, selected: List[String])(using config: Config, flags: Flags): os.Path = { val readPackages = readOrFetchPackagesMetadata(targetPath, selected) + .filter { + case (m, false) if !flags.force => + println(s"Skipping unchanged package generation: ${m.name}") + false + case _ => true + } + .map(_._1) val packages = selected.map { p => PackageId.parse(p) match { case Right(name, Some(version)) => @@ -202,7 +212,7 @@ object Packages: case Right(name, None) => readPackages .find(_.name == name) - .getOrElse(throw Exception(s"Package '$name' not found in the generated packages (${readPackages.size})")) + .getOrElse(throw Exception(s"Package '$name' not found in the packages (${readPackages.size})")) case Left(e) => throw e } }.toVector @@ -272,19 +282,17 @@ object Packages: }.toVector def resolveMavenAll(targetDir: Path)(using config: Config): Unit = - val _ = downloadPackagesMetadata(targetDir, Nil) - val latest = readPackagesMetadata(targetDir).map(_.copy(version = None)) - val resolved = resolveMavenVersions(targetDir, latest) - compareWithLatest(targetDir, resolved, latest) + val latest = readOrDownloadPackagesMetadata(targetDir, Nil) + val resolved = resolveMavenVersions(targetDir, latest.map(_._1)) + compareWithLatest(targetDir, resolved, latest.map(_._1)) end resolveMavenAll def resolveMavenSelected(targetDir: Path, selected: List[String])(using config: Config): Unit = val selectedPackages = parsePackages(selected) val resolved = resolveMavenVersions(targetDir, selectedPackages) - val _ = downloadPackagesMetadata(targetDir, selected) - val latestPackages = readPackagesMetadata(targetDir) - compareWithLatest(targetDir, resolved, latestPackages) + val latestPackages = readOrDownloadPackagesMetadata(targetDir, selected) + compareWithLatest(targetDir, resolved, latestPackages.map(_._1)) end resolveMavenSelected def resolveMavenVersions(targetDir: os.Path, packages: Vector[PackageMetadata])(using config: Config): Map[String, SemanticVersion] = @@ -564,63 +572,17 @@ object Packages: end if } - private def readPackageMetadataFiles(path: os.Path): Map[String, Path] = ListMap.from( - os - .list(path) - .filter(_.last.endsWith("metadata.json")) - .map(p => p.last.stripSuffix(".metadata.json") -> p) - ) - - def readPackagesMetadata(targetPath: os.Path): Vector[PackageMetadata] = - readPackageMetadataFiles(targetPath) - .map((_, p) => PackageMetadata.fromJsonFile(p)) - .toVector - + // TODO: should be split into two functions, with one that does not touch schemas, to be used in Version.scala def readOrFetchPackagesMetadata( targetPath: os.Path, selected: List[String] - )(using config: Config, flags: Flags): Vector[PackageMetadata] = { - // download metadata if none found - if flags.force || !os.exists(targetPath) then - println(s"No packages metadata found in: '$targetPath', downloading...") - downloadPackagesMetadataAndSchema(targetPath, selected) - else println(s"Reading packages metadata from: '$targetPath'") - - // read cached metadata - val cached = readPackageMetadataFiles(targetPath) - - // download all metadata if selected packages are not found - val selectedNames = selected.map(_.takeWhile(_ != ':')).toSet - val selectedAreSubsetOfCached = selectedNames.subsetOf(cached.keys.toSet) - if selected.nonEmpty then - println(s"Selected: ${selected.mkString(", ")}, cached: ${cached.keys.mkString(", ")}, subset: $selectedAreSubsetOfCached") - val downloaded = - if !selectedAreSubsetOfCached - then - downloadPackagesMetadataAndSchema(targetPath, selected) - readPackageMetadataFiles(targetPath) - else cached - - // double check if selected packages are found - selectedNames.map { p => - downloaded.keys - .find(_ == p) - .getOrElse(throw Exception(s"Package '$p' not found in downloaded packages metadata (${downloaded.size})")) - }.toVector - - val metadata = downloaded - .filter { (name, _) => - selectedNames match - case _ if selectedNames.isEmpty => true - case s => s.contains(name) // filter out selected packages only if selected is not empty - } - .map((_, p) => PackageMetadata.fromJsonFile(p)) + )(using config: Config, flags: Flags): Vector[(PackageMetadata, Boolean)] = { + val metadata = readOrDownloadPackagesMetadataAndSchema(targetPath, selected) .collect { - case metadata if selected.nonEmpty => metadata - case metadata if !pluginDownloadProblemPackages.contains(metadata.name) => - metadata // filter out packages with known problems only if selected is not empty + case (metadata, changed) if selected.nonEmpty => (metadata, changed) + case (metadata, changed) if !pluginDownloadProblemPackages.contains(metadata.name) => + (metadata, changed) // filter out packages with known problems only if selected is not empty } - .toVector if metadata.isEmpty then throw Exception(s"No packages metadata found in: '$targetPath'") metadata @@ -629,12 +591,24 @@ object Packages: private case class PackageYAML(name: String, repo_url: String, schema_file_path: String, version: String) derives YamlCodec // downloads latest package metadata and schemas using Pulumi packages repository - def downloadPackagesMetadataAndSchema(targetPath: os.Path, selected: List[String])(using config: Config): Unit = - downloadPackagesSchema(downloadPackagesMetadata(targetPath, selected)) - end downloadPackagesMetadataAndSchema + private def readOrDownloadPackagesMetadataAndSchema( + targetPath: os.Path, + selected: List[String] + )(using config: Config): Vector[(PackageMetadata, Boolean)] = + val metadata = readOrDownloadPackagesMetadata(targetPath, selected) + readOrDownloadPackagesSchema(metadata.map(t => (t._2, t._3))) + metadata.map(t => (t._1, t._3)) + end readOrDownloadPackagesMetadataAndSchema + + case class PackageSource(name: String, download_url: String, sha: String) derives UpickleApi.ReadWriter + object PackageSource { + def fromJsonArray(json: ujson.Readable): List[PackageSource] = UpickleApi.read(json, trace = true) + } - private def downloadPackagesMetadata(targetPath: os.Path, selected: List[String])(using config: Config): Vector[PackageYAML] = - println("Downloading packages metadata...") + private def readOrDownloadPackagesMetadata(targetPath: os.Path, selected: List[String])(using + config: Config + ): Vector[(PackageMetadata, PackageYAML, Boolean)] = + println("Downloading or reading packages metadata...") val packagesRepoApi = "https://api.github.com/repos/pulumi/registry/contents/themes/default/data/registry/packages" @@ -645,14 +619,11 @@ object Packages: if packagesResponse.statusCode != 200 then throw Exception(s"Failed to fetch packages list from: '$packagesRepoApi'") - case class PackageSource(name: String, download_url: String, sha: String) derives UpickleApi.ReadWriter - object PackageSource { - def fromJsonArray(json: ujson.Readable): List[PackageSource] = UpickleApi.read(json, trace = true) - } - + val packagesContent = packagesResponse.text() + // use ListMap to preserve the order val packages: ListMap[String, PackageSource] = ListMap.from( PackageSource - .fromJsonArray(packagesResponse.text()) + .fromJsonArray(packagesContent) .map { p => val packageName = p.name.stripSuffix(".yaml") packageName -> p @@ -670,7 +641,7 @@ object Packages: if selectedNames.nonEmpty then println(s"Selected for download: ${selectedNames.mkString(", ")}") // fetch all production schema metadata - val downloaded = withProgress(s"Downloading $size packages metadata", size) { + val downloaded = withProgress(s"Downloading or reading $size packages metadata", size) { packages .filter { (name, _) => selectedNames match @@ -695,8 +666,8 @@ object Packages: sha != p.sha else true // no sha file, assume it has changed - val metadataRaw: Either[Error, PackageYAML] = { - if hasChanged || !os.exists(metadataPath) then + val content: Either[Error, String] = + if hasChanged then val schemaResponse = requests.get(p.download_url, headers = authHeader) schemaResponse.statusCode match case 200 => @@ -707,22 +678,29 @@ object Packages: case _ => Left(s"failed to download metadata for package: '${p.name}'") else Right(os.read(metadataPath)) - }.flatMap { - _.as[PackageYAML].fold( - error => Left(s"failed to deserialize metadata for package: '${p.name}', error: ${error}"), - p => Right(p) - ) - } - val metadata: Either[Error, PackageMetadata] = - metadataRaw.map(m => PackageMetadata(m.name, m.version).withUrl(m.repo_url)) + val metadataRaw: Either[Error, (PackageYAML, Boolean)] = + content + .flatMap( + _.as[PackageYAML].fold( + error => Left(s"failed to deserialize metadata for package: '${p.name}', error: ${error}"), + p => Right(p) + ) + ) + .map((_, hasChanged)) + + val metadata: Either[Error, (PackageMetadata, PackageYAML, Boolean)] = + metadataRaw.map { case (m, changed) => + (PackageMetadata(m.name, m.version).withUrl(m.repo_url), m, changed) + } metadata match - case Left(error) => Progress.fail(error) - case Right(value) => + case left @ Left(error) => + Progress.fail(error) + left + case Right(t @ (value, _, _)) => os.write.over(targetPath / s"${packageName}.metadata.json", value.toJson, createFolders = true) - - metadataRaw + Right(t) catch case NonFatal(e) => val msg = s"Failed to download metadata for package: '${p.name}', error: ${e.getMessage}" @@ -731,23 +709,22 @@ object Packages: finally Progress.end end try } - .map(_.toOption) - .collect { case Some(p) => p } + .collect { case Right(p) => p } // ignore the failures and try to go forward .toVector } println(s"Packages directory: '$targetPath'") downloaded - end downloadPackagesMetadata + end readOrDownloadPackagesMetadata - private def downloadPackagesSchema(downloaded: Vector[PackageYAML])(using config: Config): Unit = { + private def readOrDownloadPackagesSchema(downloaded: Vector[(PackageYAML, Boolean)])(using config: Config): Unit = { // pre-fetch all available production schemas, if any are missing here code generation will use a fallback mechanism withProgress(s"Downloading ${downloaded.size} packages schemas", downloaded.size) { - downloaded.foreach(p => { + downloaded.foreach((p, changed) => { Progress.report(label = p.name) try val ext = if p.schema_file_path.endsWith(".yaml") || p.schema_file_path.endsWith(".yml") then "yaml" else "json" val schemaPath = config.schemasDir / p.name / PackageVersion(p.version).get / s"schema.$ext" - if !os.exists(schemaPath) then + if changed || !os.exists(schemaPath) then val rawUrlPrefix = p.repo_url.replace("https://github.com/", "https://raw.githubusercontent.com/") val url = s"$rawUrlPrefix/${p.version}/${p.schema_file_path}" val schemaResponse = requests.get(url) @@ -769,7 +746,7 @@ object Packages: } def listLatestPackages(targetPath: os.Path)(using config: Config, flags: Flags): Unit = { - val metadata = readOrFetchPackagesMetadata(targetPath, Nil) + val metadata = readOrFetchPackagesMetadata(targetPath, Nil).map(_._1) val active = metadata .filterNot(m => blockedPackages.contains(m.name)) .filterNot(m => codegenProblemPackages.contains(m.name)) diff --git a/scripts/Version.scala b/scripts/Version.scala index 10c7b2c9..16017378 100644 --- a/scripts/Version.scala +++ b/scripts/Version.scala @@ -156,7 +156,7 @@ object Version: def latestPackageVersion(name: String)(using Config): String = val latestPackageVersions = fetchLatestPackageVersions Try(latestPackageVersions(name)).recover { case e: NoSuchElementException => - throw Exception(s"package $name not found", e) + throw Exception(s"package $name not found, total packages: ${latestPackageVersions.size}", e) }.get private def fetchLatestPackageVersions(using Config): Map[String, String] = @@ -164,7 +164,7 @@ object Version: given Flags = Flags() Packages .readOrFetchPackagesMetadata(Packages.packagesDir, Nil) - .map { metadata => + .map { (metadata, _) => metadata.name -> metadata.version.getOrElse(throw Exception("Package version must be present at this point")).asString } .toMap