From a5963b94e797aa28650dec656eff1ee4b787630e Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 8 Oct 2025 11:13:48 +0100 Subject: [PATCH 1/9] FDN-4099 Add support for explicit configuration of production servers --- build.sbt | 3 + .../scala/io/flow/build/BuildConfig.scala | 7 +- src/main/scala/io/flow/build/Config.scala | 94 ++++++++++++++++++ src/main/scala/io/flow/build/Main.scala | 97 ++++--------------- .../scala/io/flow/build/ServerConfig.scala | 26 +++++ src/main/scala/io/flow/proxy/Controller.scala | 18 +++- .../io/flow/build/ServerConfigSpec.scala | 29 ++++++ .../scala/io/flow/proxy/ControllerSpec.scala | 36 +++++++ 8 files changed, 230 insertions(+), 80 deletions(-) create mode 100644 src/main/scala/io/flow/build/ServerConfig.scala create mode 100644 src/test/scala/io/flow/build/ServerConfigSpec.scala create mode 100644 src/test/scala/io/flow/proxy/ControllerSpec.scala diff --git a/build.sbt b/build.sbt index a5281b12..b769e3a2 100644 --- a/build.sbt +++ b/build.sbt @@ -53,6 +53,9 @@ lazy val root = project "org.typelevel" %% "cats-effect" % "2.3.3", "org.scalatest" %% "scalatest" % "3.2.19" % Test, "com.github.scopt" %% "scopt" % "4.1.0", + "io.circe" %% "circe-generic" % "0.14.9", + "io.circe" %% "circe-parser" % "0.14.9", + "io.circe" %% "circe-yaml" % "0.16.1", ), ) diff --git a/src/main/scala/io/flow/build/BuildConfig.scala b/src/main/scala/io/flow/build/BuildConfig.scala index 553a640e..10238654 100644 --- a/src/main/scala/io/flow/build/BuildConfig.scala +++ b/src/main/scala/io/flow/build/BuildConfig.scala @@ -9,4 +9,9 @@ package io.flow.build * @param output * Where controllers write files created. */ -case class BuildConfig(protocol: String, domain: String, output: java.nio.file.Path) +case class BuildConfig( + protocol: String, + domain: String, + productionServerConfigs: Seq[ServerConfig], + output: java.nio.file.Path, +) diff --git a/src/main/scala/io/flow/build/Config.scala b/src/main/scala/io/flow/build/Config.scala index ad94ce1d..646da005 100644 --- a/src/main/scala/io/flow/build/Config.scala +++ b/src/main/scala/io/flow/build/Config.scala @@ -1,10 +1,104 @@ package io.flow.build +import io.flow.{lint, oneapi, proxy, stream} +import scopt.OptionParser + +import java.nio.file.{Files, Path} + case class Config( buildType: BuildType = BuildType.Api, protocol: String = "https", domain: String = "api.flow.io", + productionConfig: Option[java.nio.file.Path] = None, buildCommand: String = "all", apis: Seq[String] = Seq(), output: java.nio.file.Path = java.nio.file.Paths.get("/tmp"), ) + +object Config { + def parseArgs(args: Array[String]): Option[Config] = makeParser().parse(args, Config()) + + private implicit val BuildTypeRead: scopt.Read[BuildType] = + scopt.Read.reads(s => BuildType.fromString(s).getOrElse(sys.error(s"Unknown BuildType '$s'"))) + + private def makeParser(): OptionParser[Config] = new scopt.OptionParser[Config]("api-build") { + override def showUsageOnError: Option[Boolean] = Some(true) + + arg[BuildType]("") + .action((bt, c) => c.copy(buildType = bt)) + .text("One of: " + BuildType.all.map(_.toString).mkString(", ")) + .required() + + arg[String]("") + .action((cmd, c) => c.copy(buildCommand = cmd)) + .text("One of: " + controllers(BuildType.Api).map(_.command).mkString("all, ", ", ", "")) + .required() + + opt[Unit]("http-only") + .text("If specified, results in http being used as the protocol in host names (default is 'https')") + .action((_, c) => c.copy(protocol = "http")) + + opt[String]('d', "domain") + .text("Domain to use when constructing the service subdomain (default is 'api.flow.io')") + .action((d, c) => c.copy(domain = d)) + + opt[Path]("production-config") + .text("Optional yaml file to provide host configuration for servers (production only)") + .action((path, c) => c.copy(productionConfig = Some(path))) + + opt[Path]('o', "output") + .text("Where to write output files (default is '/tmp')") + .validate { path => + if (!Files.exists(path)) failure(s"Path does not exist: $path") + else if (!Files.isDirectory(path)) failure(s"Path is not a directory: $path") + else success + } + .action((p, c) => c.copy(output = p)) + + arg[String]("...") + .text("API specs from APIBuilder") + .action((api, c) => c.copy(apis = c.apis :+ api)) + .unbounded() + .optional() + .validate(api => + Application.parse(api) match { + case Some(_) => success + case None => failure(s"Could not parse application[$api]") + }, + ) + + help("help") + + checkConfig(c => { + val selected = if (c.buildCommand == "all") { + controllers(c.buildType) + } else { + controllers(c.buildType).filter(_.command == c.buildCommand) + } + selected.toList match { + case Nil => { + failure( + s"Invalid command[${c.buildCommand}] for build type[${c.buildType}]. " + + s"Must be one of: all, " + controllers(c.buildType).map(_.command).mkString(", "), + ) + } + case _ => { + success + } + } + }) + } + + private def controllers(buildType: BuildType): Seq[Controller] = { + val all = scala.collection.mutable.ListBuffer[Controller]() + all.append(lint.Controller()) + all.append(stream.Controller()) + if (buildType.oneApi) { + all.append(oneapi.Controller()) + } + if (buildType.proxy) { + all.append(proxy.Controller()) + } + all.toSeq + } +} diff --git a/src/main/scala/io/flow/build/Main.scala b/src/main/scala/io/flow/build/Main.scala index ceaf538b..a12786fb 100644 --- a/src/main/scala/io/flow/build/Main.scala +++ b/src/main/scala/io/flow/build/Main.scala @@ -3,8 +3,6 @@ package io.flow.build import io.apibuilder.spec.v0.models.Service import io.flow.{lint, oneapi, proxy, stream} -import java.nio.file.{Files, Path} - object Main extends App { import scala.concurrent.ExecutionContext.Implicits.global @@ -29,75 +27,8 @@ object Main extends App { println(s"** Error loading apibuilder config: $error") System.exit(1) } - case Right(profile) => { - implicit val buildTypeRead: scopt.Read[BuildType] = - scopt.Read.reads(s => BuildType.fromString(s).getOrElse(sys.error(s"Unknown BuildType '$s'"))) - - val parser = new scopt.OptionParser[Config]("api-build") { - override def showUsageOnError = Some(true) - - arg[BuildType]("") - .action((bt, c) => c.copy(buildType = bt)) - .text("One of: " + BuildType.all.map(_.toString).mkString(", ")) - .required() - - arg[String]("") - .action((cmd, c) => c.copy(buildCommand = cmd)) - .text("One of: " + controllers(BuildType.Api).map(_.command).mkString("all, ", ", ", "")) - .required() - - opt[Unit]("http-only") - .text("If specified, results in http being used as the protocol in host names (default is 'https')") - .action((_, c) => c.copy(protocol = "http")) - - opt[String]('d', "domain") - .text("Domain to use when constructing the service subdomain (default is 'api.flow.io')") - .action((d, c) => c.copy(domain = d)) - - opt[Path]('o', "output") - .text("Where to write output files (default is '/tmp')") - .validate { path => - if (!Files.exists(path)) failure(s"Path does not exist: $path") - else if (!Files.isDirectory(path)) failure(s"Path is not a directory: $path") - else success - } - .action((p, c) => c.copy(output = p)) - - arg[String]("...") - .text("API specs from APIBuilder") - .action((api, c) => c.copy(apis = c.apis :+ api)) - .unbounded() - .optional() - .validate(api => - Application.parse(api) match { - case Some(_) => success - case None => failure(s"Could not parse application[$api]") - }, - ) - - help("help") - - checkConfig(c => { - val selected = if (c.buildCommand == "all") { - controllers(c.buildType) - } else { - controllers(c.buildType).filter(_.command == c.buildCommand) - } - selected.toList match { - case Nil => { - failure( - s"Invalid command[${c.buildCommand}] for build type[${c.buildType}]. " + - s"Must be one of: all, " + controllers(c.buildType).map(_.command).mkString(", "), - ) - } - case _ => { - success - } - } - }) - } - - parser.parse(args, Config()) match { + case Right(profile) => + Config.parseArgs(args) match { case Some(config) => val selected = if (config.buildCommand == "all") { controllers(config.buildType) @@ -108,20 +39,34 @@ object Main extends App { val allApplications: Seq[Application] = config.apis.flatMap { name => Application.parse(name) } - val buildConfig = BuildConfig(protocol = config.protocol, domain = config.domain, output = config.output) + val serverConfigs: Seq[ServerConfig] = config.productionConfig + .map { path => + ServerConfig.parseFile(path) match { + case Left(error) => sys.error(s"Failed to parse $path: '$error'") + case Right(serverConfigs) => serverConfigs + } + } + .getOrElse(Nil) + + val buildConfig = BuildConfig( + protocol = config.protocol, + domain = config.domain, + productionServerConfigs = serverConfigs, + output = config.output, + ) val dl = DownloadCache(Downloader(profile)) dl.downloadServices(allApplications) match { - case Left(errors) => { + case Left(errors) => println(s"Errors downloading services:") errors.foreach { e => println(s" - $e") } System.exit(errors.length) - } - case Right(services) => run(config.buildType, buildConfig, dl, selected, services) + + case Right(services) => + run(config.buildType, buildConfig, dl, selected, services) } case None => // error message already printed } - } } private[this] def run( diff --git a/src/main/scala/io/flow/build/ServerConfig.scala b/src/main/scala/io/flow/build/ServerConfig.scala new file mode 100644 index 00000000..d4aea692 --- /dev/null +++ b/src/main/scala/io/flow/build/ServerConfig.scala @@ -0,0 +1,26 @@ +package io.flow.build + +import cats.implicits._ +import io.circe.generic.auto._ +import io.circe.yaml + +import scala.io.Source +import scala.util.Using + +case class ServerConfig(name: String, host: String) + +object ServerConfig { + def parseFile(path: java.nio.file.Path): Either[String, Seq[ServerConfig]] = + Using.resource(Source.fromFile(path.toUri)) { source => + parseYaml(source.mkString) + } + + def parseYaml(contents: String): Either[String, Seq[ServerConfig]] = { + val parseResult = for { + json <- yaml.parser.parse(contents) + serverConfigs <- json.hcursor.downField("servers").as[List[ServerConfig]] + } yield serverConfigs + + parseResult.leftMap(_.show) + } +} diff --git a/src/main/scala/io/flow/proxy/Controller.scala b/src/main/scala/io/flow/proxy/Controller.scala index 67453f33..afc77014 100644 --- a/src/main/scala/io/flow/proxy/Controller.scala +++ b/src/main/scala/io/flow/proxy/Controller.scala @@ -8,6 +8,18 @@ import play.api.libs.json.Json import java.io.File import java.nio.file.Path +object Controller { + def makeProductionHostProvider( + buildConfig: BuildConfig, + serviceHostResolver: ServiceHostResolver, + ): Service => String = { service => + buildConfig.productionServerConfigs.find(_.name == service.name) match { + case Some(serverConfig) => serverConfig.host + case None => s"${buildConfig.protocol}://${serviceHostResolver.host(service.name)}.${buildConfig.domain}" + } + } +} + case class Controller() extends io.flow.build.Controller { /** Allowlist of applications in the 'api' repo that do not exist in registry @@ -87,9 +99,9 @@ case class Controller() extends io.flow.build.Controller { val registryClient = new RegistryClient() try { - buildProxyFile(buildType, buildConfig.output, services, version, "production") { service => - s"${buildConfig.protocol}://${serviceHostResolver.host(service.name)}.${buildConfig.domain}" - } + buildProxyFile(buildType, buildConfig.output, services, version, "production")( + Controller.makeProductionHostProvider(buildConfig, serviceHostResolver), + ) val cache = RegistryApplicationCache(registryClient) diff --git a/src/test/scala/io/flow/build/ServerConfigSpec.scala b/src/test/scala/io/flow/build/ServerConfigSpec.scala new file mode 100644 index 00000000..f9e134ef --- /dev/null +++ b/src/test/scala/io/flow/build/ServerConfigSpec.scala @@ -0,0 +1,29 @@ +package io.flow.build + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +class ServerConfigSpec extends AnyFunSpec with Matchers { + it("parses server list") { + val yaml = + """ + |# comments + | + |servers: + | - name: catalog + | host: http://localhost:7121 + | - name: billing + | host: http://localhost:6071 + |""".stripMargin + ServerConfig.parseYaml(yaml) shouldBe Right( + Seq( + ServerConfig("catalog", "http://localhost:7121"), + ServerConfig("billing", "http://localhost:6071"), + ), + ) + } + + it("fails to parse") { + ServerConfig.parseYaml("not yaml") shouldBe Left("DecodingFailure at .servers: Missing required field") + } +} diff --git a/src/test/scala/io/flow/proxy/ControllerSpec.scala b/src/test/scala/io/flow/proxy/ControllerSpec.scala new file mode 100644 index 00000000..e9938d6f --- /dev/null +++ b/src/test/scala/io/flow/proxy/ControllerSpec.scala @@ -0,0 +1,36 @@ +package io.flow.proxy + +import io.apibuilder.spec.v0.models.Service +import io.flow.build.{BuildConfig, ServerConfig} +import io.flow.lint.Services +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +class ControllerSpec extends AnyFunSpec with Matchers { + private val service: Service = Services.Base + + private val resolveHost: Service => String = { + val buildConfig = BuildConfig( + protocol = "https", + domain = "api.flow.io", + productionServerConfigs = Seq(ServerConfig(name = service.name, host = "http://lint.me")), + output = java.nio.file.Paths.get("/tmp"), + ) + val serviceHostResolver = ServiceHostResolver(Nil) + Controller.makeProductionHostProvider(buildConfig, serviceHostResolver) + } + + describe("production host resolver") { + describe("when server config available") { + it("should use configured host") { + resolveHost(service) shouldBe "http://lint.me" + } + } + + describe("when server config not available") { + it("should use constructed host name") { + resolveHost(service.copy(name = "foo")) shouldBe "https://foo.api.flow.io" + } + } + } +} From 02166bdea1d683af5399f1fa2960bce4ebc85e5c Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 8 Oct 2025 11:26:43 +0100 Subject: [PATCH 2/9] test --- src/main/scala/io/flow/build/ServerConfig.scala | 7 +++++-- src/test/scala/io/flow/build/ServerConfigSpec.scala | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/scala/io/flow/build/ServerConfig.scala b/src/main/scala/io/flow/build/ServerConfig.scala index d4aea692..da84b80c 100644 --- a/src/main/scala/io/flow/build/ServerConfig.scala +++ b/src/main/scala/io/flow/build/ServerConfig.scala @@ -5,14 +5,17 @@ import io.circe.generic.auto._ import io.circe.yaml import scala.io.Source -import scala.util.Using +import scala.util.{Failure, Success, Using} case class ServerConfig(name: String, host: String) object ServerConfig { def parseFile(path: java.nio.file.Path): Either[String, Seq[ServerConfig]] = - Using.resource(Source.fromFile(path.toUri)) { source => + Using(Source.fromFile(path.toUri)) { source => parseYaml(source.mkString) + } match { + case Failure(ex) => Left(ex.getMessage) + case Success(result) => result } def parseYaml(contents: String): Either[String, Seq[ServerConfig]] = { diff --git a/src/test/scala/io/flow/build/ServerConfigSpec.scala b/src/test/scala/io/flow/build/ServerConfigSpec.scala index f9e134ef..61f9c441 100644 --- a/src/test/scala/io/flow/build/ServerConfigSpec.scala +++ b/src/test/scala/io/flow/build/ServerConfigSpec.scala @@ -4,7 +4,7 @@ import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers class ServerConfigSpec extends AnyFunSpec with Matchers { - it("parses server list") { + it("parses server list from valid yaml") { val yaml = """ |# comments @@ -26,4 +26,10 @@ class ServerConfigSpec extends AnyFunSpec with Matchers { it("fails to parse") { ServerConfig.parseYaml("not yaml") shouldBe Left("DecodingFailure at .servers: Missing required field") } + + it("when file does not exist") { + ServerConfig.parseFile(java.nio.file.Paths.get("/tmp/wibblywobblywonder.yaml")) shouldBe Left( + "/tmp/wibblywobblywonder.yaml (No such file or directory)", + ) + } } From 40a83827c1e7cf89499a75480b879330770c367a Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 8 Oct 2025 11:27:53 +0100 Subject: [PATCH 3/9] Update src/main/scala/io/flow/build/Config.scala Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/main/scala/io/flow/build/Config.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/scala/io/flow/build/Config.scala b/src/main/scala/io/flow/build/Config.scala index 646da005..c607c3db 100644 --- a/src/main/scala/io/flow/build/Config.scala +++ b/src/main/scala/io/flow/build/Config.scala @@ -44,8 +44,12 @@ object Config { opt[Path]("production-config") .text("Optional yaml file to provide host configuration for servers (production only)") + .validate { path => + if (!Files.exists(path)) failure(s"Production config file does not exist: $path") + else if (!Files.isRegularFile(path)) failure(s"Production config path is not a file: $path") + else success + } .action((path, c) => c.copy(productionConfig = Some(path))) - opt[Path]('o', "output") .text("Where to write output files (default is '/tmp')") .validate { path => From 3ca42596103adec71a4baa5dcd891b78aaef856a Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 8 Oct 2025 11:29:52 +0100 Subject: [PATCH 4/9] Update --- src/main/scala/io/flow/build/Config.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/io/flow/build/Config.scala b/src/main/scala/io/flow/build/Config.scala index c607c3db..317b11e7 100644 --- a/src/main/scala/io/flow/build/Config.scala +++ b/src/main/scala/io/flow/build/Config.scala @@ -45,8 +45,8 @@ object Config { opt[Path]("production-config") .text("Optional yaml file to provide host configuration for servers (production only)") .validate { path => - if (!Files.exists(path)) failure(s"Production config file does not exist: $path") - else if (!Files.isRegularFile(path)) failure(s"Production config path is not a file: $path") + if (!Files.exists(path)) failure(s"Production config file does not exist: '$path'") + else if (!Files.isRegularFile(path)) failure(s"Production config path is not a file: '$path'") else success } .action((path, c) => c.copy(productionConfig = Some(path))) From 4498d589682cf14cb7c823fa700d92240d1a091d Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 8 Oct 2025 13:52:01 +0100 Subject: [PATCH 5/9] Review feedback --- src/main/scala/io/flow/build/Config.scala | 12 ++++++------ src/main/scala/io/flow/build/Main.scala | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/scala/io/flow/build/Config.scala b/src/main/scala/io/flow/build/Config.scala index 317b11e7..9705686c 100644 --- a/src/main/scala/io/flow/build/Config.scala +++ b/src/main/scala/io/flow/build/Config.scala @@ -9,7 +9,7 @@ case class Config( buildType: BuildType = BuildType.Api, protocol: String = "https", domain: String = "api.flow.io", - productionConfig: Option[java.nio.file.Path] = None, + overrideConfig: Option[java.nio.file.Path] = None, buildCommand: String = "all", apis: Seq[String] = Seq(), output: java.nio.file.Path = java.nio.file.Paths.get("/tmp"), @@ -42,14 +42,14 @@ object Config { .text("Domain to use when constructing the service subdomain (default is 'api.flow.io')") .action((d, c) => c.copy(domain = d)) - opt[Path]("production-config") - .text("Optional yaml file to provide host configuration for servers (production only)") + opt[Path]("override-config") + .text("Optional yaml file to override default configuration for servers (production only)") .validate { path => - if (!Files.exists(path)) failure(s"Production config file does not exist: '$path'") - else if (!Files.isRegularFile(path)) failure(s"Production config path is not a file: '$path'") + if (!Files.exists(path)) failure(s"Override config file does not exist: '$path'") + else if (!Files.isRegularFile(path)) failure(s"Override config path is not a file: '$path'") else success } - .action((path, c) => c.copy(productionConfig = Some(path))) + .action((path, c) => c.copy(overrideConfig = Some(path))) opt[Path]('o', "output") .text("Where to write output files (default is '/tmp')") .validate { path => diff --git a/src/main/scala/io/flow/build/Main.scala b/src/main/scala/io/flow/build/Main.scala index a12786fb..ba71c54f 100644 --- a/src/main/scala/io/flow/build/Main.scala +++ b/src/main/scala/io/flow/build/Main.scala @@ -39,7 +39,7 @@ object Main extends App { val allApplications: Seq[Application] = config.apis.flatMap { name => Application.parse(name) } - val serverConfigs: Seq[ServerConfig] = config.productionConfig + val serverConfigs: Seq[ServerConfig] = config.overrideConfig .map { path => ServerConfig.parseFile(path) match { case Left(error) => sys.error(s"Failed to parse $path: '$error'") From 482e07645a40810d10d005adc843ce56c468255b Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 8 Oct 2025 14:12:55 +0100 Subject: [PATCH 6/9] reename --- src/main/scala/io/flow/build/Config.scala | 6 +++--- src/main/scala/io/flow/build/Main.scala | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/io/flow/build/Config.scala b/src/main/scala/io/flow/build/Config.scala index 9705686c..77a72413 100644 --- a/src/main/scala/io/flow/build/Config.scala +++ b/src/main/scala/io/flow/build/Config.scala @@ -9,7 +9,7 @@ case class Config( buildType: BuildType = BuildType.Api, protocol: String = "https", domain: String = "api.flow.io", - overrideConfig: Option[java.nio.file.Path] = None, + productionConfigOverride: Option[java.nio.file.Path] = None, buildCommand: String = "all", apis: Seq[String] = Seq(), output: java.nio.file.Path = java.nio.file.Paths.get("/tmp"), @@ -42,14 +42,14 @@ object Config { .text("Domain to use when constructing the service subdomain (default is 'api.flow.io')") .action((d, c) => c.copy(domain = d)) - opt[Path]("override-config") + opt[Path]("production-override-config") .text("Optional yaml file to override default configuration for servers (production only)") .validate { path => if (!Files.exists(path)) failure(s"Override config file does not exist: '$path'") else if (!Files.isRegularFile(path)) failure(s"Override config path is not a file: '$path'") else success } - .action((path, c) => c.copy(overrideConfig = Some(path))) + .action((path, c) => c.copy(productionConfigOverride = Some(path))) opt[Path]('o', "output") .text("Where to write output files (default is '/tmp')") .validate { path => diff --git a/src/main/scala/io/flow/build/Main.scala b/src/main/scala/io/flow/build/Main.scala index ba71c54f..ba713ff2 100644 --- a/src/main/scala/io/flow/build/Main.scala +++ b/src/main/scala/io/flow/build/Main.scala @@ -39,7 +39,7 @@ object Main extends App { val allApplications: Seq[Application] = config.apis.flatMap { name => Application.parse(name) } - val serverConfigs: Seq[ServerConfig] = config.overrideConfig + val serverConfigs: Seq[ServerConfig] = config.productionConfigOverride .map { path => ServerConfig.parseFile(path) match { case Left(error) => sys.error(s"Failed to parse $path: '$error'") From 3d55cddad82414dd708297aa3db471ef5afe8074 Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 8 Oct 2025 14:14:07 +0100 Subject: [PATCH 7/9] reename --- src/main/scala/io/flow/build/Config.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/io/flow/build/Config.scala b/src/main/scala/io/flow/build/Config.scala index 77a72413..d0416f42 100644 --- a/src/main/scala/io/flow/build/Config.scala +++ b/src/main/scala/io/flow/build/Config.scala @@ -42,7 +42,7 @@ object Config { .text("Domain to use when constructing the service subdomain (default is 'api.flow.io')") .action((d, c) => c.copy(domain = d)) - opt[Path]("production-override-config") + opt[Path]("production-config-override") .text("Optional yaml file to override default configuration for servers (production only)") .validate { path => if (!Files.exists(path)) failure(s"Override config file does not exist: '$path'") From e9372042dbc3d08a6ad74848e55398f1844b022a Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 15 Oct 2025 13:29:38 +0100 Subject: [PATCH 8/9] FDN-4116 Linting of imports --- src/main/scala/io/flow/lint/Lint.scala | 1 + .../flow/lint/linters/AllImportsAreUsed.scala | 80 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala diff --git a/src/main/scala/io/flow/lint/Lint.scala b/src/main/scala/io/flow/lint/Lint.scala index 899207ed..ae712351 100644 --- a/src/main/scala/io/flow/lint/Lint.scala +++ b/src/main/scala/io/flow/lint/Lint.scala @@ -26,6 +26,7 @@ object Lint { case _ => { Seq( + linters.AllImportsAreUsed, linters.AllAttributesAreWellKnown, linters.BadNames, linters.CommonFieldTypes, diff --git a/src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala b/src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala new file mode 100644 index 00000000..9d4596de --- /dev/null +++ b/src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala @@ -0,0 +1,80 @@ +package io.flow.lint.linters + +import io.apibuilder.spec.v0.models._ +import io.flow.lint.Linter + +case class ParsedType(namespace: Option[String], category: Option[String], name: String) + +object ParsedType { + // Converts + def apply(input: String): ParsedType = { + input.lastIndexOf('.') match { + case -1 => ParsedType(None, None, input) + case lastDot => + val name = input.substring(lastDot + 1) + val beforeLast = input.substring(0, lastDot) + + beforeLast.lastIndexOf('.') match { + case -1 => + // : + sys.error(s"Unparseable type: expected either or .., got '$input'") + case secondLastDot => + val namespace = beforeLast.substring(0, secondLastDot) + val category = beforeLast.substring(secondLastDot + 1) + ParsedType(Some(namespace), Some(category), name) + } + } + } +} + +case class ReferencedType(namespace: String, name: String) + +object ReferencedType { + def from(service: Service): Seq[ReferencedType] = { + def strip(s: String): String = s.replace('[', ' ').replace(']', ' ').trim + + // Use of imported types are (almost) always fully qualified. e.g. "io.flow.common.v0.unions.Foo" + // so we can use this to determine when imports are used. Of course there is one exception, values + // in the 'annotation' field do not have to be fully qualified. + // + // See + // https://github.com/flowcommerce/api-internal/blob/main/spec-event/paypal-internal-event.json + // where the only use of the common import is the personal_data annotation. + ( + service.headers.map(_.`type`) ++: + service.interfaces.flatMap(types) ++: + service.models.flatMap(types) ++: + service.unions.flatMap(types) ++: + service.resources.flatMap(types) + ) + .map(strip) // array type [a.b] => a.b + .distinct + .map { name => + ParsedType(name).namespace match { + case Some(namespace) => ReferencedType(namespace = namespace, name) + case None => ReferencedType(namespace = service.namespace, name) + } + } + } + + private def types(m: Model): Seq[String] = m.fields.map(_.`type`) + + private def types(u: Union): Seq[String] = u.types.map(_.`type`) + + private def types(i: Interface): Seq[String] = i.fields.map(_.`type`) + + private def types(r: Resource): Seq[String] = { + val paramTypes = r.operations.flatMap(_.parameters.map(_.`type`)) + val responseTypes = r.operations.flatMap(_.responses.map(_.`type`)) + Seq(r.`type`) ++: paramTypes ++: responseTypes + } +} + +object AllImportsAreUsed extends Linter { + + override def validate(service: Service): Seq[String] = { + val referencedTypes = ReferencedType.from(service) + val unusedImports = service.imports.filterNot(i => referencedTypes.exists(_.namespace == i.namespace)) + unusedImports.map(unused => s"Unused import: '${unused.uri}'") + } +} From f47942420f1478e127ab78be2c1ba23abdb3c170 Mon Sep 17 00:00:00 2001 From: Jack Lynch Date: Wed, 15 Oct 2025 13:42:42 +0100 Subject: [PATCH 9/9] fix spec --- src/main/scala/io/flow/lint/Lint.scala | 2 +- src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/io/flow/lint/Lint.scala b/src/main/scala/io/flow/lint/Lint.scala index ae712351..c33f0089 100644 --- a/src/main/scala/io/flow/lint/Lint.scala +++ b/src/main/scala/io/flow/lint/Lint.scala @@ -26,8 +26,8 @@ object Lint { case _ => { Seq( - linters.AllImportsAreUsed, linters.AllAttributesAreWellKnown, + linters.AllImportsAreUsed, linters.BadNames, linters.CommonFieldTypes, linters.CommonParameterTypes, diff --git a/src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala b/src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala index 9d4596de..56b0f1d9 100644 --- a/src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala +++ b/src/main/scala/io/flow/lint/linters/AllImportsAreUsed.scala @@ -70,7 +70,7 @@ object ReferencedType { } } -object AllImportsAreUsed extends Linter { +case object AllImportsAreUsed extends Linter { override def validate(service: Service): Seq[String] = { val referencedTypes = ReferencedType.from(service)