Merge pull request #1318 from graphcool/graphql-database

[WIP] Prisma

server/.buildkite/pipeline.yml

@@ -2,41 +2,26 @@ steps:
   - label: ":scala: libs"
     command: cd server && ./scripts/test.sh libs
 
-  - label: ":scala: backend-api-relay"
-    command: cd server && ./scripts/test.sh backend-api-relay
+  - label: ":scala: subscriptions"
+    command: cd server && ./scripts/test.sh subscriptions
 
-  - label: ":scala: backend-api-simple"
-    command: cd server && ./scripts/test.sh backend-api-simple
+  - label: ":scala: deploy"
+    command: cd server && ./scripts/test.sh deploy
 
-  - label: ":scala: backend-api-system"
-    command: cd server && ./scripts/test.sh backend-api-system
+  - label: ":scala: api"
+    command: cd server && ./scripts/test.sh api
 
-  - label: ":scala: backend-api-simple-subscriptions"
-    command: cd server && ./scripts/test.sh backend-api-simple-subscriptions
-
-  - label: ":scala: backend-api-subscriptions-websocket"
-    command: cd server && ./scripts/test.sh backend-api-subscriptions-websocket
-
-  - label: ":scala: backend-api-fileupload"
-    command: cd server && ./scripts/test.sh backend-api-fileupload
-
-  - label: ":scala: backend-api-schema-manager"
-    command: cd server && ./scripts/test.sh backend-api-schema-manager
-
-  - label: ":scala: backend-shared"
-    command: cd server && ./scripts/test.sh backend-shared
-
-  - label: ":scala: client-shared"
-    command: cd server && ./scripts/test.sh client-shared
-
-  - label: ":scala: backend-workers"
-    command: cd server && ./scripts/test.sh backend-workers
+  - label: ":scala: single-server"
+    command: cd server && ./scripts/test.sh single-server
 
   - wait
 
-  - label: ":sbt: Publish JARs"
-    command: ./server/scripts/publish-jars.sh
-
   - label: ":docker: Build"
     command: ./server/scripts/docker-build.sh
-    branches: master
+    branches: graphql-database
+
+  - wait
+
+  - label: ":llama: Deploy"
+    command: ./server/scripts/beta_deploy.sh
+    branches: graphql-database

server/.envrc

@@ -0,0 +1,27 @@
+export PORT=9000
+export SCHEMA_MANAGER_SECRET=MUCHSECRET
+export CLUSTER_ADDRESS="http://localhost:${PORT}"
+
+export SQL_CLIENT_HOST="127.0.0.1"
+export SQL_CLIENT_PORT="3306"
+export SQL_CLIENT_USER="root"
+export SQL_CLIENT_PASSWORD="graphcool"
+export SQL_CLIENT_CONNECTION_LIMIT=10
+
+export SQL_LOGS_HOST="127.0.0.1"
+export SQL_LOGS_PORT="3306"
+export SQL_LOGS_USER="root"
+export SQL_LOGS_PASSWORD="graphcool"
+export SQL_LOGS_DATABASE="logs"
+export SQL_LOGS_CONNECTION_LIMIT=10
+
+export SQL_INTERNAL_HOST="127.0.0.1"
+export SQL_INTERNAL_PORT="3306"
+export SQL_INTERNAL_USER="root"
+export SQL_INTERNAL_PASSWORD="graphcool"
+export SQL_INTERNAL_DATABASE="graphcool"
+export SQL_INTERNAL_CONNECTION_LIMIT=10
+
+export CLUSTER_VERSION=local
+export BUGSNAG_API_KEY="empty"
+export RABBITMQ_URI=""

server/.gitignore

@@ -3,7 +3,6 @@
 *.class
 *.log
 .coursier
-.envrc
 
 .ivy2
 

server/Makefile

@@ -0,0 +1,5 @@
+dev:
+	docker-compose -f docker-compose/dev.yml up -d --remove-orphans
+
+dev-down:
+	docker-compose -f docker-compose/dev.yml down -v --remove-orphans

server/api/src/main/resources/application.conf

@@ -0,0 +1,38 @@
+akka {
+  loglevel = INFO
+  http.server {
+    parsing.max-uri-length = 50k
+    parsing.max-header-value-length = 50k
+    remote-address-header = on
+    request-timeout = 45s
+  }
+  http.host-connection-pool {
+    // see http://doc.akka.io/docs/akka-http/current/scala/http/client-side/pool-overflow.html
+    // and http://doc.akka.io/docs/akka-http/current/java/http/configuration.html
+    // These settings are relevant for Region Proxy Synchronous Request Pipeline functions and ProjectSchemaFetcher
+    max-connections = 64 // default is 4, but we have multiple servers behind lb, so need many connections to single host
+    max-open-requests = 2048 // default is 32, but we need to handle spikes
+  }
+  http.client {
+    parsing.max-content-length = 50m
+  }
+}
+
+clientDatabases {
+  client1 {
+    master {
+      connectionInitSql="set names utf8mb4"
+      dataSourceClass = "slick.jdbc.DriverDataSource"
+      properties {
+        url = "jdbc:mysql://"${SQL_CLIENT_HOST}":"${SQL_CLIENT_PORT}"/?autoReconnect=true&useSSL=false&serverTimeZone=UTC&useUnicode=true&characterEncoding=UTF-8&usePipelineAuth=false"
+        user = ${SQL_CLIENT_USER}
+        password = ${SQL_CLIENT_PASSWORD}
+      }
+      numThreads = ${SQL_CLIENT_CONNECTION_LIMIT}
+      connectionTimeout = 5000
+    }
+  }
+}
+
+schemaManagerEndpoint = ${CLUSTER_ADDRESS}/cluster/schema
+schemaManagerSecret = ${SCHEMA_MANAGER_SECRET}

server/api/src/main/scala/cool/graph/api/ApiDependencies.scala

@@ -0,0 +1,71 @@
+package cool.graph.api
+
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
+import com.prisma.errors.{BugsnagErrorReporter, ErrorReporter}
+import com.typesafe.config.{Config, ConfigFactory}
+import cool.graph.api.database.deferreds.DeferredResolverProvider
+import cool.graph.api.database.{DataResolver, Databases}
+import cool.graph.api.project.{ProjectFetcher, ProjectFetcherImpl}
+import cool.graph.api.schema.{ApiUserContext, SchemaBuilder}
+import cool.graph.api.server.RequestHandler
+import cool.graph.api.subscriptions.Webhook
+import cool.graph.auth.{Auth, AuthImpl}
+import cool.graph.client.server.{GraphQlRequestHandler, GraphQlRequestHandlerImpl}
+import cool.graph.messagebus.pubsub.inmemory.InMemoryAkkaPubSub
+import cool.graph.messagebus.queue.inmemory.InMemoryAkkaQueue
+import cool.graph.messagebus.{PubSubPublisher, Queue}
+import cool.graph.shared.models.Project
+import cool.graph.utils.await.AwaitUtils
+
+import scala.concurrent.ExecutionContext
+
+trait ApiDependencies extends AwaitUtils {
+  implicit def self: ApiDependencies
+
+  val config: Config = ConfigFactory.load()
+
+  implicit val system: ActorSystem
+  val materializer: ActorMaterializer
+  def projectFetcher: ProjectFetcher
+  def apiSchemaBuilder: SchemaBuilder
+  def databases: Databases
+  def webhookPublisher: Queue[Webhook]
+
+  implicit lazy val executionContext: ExecutionContext  = system.dispatcher
+  implicit lazy val reporter: ErrorReporter             = BugsnagErrorReporter(sys.env("BUGSNAG_API_KEY"))
+  lazy val log: String => Unit                          = println
+  lazy val graphQlRequestHandler: GraphQlRequestHandler = GraphQlRequestHandlerImpl(log)
+  lazy val auth: Auth                                   = AuthImpl
+  lazy val requestHandler: RequestHandler               = RequestHandler(projectFetcher, apiSchemaBuilder, graphQlRequestHandler, auth, log)
+  lazy val maxImportExportSize: Int                     = 10000000
+
+  val sssEventsPubSub: InMemoryAkkaPubSub[String]
+  lazy val sssEventsPublisher: PubSubPublisher[String] = sssEventsPubSub
+
+  def dataResolver(project: Project): DataResolver       = DataResolver(project)
+  def masterDataResolver(project: Project): DataResolver = DataResolver(project, useMasterDatabaseOnly = true)
+  def deferredResolverProvider(project: Project)         = new DeferredResolverProvider[ApiUserContext](dataResolver(project))
+
+  def destroy = {
+    println("ApiDependencies [DESTROY]")
+    databases.master.shutdown.await()
+    databases.readOnly.shutdown.await()
+    materializer.shutdown()
+    system.terminate().await()
+  }
+}
+
+case class ApiDependenciesImpl(sssEventsPubSub: InMemoryAkkaPubSub[String])(implicit val system: ActorSystem, val materializer: ActorMaterializer)
+    extends ApiDependencies {
+  override implicit def self: ApiDependencies = this
+
+  val databases        = Databases.initialize(config)
+  val apiSchemaBuilder = SchemaBuilder()(system, this)
+  val projectFetcher: ProjectFetcher = {
+    val schemaManagerEndpoint = config.getString("schemaManagerEndpoint")
+    val schemaManagerSecret   = config.getString("schemaManagerSecret")
+    ProjectFetcherImpl(Vector.empty, config, schemaManagerEndpoint = schemaManagerEndpoint, schemaManagerSecret = schemaManagerSecret)
+  }
+  override val webhookPublisher = InMemoryAkkaQueue[Webhook]()
+}

server/api/src/main/scala/cool/graph/api/ApiMain.scala

@@ -0,0 +1,19 @@
+package cool.graph.api
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
+import com.typesafe.scalalogging.LazyLogging
+import cool.graph.akkautil.http.ServerExecutor
+import cool.graph.api.schema.SchemaBuilder
+import cool.graph.api.server.ApiServer
+import cool.graph.messagebus.pubsub.inmemory.InMemoryAkkaPubSub
+
+object ApiMain extends App with LazyLogging {
+  implicit val system          = ActorSystem("api-main")
+  implicit val materializer    = ActorMaterializer()
+  implicit val apiDependencies = new ApiDependenciesImpl(InMemoryAkkaPubSub[String]())
+
+  val schemaBuilder = SchemaBuilder()
+  val server        = ApiServer(schemaBuilder = schemaBuilder)
+
+  ServerExecutor(9000, server).startBlocking()
+}

server/api/src/main/scala/cool/graph/api/ApiMetrics.scala

@@ -0,0 +1,33 @@
+package cool.graph.api
+
+import com.prisma.errors.BugsnagErrorReporter
+import cool.graph.metrics.{CustomTag, MetricsManager}
+import cool.graph.profiling.MemoryProfiler
+
+object ApiMetrics extends MetricsManager {
+  // this is intentionally empty. Since we don't define metrics here, we need to load the object once so the profiler kicks in.
+  // This way it does not look so ugly on the caller side.
+  def init(): Unit = {}
+
+  implicit val reporter = BugsnagErrorReporter(sys.env.getOrElse("BUGSNAG_API_KEY", ""))
+
+  // CamelCase the service name read from env
+  override def serviceName =
+    sys.env
+      .getOrElse("SERVICE_NAME", "SystemShared")
+      .split("-")
+      .map { x =>
+        x.head.toUpper + x.tail
+      }
+      .mkString
+
+  MemoryProfiler.schedule(this)
+
+  val projectCacheGetCount          = defineCounter("projectCacheGetCount")
+  val projectCacheMissCount         = defineCounter("projectCacheMissCount")
+  val schemaBuilderBuildTimerMetric = defineTimer("schemaBuilderBuildTimer", CustomTag("projectId", recordingThreshold = 600))
+  val sqlQueryTimer                 = defineTimer("sqlQueryTimer", CustomTag("projectId", recordingThreshold = 1000), CustomTag("queryName", recordingThreshold = 1000))
+  val sqlDataChangeMutactionTimer   = defineTimer("sqlDataChangeMutactionTimer", CustomTag("projectId", recordingThreshold = 1000))
+  val requestDuration               = defineTimer("requestDuration", CustomTag("projectId", recordingThreshold = 1500))
+
+}