diff --git a/README.md b/README.md index 49a682d..2d43ab8 100755 --- a/README.md +++ b/README.md @@ -58,7 +58,6 @@ Time-series database, which can be placed on several nodes TODO: * [ ] Add TLS support - ## mongodb Document database, which can be replicated on several nodes @@ -114,28 +113,28 @@ TODO: OpenLDAP administration, for adding users and groups, and changing passwords - * [Documentation](https://www.openldap.org/) - * [Terraform Example](_examples/openldap.tf) - * [Nomad Job](openldap/nomad/openldap.hcl) + * [Documentation](https://github.com/wheelybird/ldap-user-manager) + * [Terraform Example](_examples/openldap-admin.tf) + * [Nomad Job](postgresql/openldap-admin/openldap-admin.hcl) TODO: - * [ ] In progress * [ ] Add TLS support - * [ ] Add replication support - * [ ] Add custom schema support - + * [ ] Add SNMP support ## PostgreSQL PostgreSQL is a database server - * [Documentation](https://github.com/wheelybird/ldap-user-manager) - * [Terraform Example](_examples/openldap-admin.tf) - * [Nomad Job](postgresql/openldap-admin/openldap-admin.hcl) + * [Documentation](https://www.postgresql.org/) + * [Terraform Example](_examples/postgresql.tf) + * [Nomad Job](postgresql/nomad/postgresql.hcl) TODO: + * [ ] LDAP integration * [ ] Add TLS support - * [ ] Add SNMP support + * [ ] Add replication support + * [ ] Use volume instead when the data does not have '/' as prefix + * [ ] Add users, databases and roles support on initialization ## seaweedfs @@ -146,7 +145,6 @@ Cluster filesystem, which can be spread across multiple nodes. * [Nomad Job](seaweedfs/nomad/seaweedfs.hcl) TODO: - * [ ] In progress * [ ] A lot of testing is needed ## semaphore diff --git a/_examples/seaweedfs.tf b/_examples/seaweedfs.tf index 6552563..8763d6c 100755 --- a/_examples/seaweedfs.tf +++ b/_examples/seaweedfs.tf @@ -1,40 +1,41 @@ -// Example Cluster filesystem using seaweedfs, with one master, two volumes and three filers -module "clusterfs" { - source = "github.com/mutablelogic/tf-nomad//seaweedfs" - enabled = true // If false, no-op - dc = "datacenter" // Nomad datacenter for the cluster - namespace = "clusterfs" // Nomad namespace for the cluster - docker_tag = "latest" // Pull the latest version of the docker image every job restart - metrics = true // Provide metrics ports for prometheus pulls - webdav = false // Enable webdav service on filers - s3 = false // Enable s3 service on filers - replication = "000" // https://github.com/seaweedfs/seaweedfs/wiki/Replication#the-meaning-of-replication-type - masters = [ - { - ip = "192.168.86.11" +// Example Cluster filesystem using seaweedfs, with one master, two volumes and one filer +module "seaweedfs" { + source = "github.com/mutablelogic/tf-nomad//seaweedfs" + enabled = true // If false, no-op + dc = local.datacenter + namespace = local.namespace + docker_tag = "latest" // Pull the latest version of the docker image every job restart + + replication = "000" // https://github.com/seaweedfs/seaweedfs/wiki/Replication#the-meaning-of-replication-type + metrics = false // If true, allow metrics collection by prometheus + + masters = { + "192.168.86.12" : { + "name" : "cm2", // Unique name identifying the master server + "data" : "/var/lib/seaweedfs" // Persistent data } - ] - volumes = [ - { - ip = "192.168.86.12", - disks = "/mnt/clusterfs", // comma-separated list of mounted disks for storage - rack = "rack1", // Rack location for this server - max = 0 // Maximum number of volumes to create on this server, or 0 for auto - }, { - ip = "192.168.86.13", - disks = "/mnt/clusterfs", // comma-separated list of mounted disks for storage - rack = "rack1", // Rack location for this server - max = 0 // Maximum number of volumes to create on this server, or 0 for auto + } + + volumes = { + "192.168.86.12" : { + "name" : "cm2", // Unique name identifying the volume server + "data" : ["/mnt/clusterfs"], // Location of the volume data + "rack" : "rack1", // Rack name where server is located } - ] - filers = [ - { - ip = "192.168.86.11" - }, { - ip = "192.168.86.12" - }, { - ip = "192.168.86.13" + "192.168.86.13" : { + "name" : "cm3", // Unique name identifying the volume server + "data" : ["/mnt/clusterfs"], // Location of the volume data + "rack" : "rack1", // Rack name where server is located + } + } + + filers = { + "192.168.86.13" : { + "name" : "cm3", // Unique name identifying the filer server + "data" : "/var/lib/seaweedfs" // Persistent data + "collection" : "drobo", // Store data in this collection + "rack" : "rack1", // Preferred rack to write data in } - ] + } } diff --git a/openldap-admin/input.tf b/openldap-admin/input.tf index 79d2645..e36a98d 100755 --- a/openldap-admin/input.tf +++ b/openldap-admin/input.tf @@ -31,7 +31,7 @@ variable "service_provider" { variable "service_name" { description = "Service name" type = string - default = "openldap-admin" + default = "openldap-admin-http" } variable "service_dns" { diff --git a/openldap-admin/nomad/openldap-admin.hcl b/openldap-admin/nomad/openldap-admin.hcl index 72f6241..88ebb2e 100755 --- a/openldap-admin/nomad/openldap-admin.hcl +++ b/openldap-admin/nomad/openldap-admin.hcl @@ -31,7 +31,7 @@ variable "service_provider" { variable "service_name" { description = "Service name" type = string - default = "openldap-admin" + default = "openldap-admin-http" } variable "service_dns" { diff --git a/seaweedfs/input.tf b/seaweedfs/input.tf index 020ef39..96a7eaa 100755 --- a/seaweedfs/input.tf +++ b/seaweedfs/input.tf @@ -10,6 +10,24 @@ variable "namespace" { default = "default" } +variable "service_provider" { + description = "Service provider, either consul or nomad" + type = string + default = "nomad" +} + +variable "service_name" { + description = "Service name" + type = string + default = "seaweedfs" +} + +variable "service_dns" { + description = "Service discovery DNS" + type = list(string) + default = [] +} + variable "enabled" { type = bool description = "If false, then no job is deployed" @@ -22,35 +40,98 @@ variable "docker_tag" { default = "latest" } +variable "replication" { + description = "Default replication" + type = string + default = "000" +} + variable "masters" { - type = list(object) - description = "List of master servers, needs to be an odd number, with elements: ip" + description = "Master servers" + type = map(object({ + name = string // Name for the nomad job + data = optional(string) // Persisent data directories + })) } variable "volumes" { - type = list(object) - description = "List of volume servers, with elements: ip, disks, max, rack" + description = "Volume servers" + type = map(object({ + name = string // Name for the nomad job + data = list(string) // Persisent data directories + rack = optional(string) // Rack name + public_url = optional(string) // Publicly accessible address + })) } variable "filers" { - type = list(object) - description = "List of filer servers, with elements: ip" + description = "Filer servers" + type = map(object({ + name = string // Name for the nomad job + data = optional(string) // Persisent data directory + rack = optional(string) // Preferred rack to write data in + collection = optional(string) // Use this collection name + })) } variable "metrics" { + description = "Enable prometheus metrics ports" type = bool - description = "Allocate a port on each server for pulling prometheus metrics" default = false } -variable "s3" { - type = bool - description = "Enable S3 service on filers" - default = false +/////////////////////////////////////////////////////////////////////////////// + +variable "http_port_master" { + description = "HTTP port for masters" + type = number + default = 9333 } -variable "webdav" { - type = bool - description = "Enable WebDAV service on filers" - default = false +variable "grpc_port_master" { + description = "gRPC port for masters, set as zero to use default" + type = number + default = 0 +} + +variable "metrics_port_master" { + description = "Prometheus metrics port for masters" + type = number + default = 9090 +} + +variable "http_port_volume" { + description = "HTTP port for volume servers" + type = number + default = 9334 +} + +variable "grpc_port_volume" { + description = "gRPC port for volume servers, set as zero to use default" + type = number + default = 0 +} + +variable "metrics_port_volume" { + description = "Prometheus metrics port for volume servers" + type = number + default = 9091 +} + +variable "http_port_filer" { + description = "HTTP port for filer servers" + type = number + default = 9335 +} + +variable "grpc_port_filer" { + description = "gRPC port for filer servers, set as zero to use default" + type = number + default = 0 +} + +variable "metrics_port_filer" { + description = "Prometheus metrics port for filer servers" + type = number + default = 9092 } diff --git a/seaweedfs/locals.tf b/seaweedfs/locals.tf index 7367ff4..f1d0982 100755 --- a/seaweedfs/locals.tf +++ b/seaweedfs/locals.tf @@ -1,5 +1,9 @@ locals { - docker_image = "chrislusf/seaweedfs:${var.docker_tag}" - docker_always_pull = var.docker_tag == "latest" ? true : false + docker_image = "chrislusf/seaweedfs:${var.docker_tag}" + docker_always_pull = var.docker_tag == "latest" ? true : false + grpc_offset = 100 + master_peers = [ + for ip, attr in var.masters : format("%s:%d.%d", ip, var.http_port_master, var.grpc_port_master == 0 ? var.http_port_master + local.grpc_offset : var.grpc_port_master) + ] } diff --git a/seaweedfs/main.tf b/seaweedfs/main.tf index ae6a3e7..e8e3a95 100755 --- a/seaweedfs/main.tf +++ b/seaweedfs/main.tf @@ -1,14 +1,83 @@ -resource "nomad_job" "seaweedfs" { - count = var.enabled ? 1 : 0 - jobspec = file("${path.module}/nomad/seaweedfs.hcl") + +resource "nomad_job" "seaweedfs-master" { + for_each = var.enabled ? var.masters : {} + jobspec = templatefile("${path.module}/nomad/master.hcl", { + name = each.value.name + }) + + hcl2 { + allow_fs = true + vars = { + dc = jsonencode([var.dc]) + namespace = var.namespace + service_provider = var.service_provider + service_name = var.service_name + service_dns = jsonencode(var.service_dns) + docker_image = local.docker_image + docker_always_pull = local.docker_always_pull + ip = each.key + peers = jsonencode(local.master_peers) + data = each.value.data + port = var.http_port_master + grpc_port = var.grpc_port_master == 0 ? var.http_port_master + local.grpc_offset : var.grpc_port_master + metrics_port = var.metrics ? var.metrics_port_master : 0 + replication = var.replication + } + } +} + +resource "nomad_job" "seaweedfs-volume" { + for_each = var.enabled ? var.volumes : {} + jobspec = templatefile("${path.module}/nomad/volume.hcl", { + name = each.value.name + }) hcl2 { allow_fs = true vars = { dc = jsonencode([var.dc]) namespace = var.namespace + service_provider = var.service_provider + service_name = var.service_name + service_dns = jsonencode(var.service_dns) docker_image = local.docker_image - docker_always_pull = jsonencode(local.docker_always_pull) + docker_always_pull = local.docker_always_pull + ip = each.key + masters = jsonencode(local.master_peers) + data = jsonencode(each.value.data) + port = var.http_port_volume + grpc_port = var.grpc_port_volume == 0 ? var.http_port_volume + local.grpc_offset : var.grpc_port_volume + metrics_port = var.metrics ? var.metrics_port_volume : 0 + rack = each.value.rack + public_url = each.value.public_url } } } + +resource "nomad_job" "seaweedfs-filer" { + for_each = var.enabled ? var.filers : {} + jobspec = templatefile("${path.module}/nomad/filer.hcl", { + name = each.value.name + }) + + hcl2 { + allow_fs = true + vars = { + dc = jsonencode([var.dc]) + namespace = var.namespace + service_provider = var.service_provider + service_name = var.service_name + service_dns = jsonencode(var.service_dns) + docker_image = local.docker_image + docker_always_pull = local.docker_always_pull + ip = each.key + masters = jsonencode(local.master_peers) + data = jsonencode(each.value.data) + port = var.http_port_filer + grpc_port = var.grpc_port_filer == 0 ? var.http_port_filer + local.grpc_offset : var.grpc_port_filer + metrics_port = var.metrics ? var.metrics_port_filer : 0 + rack = each.value.rack + collection = each.value.collection + } + } +} \ No newline at end of file diff --git a/seaweedfs/nomad/filer.hcl b/seaweedfs/nomad/filer.hcl new file mode 100644 index 0000000..b34ca35 --- /dev/null +++ b/seaweedfs/nomad/filer.hcl @@ -0,0 +1,207 @@ + +/////////////////////////////////////////////////////////////////////////////// +// VARIABLES + +variable "dc" { + description = "data centers that the job runs in" + type = list(string) +} + +variable "namespace" { + description = "namespace that the job runs in" + type = string + default = "default" +} + +variable "service_provider" { + description = "Service provider, either consul or nomad" + type = string + default = "nomad" +} + +variable "service_name" { + description = "Service name" + type = string + default = "seaweedfs" +} + +variable "service_dns" { + description = "Service discovery DNS" + type = list(string) + default = [] +} + +variable "docker_image" { + description = "Docker image" + type = string +} + +variable "docker_always_pull" { + description = "Pull docker image on every job restart" + type = bool + default = false +} + +variable "ip" { + description = "Filer server IP address" + type = string +} + +variable "masters" { + description = "Master server IP addresses" + type = list(string) +} + +variable "data" { + description = "Persistent data path" + type = string +} + +variable "port" { + description = "HTTP port" + type = number +} + +variable "grpc_port" { + description = "gRPC port" + type = number + default = 0 +} + +variable "metrics_port" { + description = "Prometheus metrics port " + type = number + default = 0 +} + +variable "rack" { + description = "Prefer to write to volumes in this rack" + type = string + default = "" +} + +variable "collection" { + description = "All data will be stored in this collection" + type = string + default = "default" +} + +/////////////////////////////////////////////////////////////////////////////// +// LOCALS + +locals { + // gRPC offset if the port is not set explicitly + grpc_offset = 10000 + // Service + service = "filer" +} + +/////////////////////////////////////////////////////////////////////////////// +// JOB + +job "seaweedfs-filer-${ name }" { + type = "service" + datacenters = var.dc + namespace = var.namespace + + update { + min_healthy_time = "10s" + healthy_deadline = "5m" + health_check = "task_states" + } + + /////////////////////////////////////////////////////////////////////////////// + + group "filer" { + count = 1 + + constraint { + attribute = var.ip + operator = "set_contains" + value = "$${attr.unique.network.ip-address}" + } + + network { + mode = "host" + + // HTTP port is always exposed + port "http" { + static = var.port + to = var.port + } + + // gRPC port is always exposed + port "grpc" { + static = var.grpc_port == 0 ? var.port + local.grpc_offset : var.grpc_port + to = var.grpc_port == 0 ? var.port + local.grpc_offset : var.grpc_port + } + + // metrics port is only exposed if the volume is configured to use it + dynamic "port" { + for_each = var.metrics_port > 0 ? [1] : [] + labels = ["metrics"] + content { + static = var.metrics_port + to = var.metrics_port + } + } + } + + service { + tags = ["http", var.service_name, local.service] + name = format("%s-%s-http", local.service, var.service_name) + port = "http" + provider = var.service_provider + } + + service { + tags = ["grpc", var.service_name, local.service] + name = format("%s-%s-grpc", local.service,var.service_name) + port = "grpc" + provider = var.service_provider + } + + dynamic "service" { + for_each = var.metrics_port > 0 ? [1] : [] + content { + tags = ["metrics", var.service_name, local.service] + name = format("%s-%s-metrics", local.service,var.service_name) + port = "metrics" + provider = var.service_provider + } + } + + task "filer" { + driver = "docker" + config { + image = var.docker_image + force_pull = var.docker_always_pull + dns_servers = var.service_dns + args = compact([ + "-logtostderr", + "filer", + "-ip=$${node.unique.name}", + "-ip.bind=0.0.0.0", + "-port=$${NOMAD_PORT_http}", + "-port.grpc=$${NOMAD_PORT_grpc}", + format("-master=%s", join(",", var.masters)), + var.metrics_port == 0 ? "" : "-metricsPort=$${NOMAD_PORT_metrics}", + "-defaultStoreDir=/data", + var.rack == "" ? "" : format("-rack=%s", var.rack), + var.collection == "" ? "" : format("-collection=%s", var.collection), + ]) + volumes = compact([ + var.data == "" ? "" : format("%s:/data", var.data), + ]) + ports = compact([ + "http", + "grpc", + var.metrics_port > 0 ? "metrics" : "", + ]) + } // config + } // task "filer" + } // group "filer" + + /////////////////////////////////////////////////////////////////////////////// + +} // job "seaweedfs" diff --git a/seaweedfs/nomad/master.hcl b/seaweedfs/nomad/master.hcl new file mode 100644 index 0000000..c807f98 --- /dev/null +++ b/seaweedfs/nomad/master.hcl @@ -0,0 +1,195 @@ + +/////////////////////////////////////////////////////////////////////////////// +// VARIABLES + +variable "dc" { + description = "data centers that the job runs in" + type = list(string) +} + +variable "namespace" { + description = "namespace that the job runs in" + type = string + default = "default" +} + +variable "service_provider" { + description = "Service provider, either consul or nomad" + type = string + default = "nomad" +} + +variable "service_name" { + description = "Service name" + type = string + default = "seaweedfs" +} + +variable "service_dns" { + description = "Service discovery DNS" + type = list(string) + default = [] +} + +variable "docker_image" { + description = "Docker image" + type = string +} + +variable "docker_always_pull" { + description = "Pull docker image on every job restart" + type = bool + default = false +} + +variable "ip" { + description = "Master server IP address" + type = string +} + +variable "data" { + description = "Persistent data directory" + type = string + default = "" +} + +variable "peers" { + description = "Peers who are master servers" + type = list(string) +} + +variable "port" { + description = "HTTP port" + type = number +} + +variable "grpc_port" { + description = "gRPC port" + type = number +} + +variable "metrics_port" { + description = "Prometheus metrics port" + type = number +} + +variable "replication" { + description = "Default replication" + type = string + default = "000" +} + +/////////////////////////////////////////////////////////////////////////////// +// LOCALS + +locals { + // gRPC offset if the port is not set explicitly + grpc_offset = 10000 +} + +/////////////////////////////////////////////////////////////////////////////// +// JOB + +job "seaweedfs-master-${ name }" { + type = "service" + datacenters = var.dc + namespace = var.namespace + + update { + min_healthy_time = "10s" + healthy_deadline = "5m" + health_check = "task_states" + } + + /////////////////////////////////////////////////////////////////////////////// + // GROUP + + group "master" { + count = 1 + + constraint { + attribute = var.ip + operator = "set_contains" + value = "$${attr.unique.network.ip-address}" + } + + network { + mode = "host" + + // HTTP port is always exposed + port "http" { + static = var.port + to = var.port + } + + // gRPC port is always exposed + port "grpc" { + static = var.grpc_port == 0 ? var.port + local.grpc_offset : var.grpc_port + to = var.grpc_port == 0 ? var.port + local.grpc_offset : var.grpc_port + } + + // metrics port is only exposed if the volume is configured to use it + dynamic "port" { + for_each = var.metrics_port > 0 ? [1] : [] + labels = ["metrics"] + content { + static = var.metrics_port + to = var.metrics_port + } + } + } + + service { + tags = ["http", var.service_name, "master"] + name = format("%s-master-http", var.service_name) + port = "http" + provider = var.service_provider + } + + service { + tags = ["grpc", var.service_name, "master"] + name = format("%s-master-grpc", var.service_name) + port = "grpc" + provider = var.service_provider + } + + dynamic "service" { + for_each = var.metrics_port > 0 ? [1] : [] + content { + tags = ["metrics", var.service_name, "master"] + name = format("%s-master-metrics", var.service_name) + port = "metrics" + provider = var.service_provider + } + } + + task "master" { + driver = "docker" + config { + image = var.docker_image + force_pull = var.docker_always_pull + dns_servers = var.service_dns + args = compact([ + "-logtostderr", + "master", + "-ip=$${node.unique.name}", + "-ip.bind=0.0.0.0", + "-port=$${NOMAD_PORT_http}", + "-port.grpc=$${NOMAD_PORT_grpc}", + length(var.peers) <= 1 ? "" : format("-peers=%s", join(",", var.peers)), + var.replication == "" ? "" : format("-defaultReplication=%s", var.replication), + var.metrics_port == 0 ? "" : "-metricsPort=$${NOMAD_PORT_metrics}", + "-mdir=/data", + ]) + volumes = compact([ + var.data == "" ? "" : format("%s:/data", var.data), + ]) + ports = compact([ + "http", + "grpc", + var.metrics_port > 0 ? "metrics" : "", + ]) + } + } // task "master" + } // group "master" +} // job "seaweedfs-master" diff --git a/seaweedfs/nomad/seaweedfs.hcl b/seaweedfs/nomad/seaweedfs.hcl deleted file mode 100755 index baea84a..0000000 --- a/seaweedfs/nomad/seaweedfs.hcl +++ /dev/null @@ -1,138 +0,0 @@ - -/////////////////////////////////////////////////////////////////////////////// -// VARIABLES - -variable "dc" { - description = "data centers that the job runs in" - type = list(string) -} - -variable "namespace" { - description = "namespace that the job runs in" - type = string - default = "default" -} - -variable "service_provider" { - description = "Service provider, either consul or nomad" - type = string - default = "nomad" -} - -variable "docker_image" { - description = "Docker image" - type = string -} - -variable "docker_always_pull" { - description = "Pull docker image on every job restart" - type = bool - default = false -} - -variable "masters" { - description = "Master servers" - type = list(object({ ip = string, data = string, replication = string, http_port = number, grpc_port = number, metrics_port = number })) -} - -variable "volumes" { - description = "Volume servers" - type = list(object({ ip = string, data = string, rack = string, max = number, http_port = number, grpc_port = number, metrics_port = number })) -} - -variable "filers" { - description = "Filer servers" - type = list(object({ ip = string, data = string, http_port = number, grpc_port = number, metrics_port = number, s3_port = number, webdav_port = number })) -} - -/////////////////////////////////////////////////////////////////////////////// -// LOCALS - -locals { - grpc_offset = 10000 - master_ips = [for master in var.masters : master.ip] - master_addrs_http = [for master in var.masters : format("%s:%d", master.ip, var.master_port)] - master_addrs_http_grpc = [for master in var.masters : format("%s:%d.%d", master.ip, var.master_port, var.master_port + local.grpc_offset)] - master_data = { for master in var.masters : master.ip => master.data } - master_metrics_port = { for master in var.masters : master.ip => master.metrics_port } -} - -/////////////////////////////////////////////////////////////////////////////// -// JOB - -job "seaweedfs" { - type = "service" - datacenters = var.dc - namespace = var.namespace - - update { - min_healthy_time = "10s" - healthy_deadline = "5m" - health_check = "task_states" - } - -/////////////////////////////////////////////////////////////////////////////// - - group "master" { - count = length(var.masters) - - constraint { - attribute = attr.unique.network.ip-address - operator = "set_contains_any" - value = join(",", local.master_ips) - } - - network { - mode = "host" - port "http" { - static = var.master_port - to = var.master_port - } - port "grpc" { - static = var.master_port + local.grpc_offset - to = var.master_port + local.grpc_offset - } - } - - service { - tags = ["http", "seedweedfs", "master"] - name = "seaweedfs-master-http" - port = "http" - provider = var.service_provider - } - - service { - tags = ["grpc", "seedweedfs", "master"] - name = "seaweedfs-master-grpc" - port = "grpc" - provider = var.service_provider - } - - task "weed-master" { - driver = "docker" - - config { - image = var.docker_image - force_pull = var.docker_always_pull - args = compact([ - "-logtostderr", - "master", - "-ip=${NOMAD_IP_http}", - "-ip.bind=0.0.0.0", - "-peers=${join(",", local.master_addrs_http)}", - master_data["${NOMAD_IP_http}"] && master_data["${NOMAD_IP_http}"].data ? "-mdir=/data" : "", - "-port=${NOMAD_PORT_http}", - "-port.grpc=${NOMAD_PORT_grpc}" - ]) - volumes = compact([ - master_data["${NOMAD_IP_http}"] && master_data["${NOMAD_IP_http}"].data ? format("%s:/data", var.data) : "", - ]) - ports = ["http", "grpc"] - privileged = true - } - - } // task "weed-master" - - } // group "master" - -} // job "seaweedfs" diff --git a/seaweedfs/nomad/volume.hcl b/seaweedfs/nomad/volume.hcl new file mode 100644 index 0000000..2de8cd8 --- /dev/null +++ b/seaweedfs/nomad/volume.hcl @@ -0,0 +1,210 @@ + +/////////////////////////////////////////////////////////////////////////////// +// VARIABLES + +variable "dc" { + description = "data centers that the job runs in" + type = list(string) +} + +variable "namespace" { + description = "namespace that the job runs in" + type = string + default = "default" +} + +variable "service_provider" { + description = "Service provider, either consul or nomad" + type = string + default = "nomad" +} + +variable "service_name" { + description = "Service name" + type = string + default = "seaweedfs" +} + +variable "service_dns" { + description = "Service discovery DNS" + type = list(string) + default = [] +} + +variable "docker_image" { + description = "Docker image" + type = string +} + +variable "docker_always_pull" { + description = "Pull docker image on every job restart" + type = bool + default = false +} + +variable "ip" { + description = "Volume server IP address" + type = string +} + +variable "masters" { + description = "Master server IP addresses" + type = list(string) +} + +variable "data" { + description = "Persistent data paths" + type = list(string) +} + +variable "port" { + description = "HTTP port" + type = number +} + +variable "grpc_port" { + description = "gRPC port" + type = number + default = 0 +} + +variable "metrics_port" { + description = "Prometheus metrics port " + type = number + default = 0 +} + +variable "rack" { + description = "Rack for volume server" + type = string + default = "default" +} + +variable "public_url" { + description = "Publicly accessible address" + type = string + default = "" +} + +/////////////////////////////////////////////////////////////////////////////// +// LOCALS + +locals { + // gRPC offset if the port is not set explicitly + grpc_offset = 10000 + + // Use max=0 for each data path + data_dir = compact([for i, v in var.data : format("/data%s", i)]) + data_volume = compact([for i, v in var.data : v == "" ? "" : format("%s:/data%s", v, i)]) + data_max = compact([for i in var.data : 0]) +} + +/////////////////////////////////////////////////////////////////////////////// +// JOB + +job "seaweedfs-volume-${ name }" { + type = "service" + datacenters = var.dc + namespace = var.namespace + + update { + min_healthy_time = "10s" + healthy_deadline = "5m" + health_check = "task_states" + } + + /////////////////////////////////////////////////////////////////////////////// + + group "volume" { + count = 1 + + constraint { + attribute = var.ip + operator = "set_contains" + value = "$${attr.unique.network.ip-address}" + } + + network { + mode = "host" + + // HTTP port is always exposed + port "http" { + static = var.port + to = var.port + } + + // gRPC port is always exposed + port "grpc" { + static = var.grpc_port == 0 ? var.port + local.grpc_offset : var.grpc_port + to = var.grpc_port == 0 ? var.port + local.grpc_offset : var.grpc_port + } + + // metrics port is only exposed if the volume is configured to use it + dynamic "port" { + for_each = var.metrics_port > 0 ? [1] : [] + labels = ["metrics"] + content { + static = var.metrics_port + to = var.metrics_port + } + } + } + + service { + tags = ["http", var.service_name, "volume"] + name = format("%s-volume-http", var.service_name) + port = "http" + provider = var.service_provider + } + + service { + tags = ["grpc", var.service_name, "volume"] + name = format("%s-volume-grpc", var.service_name) + port = "grpc" + provider = var.service_provider + } + + dynamic "service" { + for_each = var.metrics_port > 0 ? [1] : [] + content { + tags = ["metrics", var.service_name, "volume"] + name = format("%s-volume-metrics", var.service_name) + port = "metrics" + provider = var.service_provider + } + } + + task "volume" { + driver = "docker" + config { + image = var.docker_image + force_pull = var.docker_always_pull + dns_servers = var.service_dns + args = compact([ + "-logtostderr", + "volume", + "-ip=$${node.unique.name}", + "-ip.bind=0.0.0.0", + "-port=$${NOMAD_PORT_http}", + "-port.grpc=$${NOMAD_PORT_grpc}", + format("-mserver=%s", join(",", var.masters)), + format("-dir=%s", join(",", local.data_dir)), + format("-max=%s", join(",", local.data_max)), + var.metrics_port == 0 ? "" : "-metricsPort=$${NOMAD_PORT_metrics}", + "-dataCenter=$${NOMAD_DC}", + "-rack=$${var.rack}", + var.public_url == "" ? "" : "-publicUrl=$${var.public_url}" + ]) + volumes = local.data_volume + ports = compact([ + "http", + "grpc", + var.metrics_port > 0 ? "metrics" : "", + ]) + } + } // task "volume" + } // group "volume" + + /////////////////////////////////////////////////////////////////////////////// + +} // job "seaweedfs" diff --git a/seaweedfs/output.tf b/seaweedfs/output.tf deleted file mode 100755 index e69de29..0000000