Skip to content

Commit

Permalink
Merge pull request #23 from mutablelogic/dev
Browse files Browse the repository at this point in the history 
New version of seaweedfs
  • Loading branch information
@djthorpe
djthorpe authored Jul 7, 2024
2 parents 004adca + 742e8ea commit 4952961
Show file tree
Hide file tree
Showing 12 changed files with 836 additions and 209 deletions.
24 changes: 11 additions & 13 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ Time-series database, which can be placed on several nodes
TODO:
* [ ] Add TLS support


## mongodb

Document database, which can be replicated on several nodes
Expand Down Expand Up @@ -114,28 +113,28 @@ TODO:
OpenLDAP administration, for adding users and groups, and changing
passwords

* [Documentation](https://www.openldap.org/)
* [Terraform Example](_examples/openldap.tf)
* [Nomad Job](openldap/nomad/openldap.hcl)
* [Documentation](https://github.com/wheelybird/ldap-user-manager)
* [Terraform Example](_examples/openldap-admin.tf)
* [Nomad Job](postgresql/openldap-admin/openldap-admin.hcl)

TODO:
* [ ] In progress
* [ ] Add TLS support
* [ ] Add replication support
* [ ] Add custom schema support

* [ ] Add SNMP support

## PostgreSQL

PostgreSQL is a database server

* [Documentation](https://github.com/wheelybird/ldap-user-manager)
* [Terraform Example](_examples/openldap-admin.tf)
* [Nomad Job](postgresql/openldap-admin/openldap-admin.hcl)
* [Documentation](https://www.postgresql.org/)
* [Terraform Example](_examples/postgresql.tf)
* [Nomad Job](postgresql/nomad/postgresql.hcl)

TODO:
* [ ] LDAP integration
* [ ] Add TLS support
* [ ] Add SNMP support
* [ ] Add replication support
* [ ] Use volume instead when the data does not have '/' as prefix
* [ ] Add users, databases and roles support on initialization

## seaweedfs

Expand All @@ -146,7 +145,6 @@ Cluster filesystem, which can be spread across multiple nodes.
* [Nomad Job](seaweedfs/nomad/seaweedfs.hcl)

TODO:
* [ ] In progress
* [ ] A lot of testing is needed

## semaphore
Expand Down
71 changes: 36 additions & 35 deletions _examples/seaweedfs.tf
View file
Original file line number Diff line number Diff line change
@@ -1,40 +1,41 @@

// Example Cluster filesystem using seaweedfs, with one master, two volumes and three filers
module "clusterfs" {
source = "github.com/mutablelogic/tf-nomad//seaweedfs"
enabled = true // If false, no-op
dc = "datacenter" // Nomad datacenter for the cluster
namespace = "clusterfs" // Nomad namespace for the cluster
docker_tag = "latest" // Pull the latest version of the docker image every job restart
metrics = true // Provide metrics ports for prometheus pulls
webdav = false // Enable webdav service on filers
s3 = false // Enable s3 service on filers
replication = "000" // https://github.com/seaweedfs/seaweedfs/wiki/Replication#the-meaning-of-replication-type
masters = [
{
ip = "192.168.86.11"
// Example Cluster filesystem using seaweedfs, with one master, two volumes and one filer
module "seaweedfs" {
source = "github.com/mutablelogic/tf-nomad//seaweedfs"
enabled = true // If false, no-op
dc = local.datacenter
namespace = local.namespace
docker_tag = "latest" // Pull the latest version of the docker image every job restart

replication = "000" // https://github.com/seaweedfs/seaweedfs/wiki/Replication#the-meaning-of-replication-type
metrics = false // If true, allow metrics collection by prometheus

masters = {
"192.168.86.12" : {
"name" : "cm2", // Unique name identifying the master server
"data" : "/var/lib/seaweedfs" // Persistent data
}
]
volumes = [
{
ip = "192.168.86.12",
disks = "/mnt/clusterfs", // comma-separated list of mounted disks for storage
rack = "rack1", // Rack location for this server
max = 0 // Maximum number of volumes to create on this server, or 0 for auto
}, {
ip = "192.168.86.13",
disks = "/mnt/clusterfs", // comma-separated list of mounted disks for storage
rack = "rack1", // Rack location for this server
max = 0 // Maximum number of volumes to create on this server, or 0 for auto
}

volumes = {
"192.168.86.12" : {
"name" : "cm2", // Unique name identifying the volume server
"data" : ["/mnt/clusterfs"], // Location of the volume data
"rack" : "rack1", // Rack name where server is located
}
]
filers = [
{
ip = "192.168.86.11"
}, {
ip = "192.168.86.12"
}, {
ip = "192.168.86.13"
"192.168.86.13" : {
"name" : "cm3", // Unique name identifying the volume server
"data" : ["/mnt/clusterfs"], // Location of the volume data
"rack" : "rack1", // Rack name where server is located
}
}

filers = {
"192.168.86.13" : {
"name" : "cm3", // Unique name identifying the filer server
"data" : "/var/lib/seaweedfs" // Persistent data
"collection" : "drobo", // Store data in this collection
"rack" : "rack1", // Preferred rack to write data in
}
]
}
}
2 changes: 1 addition & 1 deletion openldap-admin/input.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ variable "service_provider" {
variable "service_name" {
description = "Service name"
type = string
default = "openldap-admin"
default = "openldap-admin-http"
}

variable "service_dns" {
Expand Down
2 changes: 1 addition & 1 deletion openldap-admin/nomad/openldap-admin.hcl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ variable "service_provider" {
variable "service_name" {
description = "Service name"
type = string
default = "openldap-admin"
default = "openldap-admin-http"
}

variable "service_dns" {
Expand Down
111 changes: 96 additions & 15 deletions seaweedfs/input.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,24 @@ variable "namespace" {
default = "default"
}

variable "service_provider" {
description = "Service provider, either consul or nomad"
type = string
default = "nomad"
}

variable "service_name" {
description = "Service name"
type = string
default = "seaweedfs"
}

variable "service_dns" {
description = "Service discovery DNS"
type = list(string)
default = []
}

variable "enabled" {
type = bool
description = "If false, then no job is deployed"
Expand All @@ -22,35 +40,98 @@ variable "docker_tag" {
default = "latest"
}

variable "replication" {
description = "Default replication"
type = string
default = "000"
}

variable "masters" {
type = list(object)
description = "List of master servers, needs to be an odd number, with elements: ip"
description = "Master servers"
type = map(object({
name = string // Name for the nomad job
data = optional(string) // Persisent data directories
}))
}

variable "volumes" {
type = list(object)
description = "List of volume servers, with elements: ip, disks, max, rack"
description = "Volume servers"
type = map(object({
name = string // Name for the nomad job
data = list(string) // Persisent data directories
rack = optional(string) // Rack name
public_url = optional(string) // Publicly accessible address
}))
}

variable "filers" {
type = list(object)
description = "List of filer servers, with elements: ip"
description = "Filer servers"
type = map(object({
name = string // Name for the nomad job
data = optional(string) // Persisent data directory
rack = optional(string) // Preferred rack to write data in
collection = optional(string) // Use this collection name
}))
}

variable "metrics" {
description = "Enable prometheus metrics ports"
type = bool
description = "Allocate a port on each server for pulling prometheus metrics"
default = false
}

variable "s3" {
type = bool
description = "Enable S3 service on filers"
default = false
///////////////////////////////////////////////////////////////////////////////

variable "http_port_master" {
description = "HTTP port for masters"
type = number
default = 9333
}

variable "webdav" {
type = bool
description = "Enable WebDAV service on filers"
default = false
variable "grpc_port_master" {
description = "gRPC port for masters, set as zero to use default"
type = number
default = 0
}

variable "metrics_port_master" {
description = "Prometheus metrics port for masters"
type = number
default = 9090
}

variable "http_port_volume" {
description = "HTTP port for volume servers"
type = number
default = 9334
}

variable "grpc_port_volume" {
description = "gRPC port for volume servers, set as zero to use default"
type = number
default = 0
}

variable "metrics_port_volume" {
description = "Prometheus metrics port for volume servers"
type = number
default = 9091
}

variable "http_port_filer" {
description = "HTTP port for filer servers"
type = number
default = 9335
}

variable "grpc_port_filer" {
description = "gRPC port for filer servers, set as zero to use default"
type = number
default = 0
}

variable "metrics_port_filer" {
description = "Prometheus metrics port for filer servers"
type = number
default = 9092
}
8 changes: 6 additions & 2 deletions seaweedfs/locals.tf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@

locals {
docker_image = "chrislusf/seaweedfs:${var.docker_tag}"
docker_always_pull = var.docker_tag == "latest" ? true : false
docker_image = "chrislusf/seaweedfs:${var.docker_tag}"
docker_always_pull = var.docker_tag == "latest" ? true : false
grpc_offset = 100
master_peers = [
for ip, attr in var.masters : format("%s:%d.%d", ip, var.http_port_master, var.grpc_port_master == 0 ? var.http_port_master + local.grpc_offset : var.grpc_port_master)
]
}
77 changes: 73 additions & 4 deletions seaweedfs/main.tf
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,83 @@
resource "nomad_job" "seaweedfs" {
count = var.enabled ? 1 : 0
jobspec = file("${path.module}/nomad/seaweedfs.hcl")

resource "nomad_job" "seaweedfs-master" {
for_each = var.enabled ? var.masters : {}
jobspec = templatefile("${path.module}/nomad/master.hcl", {
name = each.value.name
})

hcl2 {
allow_fs = true
vars = {
dc = jsonencode([var.dc])
namespace = var.namespace
service_provider = var.service_provider
service_name = var.service_name
service_dns = jsonencode(var.service_dns)
docker_image = local.docker_image
docker_always_pull = local.docker_always_pull
ip = each.key
peers = jsonencode(local.master_peers)
data = each.value.data
port = var.http_port_master
grpc_port = var.grpc_port_master == 0 ? var.http_port_master + local.grpc_offset : var.grpc_port_master
metrics_port = var.metrics ? var.metrics_port_master : 0
replication = var.replication
}
}
}

resource "nomad_job" "seaweedfs-volume" {
for_each = var.enabled ? var.volumes : {}
jobspec = templatefile("${path.module}/nomad/volume.hcl", {
name = each.value.name
})

hcl2 {
allow_fs = true
vars = {
dc = jsonencode([var.dc])
namespace = var.namespace
service_provider = var.service_provider
service_name = var.service_name
service_dns = jsonencode(var.service_dns)
docker_image = local.docker_image
docker_always_pull = jsonencode(local.docker_always_pull)
docker_always_pull = local.docker_always_pull
ip = each.key
masters = jsonencode(local.master_peers)
data = jsonencode(each.value.data)
port = var.http_port_volume
grpc_port = var.grpc_port_volume == 0 ? var.http_port_volume + local.grpc_offset : var.grpc_port_volume
metrics_port = var.metrics ? var.metrics_port_volume : 0
rack = each.value.rack
public_url = each.value.public_url
}
}
}

resource "nomad_job" "seaweedfs-filer" {
for_each = var.enabled ? var.filers : {}
jobspec = templatefile("${path.module}/nomad/filer.hcl", {
name = each.value.name
})

hcl2 {
allow_fs = true
vars = {
dc = jsonencode([var.dc])
namespace = var.namespace
service_provider = var.service_provider
service_name = var.service_name
service_dns = jsonencode(var.service_dns)
docker_image = local.docker_image
docker_always_pull = local.docker_always_pull
ip = each.key
masters = jsonencode(local.master_peers)
data = jsonencode(each.value.data)
port = var.http_port_filer
grpc_port = var.grpc_port_filer == 0 ? var.http_port_filer + local.grpc_offset : var.grpc_port_filer
metrics_port = var.metrics ? var.metrics_port_filer : 0
rack = each.value.rack
collection = each.value.collection
}
}
}
Loading

0 comments on commit 4952961

Please sign in to comment.