Merge pull request #232 from thanos-io/support-compactor-sharding

Support compactor hash sharding

CHANGELOG.md

@@ -20,6 +20,8 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ### Added
 - [#228](https://github.com/thanos-io/kube-thanos/pull/228) Allow configuring `--web.prefix-header` of query.
+- [#232](https://github.com/thanos-io/kube-thanos/pull/232) Support compactor hash sharding.
+
 ### Fixed
 
 ## [v0.19.0](https://github.com/thanos-io/kube-thanos/tree/v0.19.0) (2020-04-19)

all.jsonnet

@@ -56,6 +56,14 @@ local c = t.compact(commonConfig {
   deduplicationReplicaLabels: super.replicaLabels,  // reuse same labels for deduplication
 });
 
+local cs = t.compactShards(commonConfig {
+  shards: 3,
+  sourceLabels: ['cluster'],
+  replicas: 1,
+  serviceMonitor: true,
+  disableDownsampling: true,
+});
+
 local re = t.receive(commonConfig {
   replicas: 1,
   replicationFactor: 1,
@@ -208,6 +216,12 @@ local finalQ = t.query(q.config {
 
 { ['thanos-bucket-' + name]: b[name] for name in std.objectFields(b) if b[name] != null } +
 { ['thanos-compact-' + name]: c[name] for name in std.objectFields(c) if c[name] != null } +
+{
+  ['compact-' + shard + '-' + name]: cs.shards[shard][name]
+  for shard in std.objectFields(cs.shards)
+  for name in std.objectFields(cs.shards[shard])
+  if cs.shards[shard][name] != null
+} +
 { ['thanos-receive-' + name]: re[name] for name in std.objectFields(re) if re[name] != null } +
 { ['thanos-rule-' + name]: finalRu[name] for name in std.objectFields(finalRu) if finalRu[name] != null } +
 { ['thanos-sidecar-' + name]: sc[name] for name in std.objectFields(sc) if sc[name] != null } +
@@ -227,6 +241,7 @@ local finalQ = t.query(q.config {
   if strs.shards[shard][name] != null
 } +
 {
+  'compact-shards-serviceMonitor': cs.serviceMonitor,
   'store-shards-serviceMonitor': strs.serviceMonitor,
   'receive-hashrings-serviceMonitor': rcvs.serviceMonitor,
 }

examples/all/manifests/compact-shard0-service.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: database-compactor
+    app.kubernetes.io/instance: thanos-compact-0
+    app.kubernetes.io/name: thanos-compact
+    app.kubernetes.io/version: v0.19.0
+    compact.thanos.io/shard: shard-0
+  name: thanos-compact-0
+  namespace: thanos
+spec:
+  ports:
+  - name: http
+    port: 10902
+    targetPort: 10902
+  selector:
+    app.kubernetes.io/component: database-compactor
+    app.kubernetes.io/instance: thanos-compact-0
+    app.kubernetes.io/name: thanos-compact
+    compact.thanos.io/shard: shard-0

examples/all/manifests/compact-shard0-statefulSet.yaml

@@ -0,0 +1,139 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app.kubernetes.io/component: database-compactor
+    app.kubernetes.io/instance: thanos-compact-0
+    app.kubernetes.io/name: thanos-compact
+    app.kubernetes.io/version: v0.19.0
+    compact.thanos.io/shard: shard-0
+  name: thanos-compact-0
+  namespace: thanos
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: database-compactor
+      app.kubernetes.io/instance: thanos-compact-0
+      app.kubernetes.io/name: thanos-compact
+      compact.thanos.io/shard: shard-0
+  serviceName: thanos-compact-0
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: database-compactor
+        app.kubernetes.io/instance: thanos-compact-0
+        app.kubernetes.io/name: thanos-compact
+        app.kubernetes.io/version: v0.19.0
+        compact.thanos.io/shard: shard-0
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-compact
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - thanos-compact-0
+              namespaces:
+              - thanos
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      containers:
+      - args:
+        - compact
+        - --wait
+        - --log.level=info
+        - --log.format=logfmt
+        - --objstore.config=$(OBJSTORE_CONFIG)
+        - --data-dir=/var/thanos/compact
+        - --debug.accept-malformed-index
+        - --retention.resolution-raw=0d
+        - --retention.resolution-5m=0d
+        - --retention.resolution-1h=0d
+        - --delete-delay=48h
+        - --downsampling.disable
+        - |-
+          --tracing.config="config":
+            "sampler_param": 2
+            "sampler_type": "ratelimiting"
+            "service_name": "thanos-compact"
+          "type": "JAEGER"
+        - |
+          --selector.relabel-config=
+            - action: hashmod
+              source_labels: ["cluster"]
+              target_label: shard
+              modulus: 3
+            - action: keep
+              source_labels: ["shard"]
+              regex: 0
+        env:
+        - name: OBJSTORE_CONFIG
+          valueFrom:
+            secretKeyRef:
+              key: thanos.yaml
+              name: thanos-objectstorage
+        - name: HOST_IP_ADDRESS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        image: quay.io/thanos/thanos:v0.19.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
+        name: thanos-compact
+        ports:
+        - containerPort: 10902
+          name: http
+        readinessProbe:
+          failureThreshold: 20
+          httpGet:
+            path: /-/ready
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 5
+        resources:
+          limits:
+            cpu: 0.42
+            memory: 420Mi
+          requests:
+            cpu: 0.123
+            memory: 123Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /var/thanos/compact
+          name: data
+          readOnly: false
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      securityContext:
+        fsGroup: 65534
+        runAsUser: 65534
+      serviceAccountName: thanos-compact
+      terminationGracePeriodSeconds: 120
+      volumes: []
+  volumeClaimTemplates:
+  - metadata:
+      labels:
+        app.kubernetes.io/component: database-compactor
+        app.kubernetes.io/instance: thanos-compact-0
+        app.kubernetes.io/name: thanos-compact
+        compact.thanos.io/shard: shard-0
+      name: data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Gi

examples/all/manifests/compact-shard1-service.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: database-compactor
+    app.kubernetes.io/instance: thanos-compact-1
+    app.kubernetes.io/name: thanos-compact
+    app.kubernetes.io/version: v0.19.0
+    compact.thanos.io/shard: shard-1
+  name: thanos-compact-1
+  namespace: thanos
+spec:
+  ports:
+  - name: http
+    port: 10902
+    targetPort: 10902
+  selector:
+    app.kubernetes.io/component: database-compactor
+    app.kubernetes.io/instance: thanos-compact-1
+    app.kubernetes.io/name: thanos-compact
+    compact.thanos.io/shard: shard-1

examples/all/manifests/compact-shard1-statefulSet.yaml

@@ -0,0 +1,139 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app.kubernetes.io/component: database-compactor
+    app.kubernetes.io/instance: thanos-compact-1
+    app.kubernetes.io/name: thanos-compact
+    app.kubernetes.io/version: v0.19.0
+    compact.thanos.io/shard: shard-1
+  name: thanos-compact-1
+  namespace: thanos
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: database-compactor
+      app.kubernetes.io/instance: thanos-compact-1
+      app.kubernetes.io/name: thanos-compact
+      compact.thanos.io/shard: shard-1
+  serviceName: thanos-compact-1
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: database-compactor
+        app.kubernetes.io/instance: thanos-compact-1
+        app.kubernetes.io/name: thanos-compact
+        app.kubernetes.io/version: v0.19.0
+        compact.thanos.io/shard: shard-1
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/name
+                  operator: In
+                  values:
+                  - thanos-compact
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - thanos-compact-1
+              namespaces:
+              - thanos
+              topologyKey: kubernetes.io/hostname
+            weight: 100
+      containers:
+      - args:
+        - compact
+        - --wait
+        - --log.level=info
+        - --log.format=logfmt
+        - --objstore.config=$(OBJSTORE_CONFIG)
+        - --data-dir=/var/thanos/compact
+        - --debug.accept-malformed-index
+        - --retention.resolution-raw=0d
+        - --retention.resolution-5m=0d
+        - --retention.resolution-1h=0d
+        - --delete-delay=48h
+        - --downsampling.disable
+        - |-
+          --tracing.config="config":
+            "sampler_param": 2
+            "sampler_type": "ratelimiting"
+            "service_name": "thanos-compact"
+          "type": "JAEGER"
+        - |
+          --selector.relabel-config=
+            - action: hashmod
+              source_labels: ["cluster"]
+              target_label: shard
+              modulus: 3
+            - action: keep
+              source_labels: ["shard"]
+              regex: 1
+        env:
+        - name: OBJSTORE_CONFIG
+          valueFrom:
+            secretKeyRef:
+              key: thanos.yaml
+              name: thanos-objectstorage
+        - name: HOST_IP_ADDRESS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        image: quay.io/thanos/thanos:v0.19.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
+        name: thanos-compact
+        ports:
+        - containerPort: 10902
+          name: http
+        readinessProbe:
+          failureThreshold: 20
+          httpGet:
+            path: /-/ready
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 5
+        resources:
+          limits:
+            cpu: 0.42
+            memory: 420Mi
+          requests:
+            cpu: 0.123
+            memory: 123Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /var/thanos/compact
+          name: data
+          readOnly: false
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      securityContext:
+        fsGroup: 65534
+        runAsUser: 65534
+      serviceAccountName: thanos-compact
+      terminationGracePeriodSeconds: 120
+      volumes: []
+  volumeClaimTemplates:
+  - metadata:
+      labels:
+        app.kubernetes.io/component: database-compactor
+        app.kubernetes.io/instance: thanos-compact-1
+        app.kubernetes.io/name: thanos-compact
+        compact.thanos.io/shard: shard-1
+      name: data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Gi

examples/all/manifests/compact-shard2-service.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: database-compactor
+    app.kubernetes.io/instance: thanos-compact-2
+    app.kubernetes.io/name: thanos-compact
+    app.kubernetes.io/version: v0.19.0
+    compact.thanos.io/shard: shard-2
+  name: thanos-compact-2
+  namespace: thanos
+spec:
+  ports:
+  - name: http
+    port: 10902
+    targetPort: 10902
+  selector:
+    app.kubernetes.io/component: database-compactor
+    app.kubernetes.io/instance: thanos-compact-2
+    app.kubernetes.io/name: thanos-compact
+    compact.thanos.io/shard: shard-2