LESSON 30: SECRETS MANAGEMENT WITH HASHICORP VAULT

Deploy Microservices On-Premises with Kubernetes HA

Part 7: GitOps with ArgoCD, Helm & Vault

xdev.asia

🎯 LESSON OBJECTIVE__HTMLTAG_68___

• ✅ Deploy Vault HA cluster on Kubernetes__HTMLTAG_71___
• ✅ KV Secrets Engine v2
• ✅ Kubernetes Auth Method
• ✅ Dynamic Database Credentials
• ✅ External Secrets Operator (ESO) integration
• ✅ Secret rotation strategy__HTMLTAG_81___

---

PART 1: VAULT ARCHITECTURE

HashiCorp Vault HA on Kubernetes:

┌─────────────────────────────────────────────────┐
│              Vault HA Cluster                    │
│                                                  │
│  ┌──────────┐  ┌──────────┐  ┌──────────┐      │
│  │ Vault-0  │  │ Vault-1  │  │ Vault-2  │      │
│  │ (Active) │  │(Standby) │  │(Standby) │      │
│  │          │  │          │  │          │      │
│  │  Seal/   │  │  Seal/   │  │  Seal/   │      │
│  │  Unseal  │  │  Unseal  │  │  Unseal  │      │
│  └────┬─────┘  └────┬─────┘  └────┬─────┘      │
│       │              │              │            │
│  ┌────▼──────────────▼──────────────▼──────┐    │
│  │        Raft Integrated Storage           │    │
│  │        (replicated across pods)          │    │
│  └──────────────────────────────────────────┘    │
└──────────────────────────────────────────────────┘
          │
          ▼
┌──────────────────┐      ┌──────────────────┐
│ K8s Auth Method  │      │ External Secrets │
│ (ServiceAccount) │      │ Operator (ESO)   │
└──────────────────┘      └──────────────────┘
          │                         │
          ▼                         ▼
┌──────────────────┐      ┌──────────────────┐
│ Application Pod  │      │ K8s Secret       │
│ (inject secrets) │      │ (synced from     │
│                  │      │  Vault)          │
└──────────────────┘      └──────────────────┘

PART 2: INSTALL VAULT HA

# Add HashiCorp Helm repo:
helm repo add hashicorp https://helm.releases.hashicorp.com
helm repo update

# Install Vault:
helm install vault hashicorp/vault \
  --namespace vault \
  --create-namespace \
  -f vault-values.yaml

# vault-values.yaml:
server:
  ha:
    enabled: true
    replicas: 3
    raft:
      enabled: true
      config: |
        ui = true
        
        listener "tcp" {
          tls_disable = 1
          address = "[::]:8200"
          cluster_address = "[::]:8201"
        }
        
        storage "raft" {
          path = "/vault/data"
          
          retry_join {
            leader_api_addr = "http://vault-0.vault-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-1.vault-internal:8200"
          }
          retry_join {
            leader_api_addr = "http://vault-2.vault-internal:8200"
          }
        }
        
        service_registration "kubernetes" {}

  resources:
    requests:
      cpu: 250m
      memory: 256Mi
    limits:
      cpu: "1"
      memory: 1Gi

  dataStorage:
    enabled: true
    size: 10Gi
    storageClass: ceph-block

  auditStorage:
    enabled: true
    size: 10Gi
    storageClass: ceph-block

ui:
  enabled: true
  serviceType: ClusterIP

injector:
  enabled: true
  resources:
    requests:
      cpu: 50m
      memory: 64Mi

# Initialize Vault:
kubectl -n vault exec vault-0 -- vault operator init \
  -key-shares=5 \
  -key-threshold=3 \
  -format=json > vault-init.json

# ⚠️ CRITICAL: Store vault-init.json securely!
# Contains unseal keys and root token

# Unseal each Vault pod (need 3 of 5 keys):
for i in 0 1 2; do
  kubectl -n vault exec vault-$i -- vault operator unseal $(jq -r '.unseal_keys_b64[0]' vault-init.json)
  kubectl -n vault exec vault-$i -- vault operator unseal $(jq -r '.unseal_keys_b64[1]' vault-init.json)
  kubectl -n vault exec vault-$i -- vault operator unseal $(jq -r '.unseal_keys_b64[2]' vault-init.json)
done

# Join raft cluster:
kubectl -n vault exec vault-1 -- vault operator raft join http://vault-0.vault-internal:8200
kubectl -n vault exec vault-2 -- vault operator raft join http://vault-0.vault-internal:8200

# Verify:
kubectl -n vault exec vault-0 -- vault status
# Sealed: false
# HA Mode: active

kubectl -n vault exec vault-0 -- vault operator raft list-peers

PART 3: KV SECRETS ENGINE

# Login with root token:
export VAULT_TOKEN=$(jq -r '.root_token' vault-init.json)

# Enable KV v2:
kubectl -n vault exec vault-0 -- vault secrets enable -path=secret kv-v2

# Store secrets:
kubectl -n vault exec vault-0 -- vault kv put secret/production/order-service \
  DATABASE_URL="postgresql://appuser:[email protected]:5432/appdb" \
  RABBITMQ_URL="amqp://order:[email protected]:5672/orders" \
  REDIS_URL="redis://:[email protected]:26379" \
  JWT_SECRET="$(openssl rand -base64 32)" \
  API_KEY="$(openssl rand -hex 32)"

# Read secrets:
kubectl -n vault exec vault-0 -- vault kv get secret/production/order-service

# Version history:
kubectl -n vault exec vault-0 -- vault kv metadata get secret/production/order-service

PART 4: KUBERNETES AUTH METHOD

# Enable K8s auth:
kubectl -n vault exec vault-0 -- vault auth enable kubernetes

# Configure K8s auth:
kubectl -n vault exec vault-0 -- vault write auth/kubernetes/config \
  kubernetes_host="https://kubernetes.default.svc:443"

# Create policy:
kubectl -n vault exec vault-0 -- vault policy write order-service - <

# Install External Secrets Operator:
helm repo add external-secrets https://charts.external-secrets.io
helm repo update

helm install external-secrets external-secrets/external-secrets \
  --namespace external-secrets \
  --create-namespace

# ClusterSecretStore (Vault backend):
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: vault-backend
spec:
  provider:
    vault:
      server: "http://vault.vault:8200"
      path: "secret"
      version: "v2"
      auth:
        kubernetes:
          mountPath: "kubernetes"
          role: "external-secrets"
          serviceAccountRef:
            name: "external-secrets"
            namespace: "external-secrets"
---
# ExternalSecret (sync Vault → K8s Secret):
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: order-service-secrets
  namespace: default
spec:
  refreshInterval: 5m
  secretStoreRef:
    name: vault-backend
    kind: ClusterSecretStore
  target:
    name: order-service-secrets
    creationPolicy: Owner
  data:
    - secretKey: DATABASE_URL
      remoteRef:
        key: production/order-service
        property: DATABASE_URL
    - secretKey: RABBITMQ_URL
      remoteRef:
        key: production/order-service
        property: RABBITMQ_URL
    - secretKey: JWT_SECRET
      remoteRef:
        key: production/order-service
        property: JWT_SECRET

# Application sử dụng synced secret:
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
spec:
  template:
    spec:
      containers:
        - name: order-service
          envFrom:
            - secretRef:
                name: order-service-secrets  # ← Synced from Vault

# Enable database secrets engine:
kubectl -n vault exec vault-0 -- vault secrets enable database

# Configure PostgreSQL connection:
kubectl -n vault exec vault-0 -- vault write database/config/production-pg \
  plugin_name=postgresql-database-plugin \
  allowed_roles="order-service-db" \
  connection_url="postgresql://{{username}}:{{password}}@production-pg-rw.database:5432/appdb?sslmode=require" \
  username="vault_admin" \
  password="vault_admin_password"

# Create role (generates temp credentials):
kubectl -n vault exec vault-0 -- vault write database/roles/order-service-db \
  db_name=production-pg \
  creation_statements="CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'; \
    GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO \"{{name}}\";" \
  revocation_statements="REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM \"{{name}}\"; DROP ROLE IF EXISTS \"{{name}}\";" \
  default_ttl="1h" \
  max_ttl="24h"

# Generate dynamic credentials:
kubectl -n vault exec vault-0 -- vault read database/creds/order-service-db
# username: v-k8s-order-ser-xxxxx
# password: A1B2C3-random-generated
# ttl: 1h
# → Auto-revoked after 1h! ✅