Lesson 25: Kubernetes Operator, Monitoring and Admin CLI

Keycloak from Basic to Advanced

Part 7: Production, HA and Kubernetes

xdev.asia

1. Keycloak Kubernetes Operator Overview

Keycloak Operator is the official Kubernetes Operator, managing Keycloak deployment according to declarative approach. Operators use Custom Resource Definitions (CRDs) to define the desired state, then automatically reconcile to ensure the actual state matches.

┌──────────────────────────────────────────────────────────────────┐
│                  Keycloak Operator Architecture                  │
│                                                                  │
│  ┌─────────────────┐     ┌─────────────────────────────────┐    │
│  │  Keycloak CR     │────►│      Keycloak Operator          │    │
│  │  (desired state) │     │                                 │    │
│  └─────────────────┘     │  - Watch Keycloak CRs            │    │
│                           │  - Create/Update StatefulSet    │    │
│  ┌─────────────────┐     │  - Manage Services              │    │
│  │  KeycloakRealm   │────►│  - Configure TLS               │    │
│  │  Import CR       │     │  - Handle upgrades              │    │
│  └─────────────────┘     └──────────┬──────────────────────┘    │
│                                      │                           │
│                           ┌──────────▼──────────────────────┐    │
│                           │     Managed Resources           │    │
│                           │                                 │    │
│                           │  ┌─────────────┐                │    │
│                           │  │ StatefulSet  │ (Keycloak pods)│    │
│                           │  └─────────────┘                │    │
│                           │  ┌─────────────┐                │    │
│                           │  │  Service     │ (ClusterIP)    │    │
│                           │  └─────────────┘                │    │
│                           │  ┌─────────────┐                │    │
│                           │  │  Service     │ (Headless/     │    │
│                           │  │  (discovery) │  JGroups)      │    │
│                           │  └─────────────┘                │    │
│                           └─────────────────────────────────┘    │
└──────────────────────────────────────────────────────────────────┘

1.1 CRDs Overview

CRD	API Group	Purpose
Keycloak	k8s.keycloak.org/v2alpha1	Define Keycloak deployment (instances, DB, hostname, TLS, features)
KeycloakRealmImport	k8s.keycloak.org/v2alpha1	Import realm configuration from JSON

2. Installing Keycloak Operator

2.1 Method 1: kubectl apply (Manifests)

# Tạo namespace
kubectl create namespace keycloak

# Install CRDs
kubectl apply -f https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/refs/heads/main/kubernetes/keycloaks.k8s.keycloak.org-v1.yml
kubectl apply -f https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/refs/heads/main/kubernetes/keycloakrealmimports.k8s.keycloak.org-v1.yml

# Install Operator
kubectl apply -n keycloak \
  -f https://raw.githubusercontent.com/keycloak/keycloak-k8s-resources/refs/heads/main/kubernetes/kubernetes.yml

# Verify installation
kubectl get pods -n keycloak
kubectl get crd | grep keycloak

2.2 Method 2: OLM (Operator Lifecycle Manager)

# Install OLM nếu chưa có
curl -sL https://github.com/operator-framework/operator-lifecycle-manager/releases/download/v0.28.0/install.sh | bash -s v0.28.0

# Install Keycloak Operator via OLM
kubectl apply -f - <<EOF
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: keycloak-operator
  namespace: operators
spec:
  channel: fast
  name: keycloak-operator
  source: community-operators
  sourceNamespace: olm
  installPlanApproval: Automatic
EOF

# Verify
kubectl get csv -n operators | grep keycloak

2.3 Method 3: Helm Chart

# Thêm Bitnami repo (community Helm chart)
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update

# Install Keycloak (bao gồm Operator)
helm install keycloak bitnami/keycloak \
  --namespace keycloak \
  --create-namespace \
  --set auth.adminUser=admin \
  --set auth.adminPassword=admin \
  --set postgresql.enabled=true \
  --set postgresql.auth.postgresPassword=pg_password \
  --set production=true \
  --set proxy=edge

# Hoặc sử dụng values file
helm install keycloak bitnami/keycloak \
  --namespace keycloak \
  -f keycloak-values.yaml

3. Keycloak CR Configuration

3.1 Basic Keycloak CR

# keycloak.yaml - Keycloak Custom Resource
apiVersion: k8s.keycloak.org/v2alpha1
kind: Keycloak
metadata:
  name: keycloak
  namespace: keycloak
  labels:
    app: keycloak
spec:
  # Số instances (pods)
  instances: 3

  # Database configuration
  db:
    vendor: postgres
    url: jdbc:postgresql://postgres-service:5432/keycloak
    usernameSecret:
      name: keycloak-db-secret
      key: username
    passwordSecret:
      name: keycloak-db-secret
      key: password
    poolInitialSize: 25
    poolMinSize: 25
    poolMaxSize: 100

  # Hostname configuration
  hostname:
    hostname: auth.example.com
    admin: admin-auth.example.com
    strict: true
    backchannelDynamic: false

  # HTTP/TLS configuration
  http:
    tlsSecret: keycloak-tls-secret
    httpEnabled: false
    httpPort: 8080
    httpsPort: 8443

  # Container image
  image: quay.io/keycloak/keycloak:26.0

  # Image pull secrets (private registry)
  imagePullSecrets:
    - name: registry-credentials

  # Features
  features:
    enabled:
      - token-exchange
      - admin-fine-grained-authz
      - declarative-user-profile
    disabled:
      - impersonation

  # Additional options
  additionalOptions:
    - name: proxy-headers
      value: xforwarded
    - name: health-enabled
      value: "true"
    - name: metrics-enabled
      value: "true"
    - name: log
      value: console
    - name: log-level
      value: info
    - name: log-console-output
      value: json
    - name: cache
      value: ispn
    - name: cache-stack
      value: kubernetes

  # Resource requirements
  resources:
    requests:
      cpu: "500m"
      memory: "1Gi"
    limits:
      cpu: "2"
      memory: "2Gi"

  # Unsupported configurations (escape hatch)
  unsupported:
    podTemplate:
      metadata:
        labels:
          app.kubernetes.io/name: keycloak
          app.kubernetes.io/component: server
      spec:
        containers:
          - env:
              - name: JAVA_OPTS_KC_HEAP
                value: "-XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=70.0"
              - name: JAVA_OPTS_APPEND
                value: >-
                  -XX:+UseG1GC
                  -XX:MaxGCPauseMillis=200
                  -XX:+UseContainerSupport
                  -XX:+ExitOnOutOfMemoryError
                  -Djgroups.dns.query=keycloak-discovery.keycloak.svc.cluster.local

3.2 Supporting Resources

# keycloak-secrets.yaml
---
apiVersion: v1
kind: Secret
metadata:
  name: keycloak-db-secret
  namespace: keycloak
type: Opaque
stringData:
  username: keycloak
  password: secure_db_password
---
apiVersion: v1
kind: Secret
metadata:
  name: keycloak-tls-secret
  namespace: keycloak
type: kubernetes.io/tls
data:
  tls.crt: 
  tls.key: 
---
apiVersion: v1
kind: Secret
metadata:
  name: keycloak-admin-secret
  namespace: keycloak
type: Opaque
stringData:
  username: admin
  password: secure_admin_password

3.3 Keycloak CR Status

# Check Keycloak CR status
kubectl get keycloak -n keycloak

# Detailed status
kubectl describe keycloak keycloak -n keycloak

# Status output example:
# Status:
#   Conditions:
#     - Type: Ready
#       Status: "True"
#     - Type: HasErrors
#       Status: "False"
#     - Type: RollingUpdate
#       Status: "False"
#   Instances: 3
#   Selector: app=keycloak

4. KeycloakRealmImport CR

4.1 Realm Import Configuration

# realm-import.yaml - Import realm via Operator
apiVersion: k8s.keycloak.org/v2alpha1
kind: KeycloakRealmImport
metadata:
  name: my-realm-import
  namespace: keycloak
spec:
  # Tên Keycloak CR để import vào
  keycloakCRName: keycloak

  # Realm JSON (inline)
  realm:
    realm: my-realm
    enabled: true
    displayName: "My Application Realm"

    # Login settings
    registrationAllowed: false
    loginWithEmailAllowed: true
    duplicateEmailsAllowed: false
    resetPasswordAllowed: true
    editUsernameAllowed: false
    bruteForceProtected: true

    # Token settings
    ssoSessionIdleTimeout: 1800
    ssoSessionMaxLifespan: 36000
    accessTokenLifespan: 300
    accessCodeLifespan: 60

    # Roles
    roles:
      realm:
        - name: ADMIN
          description: "Administrator role"
        - name: USER
          description: "Regular user role"
        - name: MANAGER
          description: "Manager role"

    # Default roles
    defaultRoles:
      - USER

    # Clients
    clients:
      - clientId: my-web-app
        name: "My Web Application"
        enabled: true
        publicClient: true
        standardFlowEnabled: true
        directAccessGrantsEnabled: false
        rootUrl: "https://app.example.com"
        baseUrl: "/"
        redirectUris:
          - "https://app.example.com/*"
        webOrigins:
          - "https://app.example.com"
        protocol: openid-connect
        attributes:
          pkce.code.challenge.method: "S256"
          post.logout.redirect.uris: "https://app.example.com/*"

      - clientId: my-api
        name: "My API Service"
        enabled: true
        publicClient: false
        bearerOnly: true
        standardFlowEnabled: false
        serviceAccountsEnabled: false
        protocol: openid-connect

      - clientId: my-service-account
        name: "Service Account Client"
        enabled: true
        publicClient: false
        standardFlowEnabled: false
        serviceAccountsEnabled: true
        protocol: openid-connect
        secret: "change-me-in-production"

    # Client scopes
    clientScopes:
      - name: custom-scope
        protocol: openid-connect
        attributes:
          display.on.consent.screen: "true"
        protocolMappers:
          - name: custom-audience
            protocol: openid-connect
            protocolMapper: oidc-audience-mapper
            config:
              included.client.audience: my-api
              access.token.claim: "true"

# Apply realm import
kubectl apply -f realm-import.yaml

# Check status
kubectl get keycloakrealmimport -n keycloak

# View details
kubectl describe keycloakrealmimport my-realm-import -n keycloak

# Status shows:
# Conditions:
#   - Type: Done
#     Status: "True"
#   - Type: HasErrors
#     Status: "False"

4.2 Realm Import from JSON

file

# Tạo ConfigMap từ realm JSON file
kubectl create configmap my-realm-json \
  --from-file=realm.json=my-realm-export.json \
  -n keycloak

# Hoặc dùng kustomize để generate KeycloakRealmImport từ JSON file

5. Operator Upgrade Strategies

# Keycloak CR với upgrade strategy
apiVersion: k8s.keycloak.org/v2alpha1
kind: Keycloak
metadata:
  name: keycloak
  namespace: keycloak
spec:
  instances: 3
  image: quay.io/keycloak/keycloak:26.0  # Update version here

  # Operator sẽ dùng RollingUpdate strategy mặc định
  # StatefulSet được tạo với:
  #   updateStrategy:
  #     type: RollingUpdate
  #     rollingUpdate:
  #       partition: 0

  unsupported:
    podTemplate:
      spec:
        # Ensure proper shutdown
        terminationGracePeriodSeconds: 60

# Upgrade process
# 1. Update image version trong Keycloak CR
kubectl patch keycloak keycloak -n keycloak \
  --type=merge \
  -p '{"spec":{"image":"quay.io/keycloak/keycloak:26.1"}}'

# 2. Monitor upgrade
kubectl rollout status statefulset/keycloak -n keycloak --timeout=600s

# 3. Watch pods
kubectl get pods -n keycloak -w

# 4. Rollback nếu cần
kubectl patch keycloak keycloak -n keycloak \
  --type=merge \
  -p '{"spec":{"image":"quay.io/keycloak/keycloak:26.0"}}'

6. Monitoring with Prometheus + Grafana

6.1 Enabling Metrics

# Keycloak CR - enable metrics
apiVersion: k8s.keycloak.org/v2alpha1
kind: Keycloak
metadata:
  name: keycloak
  namespace: keycloak
spec:
  additionalOptions:
    - name: metrics-enabled
      value: "true"
    - name: health-enabled
      value: "true"

6.2 Prometheus ServiceMonitor

# servicemonitor.yaml - Prometheus scrape config
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: keycloak-metrics
  namespace: keycloak
  labels:
    app: keycloak
    release: prometheus  # Match Prometheus operator label selector
spec:
  selector:
    matchLabels:
      app: keycloak
      app.kubernetes.io/managed-by: keycloak-operator
  endpoints:
    - port: https
      path: /metrics
      scheme: https
      tlsConfig:
        insecureSkipVerify: true  # Hoặc cấu hình CA cert
      interval: 30s
      scrapeTimeout: 10s
      honorLabels: true
  namespaceSelector:
    matchNames:
      - keycloak

6.3 Key Keycloak Metrics

Metric	Type	Description
keycloak_logins_total	Counter	Total successful logins (by realm, provider, client_id)
keycloak_registrations_total	Counter	Total user registrations
keycloak_failed_login_attempts_total	Counter	Total failed login attempts (by error type)
keycloak_request_duration_seconds	Histogram	Request duration distribution
keycloak_refresh_tokens_total	Counter	Total token refreshes
keycloak_client_logins_total	Counter	Client credential logins
vendor_memory_heap_usage_bytes	Gauge	JVM heap memory usage
vendor_memory_non_heap_usage_bytes	Gauge	JVM non-heap memory (metaspace)
vendor_cpu_processCpuLoad	Gauge	JVM process CPU utilization
vendor_gc_pause_seconds	Summary	GC pause duration
vendor_thread_count	Gauge	Active JVM threads
vendor_db_pool_active_count	Gauge	Active database connections
vendor_db_pool_available_count	Gauge	Available database connections

6.4 Grafana Dashboards

# grafana-dashboard-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: keycloak-grafana-dashboard
  namespace: monitoring
  labels:
    grafana_dashboard: "1"
data:
  keycloak-dashboard.json: |
    {
      "annotations": { "list": [] },
      "title": "Keycloak Overview",
      "uid": "keycloak-overview",
      "panels": [
        {
          "title": "Login Rate (per minute)",
          "type": "timeseries",
          "targets": [
            {
              "expr": "sum(rate(keycloak_logins_total[5m])) by (realm) * 60",
              "legendFormat": "{{realm}}"
            }
          ],
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }
        },
        {
          "title": "Failed Login Rate",
          "type": "timeseries",
          "targets": [
            {
              "expr": "sum(rate(keycloak_failed_login_attempts_total[5m])) by (realm, error) * 60",
              "legendFormat": "{{realm}} - {{error}}"
            }
          ],
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }
        },
        {
          "title": "Request Duration p95",
          "type": "timeseries",
          "targets": [
            {
              "expr": "histogram_quantile(0.95, sum(rate(keycloak_request_duration_seconds_bucket[5m])) by (le))",
              "legendFormat": "p95"
            },
            {
              "expr": "histogram_quantile(0.99, sum(rate(keycloak_request_duration_seconds_bucket[5m])) by (le))",
              "legendFormat": "p99"
            }
          ],
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }
        },
        {
          "title": "JVM Heap Usage",
          "type": "gauge",
          "targets": [
            {
              "expr": "vendor_memory_heap_usage_bytes / vendor_memory_heap_max_bytes * 100",
              "legendFormat": "Heap %"
            }
          ],
          "gridPos": { "h": 8, "w": 6, "x": 12, "y": 8 }
        },
        {
          "title": "DB Connection Pool",
          "type": "timeseries",
          "targets": [
            {
              "expr": "vendor_db_pool_active_count",
              "legendFormat": "Active"
            },
            {
              "expr": "vendor_db_pool_available_count",
              "legendFormat": "Available"
            }
          ],
          "gridPos": { "h": 8, "w": 6, "x": 18, "y": 8 }
        }
      ]
    }

6.5 Alerting Rules

# prometheus-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: keycloak-alerts
  namespace: keycloak
  labels:
    release: prometheus
spec:
  groups:
    - name: keycloak.rules
      rules:
        # High failed login rate
        - alert: KeycloakHighFailedLoginRate
          expr: >
            sum(rate(keycloak_failed_login_attempts_total[5m])) by (realm) > 10
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "High failed login rate on realm {{ $labels.realm }}"
            description: >
              Failed login rate is {{ $value | humanize }}/s on realm {{ $labels.realm }}.
              Possible brute force attack.

        # Keycloak pod not ready
        - alert: KeycloakPodNotReady
          expr: >
            kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-.*"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Keycloak pod {{ $labels.pod }} is not ready"
            description: "Pod {{ $labels.pod }} has been not ready for 2 minutes."

        # High response time
        - alert: KeycloakHighResponseTime
          expr: >
            histogram_quantile(0.95,
              sum(rate(keycloak_request_duration_seconds_bucket[5m])) by (le)
            ) > 1.0
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Keycloak p95 response time exceeds 1 second"
            description: "p95 latency is {{ $value | humanize }}s for the last 10 minutes."

        # High JVM heap usage
        - alert: KeycloakHighHeapUsage
          expr: >
            vendor_memory_heap_usage_bytes / vendor_memory_heap_max_bytes * 100 > 85
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Keycloak JVM heap usage > 85%"
            description: "Heap usage is {{ $value | humanize }}%. Consider scaling or increasing memory."

        # Database connection pool exhaustion
        - alert: KeycloakDBPoolExhausted
          expr: >
            vendor_db_pool_available_count < 5
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Keycloak database connection pool nearly exhausted"
            description: "Only {{ $value }} connections available. Increase pool size or investigate slow queries."

        # No successful logins (potential outage)
        - alert: KeycloakNoLogins
          expr: >
            sum(rate(keycloak_logins_total[10m])) == 0
          for: 15m
          labels:
            severity: critical
          annotations:
            summary: "No successful logins in the last 15 minutes"
            description: "Keycloak may be experiencing an outage."

        # High error rate
        - alert: KeycloakHighErrorRate
          expr: >
            sum(rate(keycloak_request_duration_seconds_count{status=~"5.."}[5m]))
            / sum(rate(keycloak_request_duration_seconds_count[5m])) > 0.05
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Keycloak error rate exceeds 5%"
            description: "{{ $value | humanizePercentage }} of requests are returning 5xx errors."

7. Admin CLI (kcadm.sh)

7.1 Authentication

# Login vào Keycloak Admin CLI
bin/kcadm.sh config credentials \
  --server https://auth.example.com \
  --realm master \
  --user admin \
  --password admin_password

# Login với client credentials (service account)
bin/kcadm.sh config credentials \
  --server https://auth.example.com \
  --realm master \
  --client admin-cli \
  --secret client_secret

# Trong container
kubectl exec -it keycloak-0 -n keycloak -- \
  /opt/keycloak/bin/kcadm.sh config credentials \
  --server http://localhost:8080 \
  --realm master \
  --user admin \
  --password admin

7.2 Realm Operations

# ===== Realm CRUD =====

# List tất cả realms
bin/kcadm.sh get realms --fields realm,enabled

# Tạo realm mới
bin/kcadm.sh create realms \
  -s realm=my-new-realm \
  -s enabled=true \
  -s displayName="My New Realm" \
  -s registrationAllowed=false \
  -s loginWithEmailAllowed=true \
  -s resetPasswordAllowed=true \
  -s bruteForceProtected=true \
  -s permanentLockout=false \
  -s maxFailureWaitSeconds=900 \
  -s failureFactor=5

# Get realm details
bin/kcadm.sh get realms/my-new-realm

# Update realm
bin/kcadm.sh update realms/my-new-realm \
  -s "ssoSessionIdleTimeout=1800" \
  -s "ssoSessionMaxLifespan=36000" \
  -s "accessTokenLifespan=300"

# Delete realm
bin/kcadm.sh delete realms/my-new-realm

7.3 User Operations

# ===== User CRUD =====

# Tạo user mới
bin/kcadm.sh create users -r my-realm \
  -s username=john.doe \
  -s [email protected] \
  -s firstName=John \
  -s lastName=Doe \
  -s enabled=true \
  -s emailVerified=true

# Set password cho user
bin/kcadm.sh set-password -r my-realm \
  --username john.doe \
  --new-password "SecureP@ss123" \
  --temporary=false

# List users
bin/kcadm.sh get users -r my-realm --fields id,username,email,enabled

# Search users
bin/kcadm.sh get users -r my-realm -q "username=john"
bin/kcadm.sh get users -r my-realm -q "[email protected]"

# Get user by ID
bin/kcadm.sh get users/USER_ID -r my-realm

# Update user
bin/kcadm.sh update users/USER_ID -r my-realm \
  -s firstName="Jonathan" \
  -s "attributes.department=[\"Engineering\"]"

# Disable user
bin/kcadm.sh update users/USER_ID -r my-realm \
  -s enabled=false

# Delete user
bin/kcadm.sh delete users/USER_ID -r my-realm

# ===== User Roles =====

# Add realm role to user
bin/kcadm.sh add-roles -r my-realm \
  --uusername john.doe \
  --rolename ADMIN

# Add client role to user
bin/kcadm.sh add-roles -r my-realm \
  --uusername john.doe \
  --cclientid my-app \
  --rolename app-admin

# Remove role from user
bin/kcadm.sh remove-roles -r my-realm \
  --uusername john.doe \
  --rolename ADMIN

# List user roles
bin/kcadm.sh get-roles -r my-realm --uusername john.doe

# ===== User Groups =====

# Add user to group
bin/kcadm.sh update users/USER_ID/groups/GROUP_ID -r my-realm \
  -s realm=my-realm -s userId=USER_ID -s groupId=GROUP_ID -n

# Remove user from group
bin/kcadm.sh delete users/USER_ID/groups/GROUP_ID -r my-realm

# List user groups
bin/kcadm.sh get users/USER_ID/groups -r my-realm

7.4 Client Operations

# ===== Client CRUD =====

# Tạo public client (SPA)
bin/kcadm.sh create clients -r my-realm \
  -s clientId=my-spa \
  -s name="My SPA Application" \
  -s enabled=true \
  -s publicClient=true \
  -s standardFlowEnabled=true \
  -s directAccessGrantsEnabled=false \
  -s 'redirectUris=["https://app.example.com/*"]' \
  -s 'webOrigins=["https://app.example.com"]' \
  -s protocol=openid-connect \
  -s 'attributes={"pkce.code.challenge.method":"S256"}'

# Tạo confidential client (backend service)
bin/kcadm.sh create clients -r my-realm \
  -s clientId=my-backend \
  -s name="My Backend Service" \
  -s enabled=true \
  -s publicClient=false \
  -s serviceAccountsEnabled=true \
  -s standardFlowEnabled=false \
  -s protocol=openid-connect

# Tạo bearer-only client (API)
bin/kcadm.sh create clients -r my-realm \
  -s clientId=my-api \
  -s name="My API" \
  -s enabled=true \
  -s publicClient=false \
  -s bearerOnly=true \
  -s protocol=openid-connect

# List clients
bin/kcadm.sh get clients -r my-realm --fields id,clientId,enabled

# Get client secret
bin/kcadm.sh get clients/CLIENT_ID/client-secret -r my-realm

# Regenerate client secret
bin/kcadm.sh create clients/CLIENT_ID/client-secret -r my-realm

# Update client
bin/kcadm.sh update clients/CLIENT_ID -r my-realm \
  -s 'redirectUris=["https://new-app.example.com/*"]'

# Delete client
bin/kcadm.sh delete clients/CLIENT_ID -r my-realm

7.5 Role Operations

# ===== Realm Roles =====

# Create realm role
bin/kcadm.sh create roles -r my-realm \
  -s name=SUPERVISOR \
  -s description="Supervisor role"

# List realm roles
bin/kcadm.sh get roles -r my-realm

# Get role details
bin/kcadm.sh get roles/SUPERVISOR -r my-realm

# Create composite role (role chứa roles khác)
bin/kcadm.sh add-roles -r my-realm \
  --rname SUPERVISOR \
  --rolename USER \
  --rolename MANAGER

# Delete role
bin/kcadm.sh delete roles/SUPERVISOR -r my-realm

# ===== Client Roles =====

# Create client role
bin/kcadm.sh create clients/CLIENT_ID/roles -r my-realm \
  -s name=client-admin \
  -s description="Client administrator"

# List client roles
bin/kcadm.sh get clients/CLIENT_ID/roles -r my-realm

7.6 Group Operations

# ===== Group CRUD =====

# Create group
bin/kcadm.sh create groups -r my-realm \
  -s name="Engineering"

# Create sub-group
bin/kcadm.sh create groups/PARENT_GROUP_ID/children -r my-realm \
  -s name="Backend Team"

# List groups
bin/kcadm.sh get groups -r my-realm

# Add role to group
bin/kcadm.sh add-roles -r my-realm \
  --gname Engineering \
  --rolename USER

# List group members
bin/kcadm.sh get groups/GROUP_ID/members -r my-realm

# Delete group
bin/kcadm.sh delete groups/GROUP_ID -r my-realm

7.7 Identity Provider Operations

# ===== Identity Provider CRUD =====

# Create Google Identity Provider
bin/kcadm.sh create identity-provider/instances -r my-realm \
  -s alias=google \
  -s providerId=google \
  -s enabled=true \
  -s 'config.clientId=GOOGLE_CLIENT_ID' \
  -s 'config.clientSecret=GOOGLE_CLIENT_SECRET' \
  -s 'config.defaultScope=openid email profile' \
  -s trustEmail=true \
  -s firstBrokerLoginFlowAlias="first broker login"

# Create OIDC Identity Provider
bin/kcadm.sh create identity-provider/instances -r my-realm \
  -s alias=corporate-idp \
  -s providerId=oidc \
  -s enabled=true \
  -s 'config.authorizationUrl=https://idp.example.com/authorize' \
  -s 'config.tokenUrl=https://idp.example.com/token' \
  -s 'config.clientId=keycloak-client' \
  -s 'config.clientSecret=client_secret' \
  -s 'config.userInfoUrl=https://idp.example.com/userinfo' \
  -s 'config.defaultScope=openid email profile'

# Create SAML Identity Provider
bin/kcadm.sh create identity-provider/instances -r my-realm \
  -s alias=saml-idp \
  -s providerId=saml \
  -s enabled=true \
  -s 'config.singleSignOnServiceUrl=https://idp.example.com/sso' \
  -s 'config.nameIDPolicyFormat=urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress' \
  -s 'config.principalType=ATTRIBUTE' \
  -s 'config.principalAttribute=email'

# List identity providers
bin/kcadm.sh get identity-provider/instances -r my-realm

# Update identity provider
bin/kcadm.sh update identity-provider/instances/google -r my-realm \
  -s enabled=false

# Delete identity provider
bin/kcadm.sh delete identity-provider/instances/google -r my-realm

# ===== Identity Provider Mappers =====

# Create mapper
bin/kcadm.sh create identity-provider/instances/google/mappers -r my-realm \
  -s name="email-mapper" \
  -s identityProviderMapper=hardcoded-user-session-attribute-idp-mapper \
  -s identityProviderAlias=google \
  -s 'config={"syncMode":"INHERIT","attribute":"email","attribute.value":""}'

7.8 Authentication Flow Management

# ===== Authentication Flows =====

# List authentication flows
bin/kcadm.sh get authentication/flows -r my-realm --fields id,alias,builtIn

# Get flow executions
bin/kcadm.sh get authentication/flows/browser/executions -r my-realm

# Copy built-in flow (để customize)
bin/kcadm.sh create authentication/flows/browser/copy -r my-realm \
  -s newName="Custom Browser Flow"

# Add execution to flow
bin/kcadm.sh create authentication/flows/Custom%20Browser%20Flow/executions/execution -r my-realm \
  -s provider=auth-otp-form

# Update execution requirement
bin/kcadm.sh update authentication/flows/Custom%20Browser%20Flow/executions -r my-realm \
  -b '[{"id":"EXECUTION_ID","requirement":"REQUIRED"}]'

# Set flow as realm browser flow
bin/kcadm.sh update realms/my-realm \
  -s browserFlow="Custom Browser Flow"

7.9 Component Management

# ===== LDAP User Federation =====

# Create LDAP provider
bin/kcadm.sh create components -r my-realm \
  -s name="Corporate LDAP" \
  -s providerId=ldap \
  -s providerType=org.keycloak.storage.UserStorageProvider \
  -s 'config.vendor=["ad"]' \
  -s 'config.connectionUrl=["ldaps://ldap.example.com:636"]' \
  -s 'config.bindDn=["cn=admin,dc=example,dc=com"]' \
  -s 'config.bindCredential=["ldap_password"]' \
  -s 'config.usersDn=["ou=users,dc=example,dc=com"]' \
  -s 'config.userObjectClasses=["person, organizationalPerson, user"]' \
  -s 'config.usernameLDAPAttribute=["sAMAccountName"]' \
  -s 'config.uuidLDAPAttribute=["objectGUID"]' \
  -s 'config.rdnLDAPAttribute=["cn"]' \
  -s 'config.editMode=["READ_ONLY"]' \
  -s 'config.syncRegistrations=["false"]' \
  -s 'config.searchScope=["2"]' \
  -s 'config.pagination=["true"]' \
  -s 'config.batchSizeForSync=["1000"]' \
  -s 'config.importEnabled=["true"]'

# List components
bin/kcadm.sh get components -r my-realm -q type=org.keycloak.storage.UserStorageProvider

# Trigger LDAP sync
bin/kcadm.sh create user-storage/COMPONENT_ID/sync -r my-realm -s action=triggerFullSync
bin/kcadm.sh create user-storage/COMPONENT_ID/sync -r my-realm -s action=triggerChangedUsersSync

7.10 Export/Import and Scripting

# ===== Export realm =====

# Partial export (via Admin API)
bin/kcadm.sh create realms/my-realm/partial-export \
  -s exportClients=true \
  -s exportGroupsAndRoles=true \
  -o > my-realm-export.json

# ===== Scripting Automation =====

#!/bin/bash
# setup-realm.sh - Automated realm setup script

KEYCLOAK_URL="https://auth.example.com"
REALM="my-realm"
KCADM="bin/kcadm.sh"

# 1. Login
$KCADM config credentials \
  --server "$KEYCLOAK_URL" \
  --realm master \
  --user admin \
  --password "$KEYCLOAK_ADMIN_PASSWORD"

# 2. Create Realm
echo "=== Creating realm: $REALM ==="
$KCADM create realms \
  -s realm="$REALM" \
  -s enabled=true \
  -s displayName="My Application" \
  -s loginWithEmailAllowed=true \
  -s resetPasswordAllowed=true \
  -s bruteForceProtected=true \
  -s failureFactor=5 \
  -s ssoSessionIdleTimeout=1800 \
  -s accessTokenLifespan=300

# 3. Create Roles
echo "=== Creating roles ==="
for ROLE in ADMIN USER MANAGER; do
  $KCADM create roles -r "$REALM" -s name="$ROLE"
done

# 4. Create Clients
echo "=== Creating clients ==="

# SPA Client
$KCADM create clients -r "$REALM" \
  -s clientId=my-spa \
  -s publicClient=true \
  -s standardFlowEnabled=true \
  -s directAccessGrantsEnabled=false \
  -s 'redirectUris=["https://app.example.com/*"]' \
  -s 'webOrigins=["https://app.example.com"]' \
  -s 'attributes={"pkce.code.challenge.method":"S256"}'

# API Client
$KCADM create clients -r "$REALM" \
  -s clientId=my-api \
  -s publicClient=false \
  -s bearerOnly=true

# Service Account
$KCADM create clients -r "$REALM" \
  -s clientId=my-service \
  -s publicClient=false \
  -s serviceAccountsEnabled=true \
  -s standardFlowEnabled=false

# 5. Create Test Users
echo "=== Creating test users ==="
$KCADM create users -r "$REALM" \
  -s username=admin-user \
  -s [email protected] \
  -s firstName=Admin \
  -s lastName=User \
  -s enabled=true \
  -s emailVerified=true

$KCADM set-password -r "$REALM" \
  --username admin-user \
  --new-password "Admin@123" \
  --temporary=false

$KCADM add-roles -r "$REALM" \
  --uusername admin-user \
  --rolename ADMIN

echo "=== Realm setup completed ==="

8. Backup Strategies cho Kubernetes

8.1 Database Backup

# cronjob-db-backup.yaml - Automated database backup
apiVersion: batch/v1
kind: CronJob
metadata:
  name: keycloak-db-backup
  namespace: keycloak
spec:
  schedule: "0 2 * * *"  # Daily at 2 AM
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 7
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: OnFailure
          containers:
            - name: backup
              image: postgres:16-alpine
              env:
                - name: PGHOST
                  value: postgres-service
                - name: PGDATABASE
                  value: keycloak
                - name: PGUSER
                  valueFrom:
                    secretKeyRef:
                      name: keycloak-db-secret
                      key: username
                - name: PGPASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: keycloak-db-secret
                      key: password
              command:
                - /bin/sh
                - -c
                - |
                  BACKUP_FILE="/backups/keycloak-$(date +%Y%m%d_%H%M%S).dump"
                  echo "Starting backup to $BACKUP_FILE"

                  pg_dump --format=custom --compress=9 \
                    --file="$BACKUP_FILE"

                  if [ $? -eq 0 ]; then
                    echo "Backup successful: $BACKUP_FILE"
                    # Cleanup old backups (keep last 30 days)
                    find /backups -name "keycloak-*.dump" -mtime +30 -delete
                  else
                    echo "Backup failed!"
                    exit 1
                  fi
              volumeMounts:
                - name: backup-volume
                  mountPath: /backups
          volumes:
            - name: backup-volume
              persistentVolumeClaim:
                claimName: keycloak-backup-pvc

8.2 Realm Export Backup

# cronjob-realm-export.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
  name: keycloak-realm-export
  namespace: keycloak
spec:
  schedule: "0 3 * * *"  # Daily at 3 AM
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: OnFailure
          containers:
            - name: export
              image: curlimages/curl:latest
              env:
                - name: KC_URL
                  value: "http://keycloak-service:8080"
                - name: KC_ADMIN_USER
                  valueFrom:
                    secretKeyRef:
                      name: keycloak-admin-secret
                      key: username
                - name: KC_ADMIN_PASS
                  valueFrom:
                    secretKeyRef:
                      name: keycloak-admin-secret
                      key: password
              command:
                - /bin/sh
                - -c
                - |
                  # Get admin token
                  TOKEN=$(curl -s \
                    -d "client_id=admin-cli" \
                    -d "username=$KC_ADMIN_USER" \
                    -d "password=$KC_ADMIN_PASS" \
                    -d "grant_type=password" \
                    "$KC_URL/realms/master/protocol/openid-connect/token" \
                    | grep -o '"access_token":"[^"]*"' | cut -d'"' -f4)

                  if [ -z "$TOKEN" ]; then
                    echo "Failed to get admin token"
                    exit 1
                  fi

                  # Export each realm
                  REALMS=$(curl -s \
                    -H "Authorization: Bearer $TOKEN" \
                    "$KC_URL/admin/realms" \
                    | grep -o '"realm":"[^"]*"' | cut -d'"' -f4)

                  TIMESTAMP=$(date +%Y%m%d_%H%M%S)
                  for REALM in $REALMS; do
                    echo "Exporting realm: $REALM"
                    curl -s \
                      -H "Authorization: Bearer $TOKEN" \
                      "$KC_URL/admin/realms/$REALM/partial-export?exportClients=true&exportGroupsAndRoles=true" \
                      -o "/backups/realm-${REALM}-${TIMESTAMP}.json"
                  done

                  echo "Export completed"
              volumeMounts:
                - name: backup-volume
                  mountPath: /backups
          volumes:
            - name: backup-volume
              persistentVolumeClaim:
                claimName: keycloak-backup-pvc

8.3 Velero Backup cho Kubernetes

# Install Velero
velero install \
  --provider aws \
  --plugins velero/velero-plugin-for-aws:v1.9.0 \
  --bucket keycloak-backups \
  --secret-file ./credentials-velero \
  --backup-location-config region=ap-southeast-1 \
  --snapshot-location-config region=ap-southeast-1

# Create backup schedule
velero create schedule keycloak-daily \
  --schedule="0 2 * * *" \
  --include-namespaces keycloak \
  --ttl 720h  # Keep backups for 30 days

# Manual backup
velero backup create keycloak-manual-$(date +%Y%m%d) \
  --include-namespaces keycloak \
  --wait

# List backups
velero backup get

# Restore from backup
velero restore create --from-backup keycloak-manual-20260330 \
  --include-namespaces keycloak

# Verify restore
kubectl get all -n keycloak

9. Complete Kubernetes Deployment

Below are all the necessary manifests to fully deploy Keycloak on Kubernetes:

# 01-namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: keycloak
  labels:
    app.kubernetes.io/part-of: keycloak
---
# 02-secrets.yaml
apiVersion: v1
kind: Secret
metadata:
  name: keycloak-db-secret
  namespace: keycloak
type: Opaque
stringData:
  username: keycloak
  password: "secure_db_password_here"
---
apiVersion: v1
kind: Secret
metadata:
  name: keycloak-admin-secret
  namespace: keycloak
type: Opaque
stringData:
  username: admin
  password: "secure_admin_password_here"
---
# 03-postgresql-statefulset.yaml
apiVersion: v1
kind: Service
metadata:
  name: postgres
  namespace: keycloak
spec:
  type: ClusterIP
  selector:
    app: postgres
  ports:
    - port: 5432
      targetPort: 5432
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: postgres
  namespace: keycloak
spec:
  serviceName: postgres
  replicas: 1
  selector:
    matchLabels:
      app: postgres
  template:
    metadata:
      labels:
        app: postgres
    spec:
      containers:
        - name: postgres
          image: postgres:16-alpine
          ports:
            - containerPort: 5432
          env:
            - name: POSTGRES_DB
              value: keycloak
            - name: POSTGRES_USER
              valueFrom:
                secretKeyRef:
                  name: keycloak-db-secret
                  key: username
            - name: POSTGRES_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: keycloak-db-secret
                  key: password
            - name: PGDATA
              value: /var/lib/postgresql/data/pgdata
          volumeMounts:
            - name: pgdata
              mountPath: /var/lib/postgresql/data
          resources:
            requests:
              cpu: 500m
              memory: 1Gi
            limits:
              cpu: "2"
              memory: 2Gi
          readinessProbe:
            exec:
              command: ["pg_isready", "-U", "keycloak"]
            initialDelaySeconds: 5
            periodSeconds: 10
          livenessProbe:
            exec:
              command: ["pg_isready", "-U", "keycloak"]
            initialDelaySeconds: 30
            periodSeconds: 10
  volumeClaimTemplates:
    - metadata:
        name: pgdata
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 20Gi
---
# 04-keycloak-cr.yaml
apiVersion: k8s.keycloak.org/v2alpha1
kind: Keycloak
metadata:
  name: keycloak
  namespace: keycloak
spec:
  instances: 3
  db:
    vendor: postgres
    url: jdbc:postgresql://postgres:5432/keycloak
    usernameSecret:
      name: keycloak-db-secret
      key: username
    passwordSecret:
      name: keycloak-db-secret
      key: password
    poolInitialSize: 25
    poolMinSize: 25
    poolMaxSize: 100
  hostname:
    hostname: auth.example.com
    strict: true
  http:
    tlsSecret: keycloak-tls-secret
    httpEnabled: false
  features:
    enabled:
      - token-exchange
  additionalOptions:
    - name: proxy-headers
      value: xforwarded
    - name: health-enabled
      value: "true"
    - name: metrics-enabled
      value: "true"
    - name: cache
      value: ispn
    - name: cache-stack
      value: kubernetes
    - name: log-console-output
      value: json
  resources:
    requests:
      cpu: 500m
      memory: 1Gi
    limits:
      cpu: "2"
      memory: 2Gi
  unsupported:
    podTemplate:
      metadata:
        labels:
          app.kubernetes.io/name: keycloak
      spec:
        containers:
          - env:
              - name: JAVA_OPTS_KC_HEAP
                value: "-XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=70.0"
              - name: JAVA_OPTS_APPEND
                value: >-
                  -XX:+UseG1GC -XX:MaxGCPauseMillis=200
                  -XX:+UseContainerSupport -XX:+ExitOnOutOfMemoryError
                  -Djgroups.dns.query=keycloak-discovery.keycloak.svc.cluster.local
              - name: KC_BOOTSTRAP_ADMIN_USERNAME
                valueFrom:
                  secretKeyRef:
                    name: keycloak-admin-secret
                    key: username
              - name: KC_BOOTSTRAP_ADMIN_PASSWORD
                valueFrom:
                  secretKeyRef:
                    name: keycloak-admin-secret
                    key: password
---
# 05-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: keycloak
  namespace: keycloak
  annotations:
    nginx.ingress.kubernetes.io/proxy-buffer-size: "128k"
    nginx.ingress.kubernetes.io/proxy-buffers-number: "4"
    nginx.ingress.kubernetes.io/affinity: "cookie"
    nginx.ingress.kubernetes.io/session-cookie-name: "KC_ROUTE"
    nginx.ingress.kubernetes.io/session-cookie-expires: "3600"
    nginx.ingress.kubernetes.io/session-cookie-secure: "true"
    nginx.ingress.kubernetes.io/session-cookie-httponly: "true"
    cert-manager.io/cluster-issuer: "letsencrypt-prod"
spec:
  ingressClassName: nginx
  tls:
    - hosts:
        - auth.example.com
      secretName: keycloak-tls-secret
  rules:
    - host: auth.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: keycloak-service
                port:
                  number: 8443
---
# 06-servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: keycloak
  namespace: keycloak
  labels:
    release: prometheus
spec:
  selector:
    matchLabels:
      app: keycloak
  endpoints:
    - port: https
      path: /metrics
      scheme: https
      tlsConfig:
        insecureSkipVerify: true
      interval: 30s
  namespaceSelector:
    matchNames:
      - keycloak
---
# 07-prometheus-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: keycloak-alerts
  namespace: keycloak
  labels:
    release: prometheus
spec:
  groups:
    - name: keycloak.rules
      rules:
        - alert: KeycloakPodNotReady
          expr: kube_pod_status_ready{namespace="keycloak", pod=~"keycloak-.*"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Keycloak pod {{ $labels.pod }} not ready"
        - alert: KeycloakHighFailedLogins
          expr: sum(rate(keycloak_failed_login_attempts_total[5m])) by (realm) > 10
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "High failed login rate on {{ $labels.realm }}"
        - alert: KeycloakHighLatency
          expr: histogram_quantile(0.95, sum(rate(keycloak_request_duration_seconds_bucket[5m])) by (le)) > 1
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Keycloak p95 latency exceeds 1s"

# Deploy tất cả
kubectl apply -f 01-namespace.yaml
kubectl apply -f 02-secrets.yaml
kubectl apply -f 03-postgresql-statefulset.yaml

# Chờ PostgreSQL ready
kubectl wait --for=condition=ready pod/postgres-0 -n keycloak --timeout=120s

# Deploy Keycloak (cần Operator đã installed)
kubectl apply -f 04-keycloak-cr.yaml
kubectl apply -f 05-ingress.yaml
kubectl apply -f 06-servicemonitor.yaml
kubectl apply -f 07-prometheus-rules.yaml

# Verify
kubectl get all -n keycloak
kubectl get keycloak -n keycloak
kubectl logs -f keycloak-0 -n keycloak