End-to-End Secrets Management & Key Rotation Governance: Best Practices for Infrastructure, Applications, and CI/CD¶

# .github/workflows/deploy.yml
name: Deploy Secrets

on:
  push:
    branches: [main]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Install SOPS
        run: |
          wget https://github.com/mozilla/sops/releases/download/v3.8.1/sops-v3.8.1.linux
          sudo mv sops-v3.8.1.linux /usr/local/bin/sops
          sudo chmod +x /usr/local/bin/sops

      - name: Decrypt secrets
        env:
          SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
        run: |
          sops -d secrets/postgres-credentials.yaml > decrypted.yaml
          kubectl apply -f decrypted.yaml

# EncryptionConfiguration
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: <base64-encoded-32-byte-key>
      - identity: {}  # Fallback

# postgres-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: postgres-credentials
  namespace: production
type: Opaque
stringData:
  username: myapp
  password: super-secret-password
  host: postgres.example.com
  port: "5432"

# docker-compose.yml
version: '3.8'

services:
  app:
    image: myapp:latest
    secrets:
      - postgres_password
      - postgres_username
    environment:
      - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
      - POSTGRES_USERNAME_FILE=/run/secrets/postgres_username

secrets:
  postgres_password:
    external: true
  postgres_username:
    file: ./username.txt  # Plain text (not recommended for production)

# vault-secretstore.yaml
apiVersion: external-secrets.io/v1beta1
kind: SecretStore
metadata:
  name: vault-backend
  namespace: production
spec:
  provider:
    vault:
      server: "https://vault.example.com:8200"
      path: "secret"
      version: "v2"
      auth:
        kubernetes:
          mountPath: "kubernetes"
          role: "external-secrets"
          serviceAccountRef:
            name: external-secrets

# postgres-external-secret.yaml
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: postgres-credentials
  namespace: production
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: vault-backend
    kind: SecretStore
  target:
    name: postgres-credentials
    creationPolicy: Owner
  data:
    - secretKey: username
      remoteRef:
        key: database/postgres
        property: username
    - secretKey: password
      remoteRef:
        key: database/postgres
        property: password

# postgres-pod.yaml
apiVersion: v1
kind: Pod
metadata:
  name: myapp
  namespace: production
spec:
  containers:
    - name: app
      image: myapp:latest
      volumeMounts:
        - name: postgres-secret
          mountPath: /etc/secrets/postgres
          readOnly: true
  volumes:
    - name: postgres-secret
      secret:
        secretName: postgres-credentials
        items:
          - key: username
            path: username
          - key: password
            path: password

# postgres-pod-env.yaml
apiVersion: v1
kind: Pod
metadata:
  name: myapp
  namespace: production
spec:
  containers:
    - name: app
      image: myapp:latest
      env:
        - name: POSTGRES_USERNAME
          valueFrom:
            secretKeyRef:
              name: postgres-credentials
              key: username
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              name: postgres-credentials
              key: password

# postgres-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
  namespace: production
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
        - name: app
          image: myapp:latest
          envFrom:
            - secretRef:
                name: postgres-credentials
          # Or use env with secretKeyRef
          env:
            - name: POSTGRES_HOST
              value: postgres.example.com

# vault-sidecar-pod.yaml
apiVersion: v1
kind: Pod
metadata:
  name: myapp-with-vault
  namespace: production
  annotations:
    vault.hashicorp.com/agent-inject: "true"
    vault.hashicorp.com/role: "myapp"
    vault.hashicorp.com/agent-inject-secret-database: "database/creds/myapp"
    vault.hashicorp.com/agent-inject-template-database: |
      {{- with secret "database/creds/myapp" -}}
      POSTGRES_USERNAME={{ .Data.username }}
      POSTGRES_PASSWORD={{ .Data.password }}
      {{- end }}
spec:
  serviceAccountName: myapp
  containers:
    - name: app
      image: myapp:latest
      env:
        - name: POSTGRES_USERNAME
          valueFrom:
            secretKeyRef:
              name: vault-env
              key: POSTGRES_USERNAME
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              name: vault-env
              key: POSTGRES_PASSWORD
    - name: vault-agent
      image: hashicorp/vault:latest
      args:
        - "agent"
        - "-config=/vault/config/agent.hcl"
      volumeMounts:
        - name: vault-config
          mountPath: /vault/config

# docker-compose.yml
version: '3.8'

services:
  app:
    image: myapp:latest
    env_file:
      - .env
    environment:
      - POSTGRES_HOST=${POSTGRES_HOST}
      - POSTGRES_USERNAME=${POSTGRES_USERNAME}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
    depends_on:
      - postgres

  postgres:
    image: postgres:15
    environment:
      - POSTGRES_USER=${POSTGRES_USERNAME}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
    volumes:
      - postgres_data:/var/lib/postgresql/data

volumes:
  postgres_data:

# docker-compose.secrets.yml
version: '3.8'

services:
  app:
    image: myapp:latest
    secrets:
      - postgres_username
      - postgres_password
    environment:
      - POSTGRES_USERNAME_FILE=/run/secrets/postgres_username
      - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password

secrets:
  postgres_username:
    external: true
  postgres_password:
    external: true

# sealed-secret.yaml (safe to commit)
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
  name: postgres-credentials
  namespace: production
spec:
  encryptedData:
    username: AgBx...
    password: AgBy...
  template:
    metadata:
      name: postgres-credentials
      namespace: production
    type: Opaque

# Via Rancher UI or API
apiVersion: v1
kind: Secret
metadata:
  name: postgres-credentials
  namespace: production
  labels:
    cattle.io/creator: norman
type: Opaque
stringData:
  username: myapp
  password: super-secret-password

# Rancher workload YAML
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
  namespace: production
spec:
  template:
    spec:
      containers:
        - name: app
          image: myapp:latest
          envFrom:
            - secretRef:
                name: postgres-credentials

# prefect-deployment.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prefect-deployment
data:
  deployment.yaml: |
    name: my-flow
    work_queue_name: k8s-default
    work_pool_name: k8s-work-pool
    job_variables:
      env:
        - name: POSTGRES_USERNAME
          valueFrom:
            secretKeyRef:
              name: postgres-credentials
              key: username
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              name: postgres-credentials
              key: password

# rotate-postgres-password.yml
---
- name: Rotate Postgres Password
  hosts: postgres_servers
  vars:
    app_user: myapp
    new_password: "{{ vault_postgres_new_password }}"
    old_user: "{{ app_user }}_v1"
    new_user: "{{ app_user }}_v2"

  tasks:
    - name: Generate new password
      set_fact:
        generated_password: "{{ lookup('password', '/dev/null length=32 chars=ascii_letters,digits') }}"
      when: new_password is not defined

    - name: Create new user with new password
      postgresql_user:
        name: "{{ new_user }}"
        password: "{{ new_password | default(generated_password) }}"
        priv: "ALL:public"
        state: present
      become: yes
      become_user: postgres

    - name: Grant permissions to new user
      postgresql_privs:
        db: mydb
        privs: ALL
        objs: ALL_IN_SCHEMA
        schema: public
        roles: "{{ new_user }}"
      become: yes
      become_user: postgres

    - name: Update Kubernetes secret (v2)
      kubernetes.core.k8s:
        state: present
        definition:
          apiVersion: v1
          kind: Secret
          metadata:
            name: postgres-credentials-v2
            namespace: production
          type: Opaque
          stringData:
            username: "{{ new_user }}"
            password: "{{ new_password | default(generated_password) }}"

    - name: Rolling update deployment (canary)
      kubernetes.core.k8s_scale:
        state: present
        definition:
          apiVersion: apps/v1
          kind: Deployment
          metadata:
            name: myapp
            namespace: production
          spec:
            replicas: 1
        wait: true
        wait_condition:
          type: Available
          status: "True"

    - name: Update deployment to use v2 secret
      kubernetes.core.k8s:
        state: present
        definition:
          apiVersion: apps/v1
          kind: Deployment
          metadata:
            name: myapp
            namespace: production
          spec:
            template:
              spec:
                containers:
                  - name: app
                    envFrom:
                      - secretRef:
                          name: postgres-credentials-v2

    - name: Scale up deployment
      kubernetes.core.k8s_scale:
        state: present
        definition:
          apiVersion: apps/v1
          kind: Deployment
          metadata:
            name: myapp
            namespace: production
          spec:
            replicas: 3
        wait: true

    - name: Wait for all pods healthy
      kubernetes.core.k8s_info:
        api_version: v1
        kind: Pod
        namespace: production
        label_selectors:
          - app=myapp
      register: pods
      until: pods.resources | selectattr('status.phase', 'equalto', 'Running') | list | length == 3
      retries: 30
      delay: 10

    - name: Drop old user (after grace period)
      postgresql_user:
        name: "{{ old_user }}"
        state: absent
      become: yes
      become_user: postgres
      when: cleanup_old_user | default(false)

# cert-manager-issuer.yaml
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: admin@example.com
    privateKeySecretRef:
      name: letsencrypt-prod
    solvers:
      - http01:
          ingress:
            class: nginx

# tls-certificate.yaml
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: api-example-com
  namespace: production
spec:
  secretName: api-example-com-tls
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
  dnsNames:
    - api.example.com
    - www.api.example.com
  renewBefore: 720h  # Renew 30 days before expiry

# .github/workflows/deploy.yml
name: Deploy with OIDC

on:
  push:
    branches: [main]

permissions:
  id-token: write
  contents: read

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: arn:aws:iam::123456789012:role/GitHubActionsRole
          aws-region: us-east-1

      - name: Get secrets from AWS Secrets Manager
        run: |
          POSTGRES_PASSWORD=$(aws secretsmanager get-secret-value \
            --secret-id production/postgres/password \
            --query SecretString --output text)
          echo "::add-mask::$POSTGRES_PASSWORD"
          echo "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" >> $GITHUB_ENV

      - name: Deploy to Kubernetes
        env:
          KUBECONFIG: ${{ secrets.KUBECONFIG }}
        run: |
          kubectl set env deployment/myapp \
            POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
            -n production

# .github/workflows/vault-auth.yml
name: Deploy with Vault

on:
  push:
    branches: [main]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Authenticate with Vault
        uses: hashicorp/vault-action@v2
        with:
          url: https://vault.example.com
          method: jwt
          jwtGithubAudience: 'https://github.com/myorg'
          jwtMountPath: 'jwt'
          role: 'github-actions'
          secrets: |
            database/creds/myapp postgres_username POSTGRES_USERNAME
            database/creds/myapp postgres_password POSTGRES_PASSWORD

      - name: Deploy
        env:
          POSTGRES_USERNAME: ${{ env.POSTGRES_USERNAME }}
          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
        run: |
          kubectl set env deployment/myapp \
            POSTGRES_USERNAME="$POSTGRES_USERNAME" \
            POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
            -n production

# .github/workflows/sops-deploy.yml
name: Deploy with SOPS

on:
  push:
    branches: [main]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Install SOPS
        uses: mozilla/sops@v3
        with:
          sops_version: '3.8.1'

      - name: Decrypt secrets
        env:
          SOPS_AGE_KEY: ${{ secrets.SOPS_AGE_KEY }}
        run: |
          sops -d secrets/postgres-credentials.yaml > decrypted.yaml
          kubectl apply -f decrypted.yaml
          rm decrypted.yaml

# .gitlab-ci.yml
stages:
  - deploy

deploy_production:
  stage: deploy
  image: bitnami/kubectl:latest
  environment:
    name: production
  before_script:
    - kubectl config use-context production
  script:
    - |
      kubectl create secret generic postgres-credentials \
        --from-literal=username="$POSTGRES_USERNAME" \
        --from-literal=password="$POSTGRES_PASSWORD" \
        --dry-run=client -o yaml | \
        kubectl apply -f -
    - kubectl rollout restart deployment/myapp -n production
    - kubectl rollout status deployment/myapp -n production
  only:
    - main
  when: manual

# .gitlab-ci.yml with Vault
deploy_production:
  stage: deploy
  image: 
    name: vault:latest
    entrypoint: [""]
  before_script:
    - apk add --no-cache curl jq
    - |
      VAULT_TOKEN=$(vault write -field=token auth/jwt/login \
        role=gitlab-ci \
        jwt=$CI_JOB_JWT)
      export VAULT_TOKEN
  script:
    - |
      POSTGRES_CREDS=$(vault read -format=json database/creds/myapp)
      POSTGRES_USERNAME=$(echo $POSTGRES_CREDS | jq -r '.data.username')
      POSTGRES_PASSWORD=$(echo $POSTGRES_CREDS | jq -r '.data.password')

      kubectl create secret generic postgres-credentials \
        --from-literal=username="$POSTGRES_USERNAME" \
        --from-literal=password="$POSTGRES_PASSWORD" \
        --dry-run=client -o yaml | \
        kubectl apply -f -

# .github/workflows/rotate-secrets.yml
name: Rotate Secrets

on:
  schedule:
    - cron: '0 2 1 * *'  # First day of month at 02:00
  workflow_dispatch:

jobs:
  rotate-postgres:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Authenticate with Vault
        uses: hashicorp/vault-action@v2
        with:
          url: https://vault.example.com
          method: jwt
          role: 'github-actions'

      - name: Rotate Postgres password
        run: |
          # Generate new password
          NEW_PASSWORD=$(openssl rand -base64 32)

          # Update in Vault
          vault kv put database/postgres password="$NEW_PASSWORD"

          # Update Kubernetes secret
          kubectl create secret generic postgres-credentials \
            --from-literal=password="$NEW_PASSWORD" \
            --dry-run=client -o yaml | \
            kubectl apply -f -

          # Rolling restart
          kubectl rollout restart deployment/myapp -n production
          kubectl rollout status deployment/myapp -n production