From 73701b4a346c1591bc7a348c4ef85f387a3b959d Mon Sep 17 00:00:00 2001 From: scott Date: Wed, 1 Jul 2026 14:06:38 -0700 Subject: [PATCH] Inherit nginx x509 header config, add operator runbook - LookupConfig: ssl-client-cert / ssl-cert-chain-prefix / certificate-chain-length now fall back to the built-in nginx provider's settings (Config scope, then KC_SPI_X509CERT_LOOKUP_NGINX_* env vars), so switching the provider to "hybrid" needs no duplicated Helm values. Explicit hybrid-scope settings still override. - Add LookupConfigTest covering inheritance, override, and defaults (14 tests total, all passing). - Add RUNBOOK.md: phase-by-phase deployment procedure sized to 30-min test windows, with verification checklists, forged-header negative test, rollback per phase, and failure triage table. - README/Dockerfile.example updated to match; stop tracking target/ build output (.gitignore added). Co-Authored-By: Claude Fable 5 --- .gitignore | 3 + keycloak-hybrid-x509-spi/Dockerfile.example | 3 +- keycloak-hybrid-x509-spi/README.md | 30 +-- keycloak-hybrid-x509-spi/RUNBOOK.md | 188 ++++++++++++++++++ .../example/keycloak/x509/LookupConfig.java | 57 +++++- .../keycloak/x509/LookupConfigTest.java | 55 +++++ ...es.x509.X509ClientCertificateLookupFactory | 1 - .../example/keycloak/x509/CertDecoding.class | Bin 6040 -> 0 bytes .../example/keycloak/x509/CidrMatcher.class | Bin 3002 -> 0 bytes ...9ClientCertificateLookup$TrustResult.class | Bin 2157 -> 0 bytes .../HybridX509ClientCertificateLookup.class | Bin 12498 -> 0 bytes ...idX509ClientCertificateLookupFactory.class | Bin 1880 -> 0 bytes .../x509/LookupConfig$TrustMode.class | Bin 1265 -> 0 bytes .../example/keycloak/x509/LookupConfig.class | Bin 7337 -> 0 bytes .../target/keycloak-hybrid-x509-spi-1.0.0.jar | Bin 20635 -> 0 bytes .../target/maven-archiver/pom.properties | 3 - .../compile/default-compile/createdFiles.lst | 7 - .../compile/default-compile/inputFiles.lst | 5 - .../default-testCompile/createdFiles.lst | 1 - .../default-testCompile/inputFiles.lst | 1 - ...example.keycloak.x509.CertDecodingTest.xml | 73 ------- ...example.keycloak.x509.CertDecodingTest.txt | 4 - .../keycloak/x509/CertDecodingTest.class | Bin 7143 -> 0 bytes 23 files changed, 318 insertions(+), 113 deletions(-) create mode 100644 .gitignore create mode 100644 keycloak-hybrid-x509-spi/RUNBOOK.md create mode 100644 keycloak-hybrid-x509-spi/src/test/java/com/example/keycloak/x509/LookupConfigTest.java delete mode 100644 keycloak-hybrid-x509-spi/target/classes/META-INF/services/org.keycloak.services.x509.X509ClientCertificateLookupFactory delete mode 100644 keycloak-hybrid-x509-spi/target/classes/com/example/keycloak/x509/CertDecoding.class delete mode 100644 keycloak-hybrid-x509-spi/target/classes/com/example/keycloak/x509/CidrMatcher.class delete mode 100644 keycloak-hybrid-x509-spi/target/classes/com/example/keycloak/x509/HybridX509ClientCertificateLookup$TrustResult.class delete mode 100644 keycloak-hybrid-x509-spi/target/classes/com/example/keycloak/x509/HybridX509ClientCertificateLookup.class delete mode 100644 keycloak-hybrid-x509-spi/target/classes/com/example/keycloak/x509/HybridX509ClientCertificateLookupFactory.class delete mode 100644 keycloak-hybrid-x509-spi/target/classes/com/example/keycloak/x509/LookupConfig$TrustMode.class delete mode 100644 keycloak-hybrid-x509-spi/target/classes/com/example/keycloak/x509/LookupConfig.class delete mode 100644 keycloak-hybrid-x509-spi/target/keycloak-hybrid-x509-spi-1.0.0.jar delete mode 100644 keycloak-hybrid-x509-spi/target/maven-archiver/pom.properties delete mode 100644 keycloak-hybrid-x509-spi/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst delete mode 100644 keycloak-hybrid-x509-spi/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst delete mode 100644 keycloak-hybrid-x509-spi/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst delete mode 100644 keycloak-hybrid-x509-spi/target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/inputFiles.lst delete mode 100644 keycloak-hybrid-x509-spi/target/surefire-reports/TEST-com.example.keycloak.x509.CertDecodingTest.xml delete mode 100644 keycloak-hybrid-x509-spi/target/surefire-reports/com.example.keycloak.x509.CertDecodingTest.txt delete mode 100644 keycloak-hybrid-x509-spi/target/test-classes/com/example/keycloak/x509/CertDecodingTest.class diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4fd1d91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +target/ +*.class +.DS_Store diff --git a/keycloak-hybrid-x509-spi/Dockerfile.example b/keycloak-hybrid-x509-spi/Dockerfile.example index 67659c3..70da8a3 100644 --- a/keycloak-hybrid-x509-spi/Dockerfile.example +++ b/keycloak-hybrid-x509-spi/Dockerfile.example @@ -17,7 +17,8 @@ COPY --from=builder /opt/keycloak/ /opt/keycloak/ # Runtime config (usually set via Helm/manifest env instead of baked in): # KC_SPI_X509CERT_LOOKUP_PROVIDER=hybrid -# KC_SPI_X509CERT_LOOKUP_HYBRID_SSL_CLIENT_CERT= +# (header name is inherited from your existing KC_SPI_X509CERT_LOOKUP_NGINX_* env +# vars — leave those in the Helm values as-is) # KC_SPI_X509CERT_LOOKUP_HYBRID_TRUST_MODE=log # then 'enforce' after cycle 3 # KC_TRUSTSTORE_PATHS=/opt/keycloak/conf/truststores # CA bundle (KC 25+) # KC_LOG_LEVEL=INFO,com.example.keycloak.x509:debug # during rollout diff --git a/keycloak-hybrid-x509-spi/README.md b/keycloak-hybrid-x509-spi/README.md index 17a5b2e..1e175e8 100644 --- a/keycloak-hybrid-x509-spi/README.md +++ b/keycloak-hybrid-x509-spi/README.md @@ -27,6 +27,9 @@ Per-request decision logic: No changes are needed to your realm's X509 browser authentication flow — the authenticator receives the cert the same way regardless of source. +> **Deploying this?** Follow the step-by-step operator guide in [RUNBOOK.md](RUNBOOK.md). +> This README is the reference (config options, security rationale, troubleshooting). + --- ## 1. Build @@ -71,9 +74,11 @@ Do NOT use `required`. # Switch the lookup provider from nginx to hybrid KC_SPI_X509CERT_LOOKUP_PROVIDER=hybrid -# Header carrying the user cert — COPY THE VALUE from your existing -# KC_SPI_X509CERT_LOOKUP_NGINX_SSL_CLIENT_CERT (default "ssl-client-cert") -KC_SPI_X509CERT_LOOKUP_HYBRID_SSL_CLIENT_CERT=ssl-client-cert +# Header name: NOTHING TO ADD. The hybrid provider inherits ssl-client-cert, +# ssl-cert-chain-prefix and certificate-chain-length from your existing +# KC_SPI_X509CERT_LOOKUP_NGINX_* settings in the Helm values — leave them in place. +# (Set KC_SPI_X509CERT_LOOKUP_HYBRID_SSL_CLIENT_CERT only to override; the startup +# log line "inherited from the nginx provider config" confirms what was picked up.) # Day-one: permissive mode (headers accepted from anyone, like today, but everything logged) KC_SPI_X509CERT_LOOKUP_HYBRID_TRUST_MODE=log @@ -94,9 +99,9 @@ All options (prefix `spi-x509cert-lookup-hybrid-`): | Option | Default | Meaning | |---|---|---| -| `ssl-client-cert` | `ssl-client-cert` | Header with the user cert (PEM, URL-encoded PEM, or base64 DER — auto-detected) | -| `ssl-cert-chain-prefix` | `ssl-cert-chain` | Optional chain headers `-0..n` | -| `certificate-chain-length` | `1` | Max chain headers to read | +| `ssl-client-cert` | inherits nginx setting, else `ssl-client-cert` | Header with the user cert (PEM, URL-encoded PEM, or base64 DER — auto-detected) | +| `ssl-cert-chain-prefix` | inherits nginx setting, else `ssl-cert-chain` | Optional chain headers `-0..n` | +| `certificate-chain-length` | inherits nginx setting, else `1` | Max chain headers to read | | `trust-mode` | `log` | `log` = honor all headers, warn on untrusted; `enforce` = ignore headers from untrusted peers | | `trusted-proxy-cert-sha256` | — | Comma-separated SHA-256 fingerprints of the F5's client cert(s). **Strongest check.** List old+new during rotations. | | `trusted-proxy-subject-dn` | — | `\|`-separated exact subject DNs (RFC2253). Safe because TLS already validated the chain. Survives rotation. | @@ -115,15 +120,16 @@ browsers use to filter the cert-picker dialog. ## 4. Rollout / test plan (one 30-min cycle each, in order) -**Cycle 0 — no deploy, info gathering.** From the current deployment grab: -`KC_SPI_X509CERT_LOOKUP_NGINX_SSL_CLIENT_CERT` (header name), truststore config, base image -tag, and whether `start --optimized` is used. Ask the F5 team to start on the one-pager -(section 6) in parallel. +**Cycle 0 — no deploy, info gathering.** From the current deployment grab: truststore +config, base image tag, and whether `start --optimized` is used. (The header name is +inherited automatically from the existing `KC_SPI_X509CERT_LOOKUP_NGINX_*` values — no +need to look it up.) Ask the F5 team to start on the one-pager (section 6) in parallel. **Cycle 1 — deploy in log mode (zero behavior change expected).** -Jar + `KC_HTTPS_CLIENT_AUTH=request` + provider=hybrid + `trust-mode=log`. +Jar + `KC_HTTPS_CLIENT_AUTH=request` + provider=hybrid + `trust-mode=log`; keep all +existing `KC_SPI_X509CERT_LOOKUP_NGINX_*` values in the Helm values file untouched. Verify: existing F5 CAC login still works. Then grep logs for `x509-hybrid`: -- startup line shows the parsed config; +- startup lines show the parsed config, including which nginx settings were inherited; - each F5 login logs `remoteAddr=` (→ your CIDR value) and `peer=` (→ `no-tls-client-cert` until the F5 presents one); - `header cert decoded using strategy '...'` (DEBUG) confirms the F5's encoding. diff --git a/keycloak-hybrid-x509-spi/RUNBOOK.md b/keycloak-hybrid-x509-spi/RUNBOOK.md new file mode 100644 index 0000000..155ef19 --- /dev/null +++ b/keycloak-hybrid-x509-spi/RUNBOOK.md @@ -0,0 +1,188 @@ +# Runbook: Rolling out the Hybrid X509 Lookup SPI + +Step-by-step deployment procedure for operators. Background, config reference, and +troubleshooting tables live in [README.md](README.md) — this document is the "what do I +actually type" guide. + +**What we're changing and why (30 seconds):** Today all user traffic passes through the +F5 BigIP, which validates user CAC/x509 certs and forwards the cert to Keycloak in an HTTP +header. New Transit Gateway (TGW) connectivity from brokerage AWS accounts bypasses the F5, +so Keycloak must additionally accept certs **directly from the TLS handshake**, while +making sure the F5's cert header **can't be forged** by TGW clients. This SPI does both. + +``` +Today: user ──mTLS──> F5 (validates cert, adds header) ──TLS──> NLB(L4) ──> Keycloak +New TGW path: user ────────────────────mTLS───────────────────────────> NLB(L4) ──> Keycloak +``` + +--- + +## Roles and coordination + +| Who | Needed for | Task | +|---|---|---| +| Keycloak/EKS operator | every phase | image build, Helm values, log checks | +| F5/BigIP team | Phase 3 | add client cert to the Server SSL profile (see README §6 one-pager) | +| TGW-side tester | Phases 2 & 4 | a browser + CAC on a machine that reaches Keycloak *without* the F5 | + +Each phase is designed to fit one ~30-minute test window and to be independently +verifiable and reversible. **Do not combine phases** — the log output of each phase is the +input to the next. + +--- + +## Phase 0 — Prep (no deployment) + +1. Collect from the current deployment: + - [ ] Keycloak version → set `` in `pom.xml` to match (24.x–26.x). + - [ ] Base image tag and whether the entrypoint uses `start --optimized`. + - [ ] Truststore config: `KC_TRUSTSTORE_PATHS` (KC 25+) or `spi-truststore-file-*` + env vars. The user-CA bundle must be present (it already is if header login works). + - [ ] Confirm the existing `KC_SPI_X509CERT_LOOKUP_NGINX_*` env vars are in the Helm + values (the new provider inherits them automatically — do NOT remove them). +2. Hand the F5 team the one-pager (README §6) so their change request runs in parallel. + Ask them for: cert SHA-256 fingerprint, subject DN, issuing CA chain (PEM), and the + self-IP/SNAT CIDR the BigIP uses toward the Keycloak NLB. +3. Confirm with networking whether the NLB target group preserves client IPs + (needed only if you plan to use the CIDR trust rule; Phase 2 logs verify it either way). +4. Brokerage-user readiness (should already be true): users exist/are federated in + Keycloak, and their certs chain to a CA in the truststore. + +## Phase 1 — Build the jar and image + +```bash +cd keycloak-hybrid-x509-spi +mvn package # or the podman one-liner in README §1 +ls target/keycloak-hybrid-x509-spi-1.0.0.jar +``` + +Add to the Keycloak image (see `Dockerfile.example` for the full picture): + +```dockerfile +COPY keycloak-hybrid-x509-spi-1.0.0.jar /opt/keycloak/providers/ +ENV KC_HTTPS_CLIENT_AUTH=request # BUILD-time option: must precede `kc.sh build` +RUN /opt/keycloak/bin/kc.sh build +``` + +> `request` = certificate **optional**. Never use `required` — it would break every +> certless connection (OIDC backchannels, admin console, health checks). + +## Phase 2 — Deploy in log mode *(zero expected behavior change)* + +Helm values — add **only** these (leave every existing `..._NGINX_...` var untouched): + +```yaml +extraEnv: # or however your chart names it + - name: KC_SPI_X509CERT_LOOKUP_PROVIDER + value: hybrid + - name: KC_SPI_X509CERT_LOOKUP_HYBRID_TRUST_MODE + value: log + - name: KC_LOG_LEVEL + value: INFO,com.example.keycloak.x509:debug +``` + +Deploy, then verify **in this order**: + +1. Startup config was parsed correctly: + ```bash + kubectl logs -n | grep "x509-hybrid: initialized" + kubectl logs -n | grep "inherited from the nginx provider config" + ``` + The `header=` value must match what the F5 injects. A WARN about "no trusted proxy + rules" is expected at this phase. +2. [ ] **Regression check:** a normal user logs in through the F5 with their CAC. Must + work identically to before. +3. Harvest values for later phases from the login's log lines: + ```bash + kubectl logs -n | grep "x509-hybrid:" + ``` + - `remoteAddr=` → the F5's egress IP (→ `TRUSTED_PROXY_CIDRS` value; also tells + you whether client-IP preservation is on — if you see NLB private IPs here, the + CIDR rule is unusable and you must rely on the cert fingerprint). + - `peer=no-tls-client-cert` → expected until Phase 3. + - `header cert decoded using strategy '...'` (DEBUG) → confirms the F5's encoding. +4. [ ] If a TGW-side tester is available: browse to + `https:///realms//account`, expect the browser cert-picker, pick + the CAC cert, expect login. Log shows `using DIRECT TLS peer cert`. + +**Rollback:** set `KC_SPI_X509CERT_LOOKUP_PROVIDER=nginx`, redeploy. (Keep the image; +`https-client-auth=request` is harmless on its own.) + +## Phase 3 — F5 presents its client certificate + +After the F5 team applies their Server SSL profile change: + +1. [ ] Ensure the CA that issued the F5's client cert is in Keycloak's truststore + (append to the bundle in `KC_TRUSTSTORE_PATHS` if it's a different CA). In `request` + mode a client cert that fails validation aborts the handshake — coordinate cert + + truststore in the same window if the CA is new. +2. [ ] Regression check: F5 CAC login still works. +3. Harvest the fingerprint (compare with what the F5 team reported): + ```bash + kubectl logs -n | grep -o 'peer=subject=\[[^]]*\] sha256=[a-f0-9]*' | sort -u + ``` + +**Rollback:** F5 team removes the client cert from the profile; everything else keeps working. + +## Phase 4 — Enforce + +Add the trust rules and flip the mode: + +```yaml + - name: KC_SPI_X509CERT_LOOKUP_HYBRID_TRUSTED_PROXY_CERT_SHA256 + value: "" # comma-separate old,new during F5 cert rotations + - name: KC_SPI_X509CERT_LOOKUP_HYBRID_TRUSTED_PROXY_CIDRS + value: "" # optional second factor; omit if IPs weren't preserved + - name: KC_SPI_X509CERT_LOOKUP_HYBRID_TRUST_MODE + value: enforce +``` + +Verify all three, in order: + +1. [ ] F5 path: CAC login through the F5 works; log shows + `using HEADER cert from trusted proxy (cert-fingerprint)`. +2. [ ] Direct path: TGW-side CAC login works; log shows `using DIRECT TLS peer cert`. +3. [ ] **Negative test (the point of this whole exercise):** from a TGW-side box, send a + forged header and confirm it is rejected: + ```bash + curl -vk "https:///realms//protocol/openid-connect/auth?client_id=account-console&response_type=code&redirect_uri=https:///realms//account/" \ + -H ": $(cat any-user-cert.pem | python3 -c 'import sys,urllib.parse;print(urllib.parse.quote(sys.stdin.read()))')" + ``` + Expected: NO authenticated session, and the log shows + `IGNORED cert header from untrusted peer`. If instead you see the login succeed, + **stop** — the trust rules are wrong; re-check the fingerprint and revert to `log`. + +**Rollback:** `KC_SPI_X509CERT_LOOKUP_HYBRID_TRUST_MODE=log` (keeps both paths working +while you investigate), or provider back to `nginx` (F5 path only, pre-project behavior). + +## Phase 5 — Quiet down (housekeeping) + +```yaml + - name: KC_SPI_X509CERT_LOOKUP_HYBRID_VERBOSE + value: "false" + - name: KC_LOG_LEVEL + value: INFO +``` + +Also confirm (once, not per-deploy): +- [ ] OCSP/CRL revocation checking is enabled in the realm's X509 authenticator config + (Authentication → browser x509 flow → config). The F5 pre-screens users on its path; + the direct path relies entirely on this setting. +- [ ] A calendar/reminder exists for the F5 client-cert expiry: add the *next* cert's + fingerprint to `TRUSTED_PROXY_CERT_SHA256` (comma-separated) **before** rotation. + +--- + +## If something breaks + +| Symptom | Likely cause | Action | +|---|---|---| +| F5 logins fail right after Phase 2 deploy | header name mismatch (inheritance picked wrong value) | check the `initialized`/`inherited` startup lines; set `KC_SPI_X509CERT_LOOKUP_HYBRID_SSL_CLIENT_CERT` explicitly | +| F5 logins fail right after Phase 3 | F5 client cert fails TLS validation (CA missing from truststore) | add issuing CA to the bundle, restart; or F5 team pulls the cert | +| F5 logins fail right after Phase 4 | fingerprint/CIDR wrong or F5 rotated its cert | revert `TRUST_MODE=log`, re-harvest fingerprint from logs | +| TGW users get no cert prompt | truststore CAs don't cover their issuing CA (browser filters the picker), or TLS isn't reaching Keycloak at L4 | verify CA bundle; verify NLB listener is TCP passthrough, not TLS | +| TGW users get a prompt but login fails | user mapping — cert attribute doesn't resolve to a Keycloak user | check the X509 authenticator's user-identity mapping config | +| Everything on fire | — | `KC_SPI_X509CERT_LOOKUP_PROVIDER=nginx` + redeploy = exact pre-project behavior | + +Full log-line reference: README §7. Every decision the provider makes is logged with the +`x509-hybrid:` prefix while `verbose=true`. diff --git a/keycloak-hybrid-x509-spi/src/main/java/com/example/keycloak/x509/LookupConfig.java b/keycloak-hybrid-x509-spi/src/main/java/com/example/keycloak/x509/LookupConfig.java index 21080b0..9bcc1b0 100644 --- a/keycloak-hybrid-x509-spi/src/main/java/com/example/keycloak/x509/LookupConfig.java +++ b/keycloak-hybrid-x509-spi/src/main/java/com/example/keycloak/x509/LookupConfig.java @@ -17,6 +17,11 @@ import java.util.stream.Collectors; * (env: KC_SPI_X509CERT_LOOKUP_HYBRID_) * * Keycloak 26+ also accepts the double-dash form --spi-x509cert-lookup--hybrid--