diff --git a/apps/blog-ahmza-com.yaml b/apps/blog-ahmza-com.yaml new file mode 100644 index 0000000..2c77651 --- /dev/null +++ b/apps/blog-ahmza-com.yaml @@ -0,0 +1,26 @@ +# CI publishes the static site image; this Application makes Git, rather than +# a runner's kubeconfig, the source of truth for which image Kubernetes runs. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: blog-ahmza-com + namespace: argocd + # No cascading-deletion finalizer during initial migration: removing this + # Application must not unexpectedly delete the public site's resources. +spec: + project: default + source: + repoURL: https://tea.ahmza.com/ahmza.com/k8s-starter-pack.git + targetRevision: main + path: apps/blog-ahmza-com + destination: + server: https://kubernetes.default.svc + namespace: blog-ahmza-com + syncPolicy: + automated: + # Review the initial sync before allowing Git deletions to remove live + # resources. Enabling prune later is separate from cascading app deletion. + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/apps/blog-ahmza-com/deployment.yaml b/apps/blog-ahmza-com/deployment.yaml new file mode 100644 index 0000000..c759304 --- /dev/null +++ b/apps/blog-ahmza-com/deployment.yaml @@ -0,0 +1,78 @@ +# Hugo generates the complete site at image-build time. SWS serves only those +# immutable files, with no writable content volume or Kubernetes API access. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: blog + namespace: blog-ahmza-com +spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + # Keep the existing site available while replacement Pods become ready. + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: blog + template: + metadata: + labels: + app: blog + spec: + automountServiceAccountToken: false + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + # Ignore tainted control-plane nodes when balancing the two workers; + # otherwise a surge Pod could be blocked by an unusable third node. + nodeTaintsPolicy: Honor + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: blog + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: blog + # Bootstrap with the verified published amd64/arm64 index. CI updates + # this line to each build's digest, never a mutable latest tag. + # yamllint disable-line rule:line-length + image: tea.ahmza.com/ahmza/blog-ahmza-com@sha256:25dce7a45c8869879bbb6f056b3e4d18971c7da542255867ad83f1db26ffefc3 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 80 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + # SWS does not declare USER in the image. An explicit unprivileged + # identity lets Kubernetes enforce runAsNonRoot safely. + runAsUser: 10001 + runAsGroup: 10001 + capabilities: + drop: + - ALL + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 2 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 30 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 250m + memory: 128Mi diff --git a/apps/blog-ahmza-com/ingress.yaml b/apps/blog-ahmza-com/ingress.yaml new file mode 100644 index 0000000..7215651 --- /dev/null +++ b/apps/blog-ahmza-com/ingress.yaml @@ -0,0 +1,27 @@ +# Share ingress-nginx at 10.10.60.12. DNS and any external Caddy/NetBird +# hostname routing must also be pointed at this ingress before public cutover. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: blog + namespace: blog-ahmza-com + annotations: + # Reuse the production DNS-01 issuer already used by the cluster's sites. + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + ingressClassName: nginx + tls: + - hosts: + - blog.ahmza.com + secretName: blog-tls + rules: + - host: blog.ahmza.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: blog + port: + name: http diff --git a/apps/blog-ahmza-com/namespace.yaml b/apps/blog-ahmza-com/namespace.yaml new file mode 100644 index 0000000..f9c7671 --- /dev/null +++ b/apps/blog-ahmza-com/namespace.yaml @@ -0,0 +1,5 @@ +# Isolate the blog's lifecycle and access controls from unrelated applications. +apiVersion: v1 +kind: Namespace +metadata: + name: blog-ahmza-com diff --git a/apps/blog-ahmza-com/service.yaml b/apps/blog-ahmza-com/service.yaml new file mode 100644 index 0000000..41cfa9c --- /dev/null +++ b/apps/blog-ahmza-com/service.yaml @@ -0,0 +1,15 @@ +# A stable in-cluster address for ingress-nginx; no extra MetalLB IP is needed. +apiVersion: v1 +kind: Service +metadata: + name: blog + namespace: blog-ahmza-com +spec: + type: ClusterIP + selector: + app: blog + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP diff --git a/docs/13-blog-gitops.md b/docs/13-blog-gitops.md new file mode 100644 index 0000000..7c0efaa --- /dev/null +++ b/docs/13-blog-gitops.md @@ -0,0 +1,139 @@ +# 13 — Deploy the Hugo blog through Argo CD + +The blog source lives in `ahmza/blog-ahmza-com`. This GitOps repository owns +**desired state**: the Kubernetes resources and exact image that should run. +Argo CD compares those files with the cluster and applies changes. CI no longer +needs a kubeconfig, Kubernetes credentials, or a per-job NetBird connection. +The runner still needs working HTTPS access to Gitea and its container registry. + +## What owns what + +```text +blog source push to main + -> Gitea Actions builds amd64 + arm64 images + -> registry returns the multi-architecture image digest + -> CI commits that digest to this repository + -> app-of-apps discovers apps/blog-ahmza-com.yaml + -> Argo CD applies apps/blog-ahmza-com/ to the cluster +``` + +A **digest** (`sha256:...`) identifies exact image contents, unlike the mutable +`latest` tag. Both architecture variants are referenced by the same index digest, +so the Pi and x86 workers select their matching image automatically. + +The new files are: + +- `apps/blog-ahmza-com.yaml`: the Argo CD Application, meaning which Git path to + watch and where to apply it. +- `apps/blog-ahmza-com/namespace.yaml`: the namespace, an isolation boundary for + this application's resources. +- `apps/blog-ahmza-com/deployment.yaml`: two Pods (running container instances), + with zero-unavailable rolling updates and the SWS `/health` probes. +- `apps/blog-ahmza-com/service.yaml`: a ClusterIP Service, a stable internal + destination for those replaceable Pods. +- `apps/blog-ahmza-com/ingress.yaml`: routes HTTPS `blog.ahmza.com` to that Service + through the already-running ingress-nginx at `10.10.60.12`. + +There is no PVC or extra Caddy Deployment. The image includes the full generated +site, and TLS is handled by the existing ingress/cert-manager infrastructure. +The Application starts with automatic pruning disabled and no cascading-delete +finalizer. Review deletion behavior before changing either: removing the +Application initially leaves its workloads running rather than deleting them. + +## Prepare credentials before enabling the new workflow + +In the **blog source repository**, configure these Gitea Actions secrets: + +| Secret | Purpose | +|---|---| +| `GITEATOKEN` | Existing credential that can push `ahmza/blog-ahmza-com` registry packages | +| `GITOPS_TOKEN` | Separate bot PAT permitted to read/write `ahmza.com/k8s-starter-pack` | + +The bot must be allowed to push to the GitOps repository's `main` branch. Scope +its token and membership as narrowly as Gitea supports; it does not need +Kubernetes credentials or administrator access. Do not commit either token. + +At preparation time, the published blog image was readable with an anonymous +registry bearer-token exchange. Therefore no image pull secret is shipped in +these manifests. If package visibility is changed to private, manage a read-only +registry credential in this namespace through the existing SOPS mechanism +(`docs/10-secrets.md`) and reference it using `imagePullSecrets`. The source +repository and registry package have separate visibility settings. + +## First deployment: GitOps first, workflow second + +1. Review and commit the Application, manifests, and documentation in this + repository, then push to `main`. They must exist remotely before the new CI + workflow can update the Deployment file. Nothing local is visible to Argo CD. + +2. The existing app-of-apps watches `apps/` non-recursively. It should create the + child Application without another manual `kubectl apply`. Initial deployment + uses the already-published image index pinned in the manifest. Check: + + ```sh + kubectl -n argocd get application blog-ahmza-com + kubectl -n blog-ahmza-com get deployment,pods,service,ingress,certificate + kubectl -n blog-ahmza-com rollout status deployment/blog --timeout=5m + ``` + + These commands are diagnostics, not the deployment mechanism. If the child + Application does not appear, inspect `app-of-apps` sync status and conditions. + Do not solve this by letting CI apply resources directly again. + +3. Verify the hostname without changing public routing first: + + ```sh + curl --fail --show-error --resolve blog.ahmza.com:443:10.10.60.12 \ + https://blog.ahmza.com/ + ``` + + TLS verification remains enabled. If certificate issuance is still pending, + inspect `certificate/blog-tls` and cert-manager events; do not bypass TLS. + Reuse of the ready production DNS-01 issuer follows the other cluster sites. + +4. Update external routing only after the new site is healthy. For the existing + NetBird -> ingress1 Caddy -> Kubernetes topology, add `blog.ahmza.com` to the + Caddy matcher forwarding to `10.10.60.12:443`, preserving the hostname for TLS + SNI and HTTP Host. Check that the public NetBird service sends that hostname + to Caddy. Set the LAN's Technitium override to `10.10.60.12` if LAN clients + should use the internal ingress directly. These settings are outside this + repository and have not been changed by adding an Ingress manifest. + + Keep public TLS termination, compression and security headers owned by the + intended proxy layer; SWS deliberately has compression/security headers off. + Cloudflare Full/Full (strict), if used, requires HTTPS to Cloudflare's actual + origin. An encrypted NetBird hop later in the path is not a substitute for + that TLS connection. + +5. With `GITOPS_TOKEN` configured, commit/push the updated workflow and helper + files in the blog source repository. That push builds the site and updates + only `apps/blog-ahmza-com/deployment.yaml` here to the new digest. + +6. Check both the Gitea job and Argo CD. A successful CI job means the image and + desired-state commit were published, NOT that Kubernetes is already healthy. + Argo CD applies the commit asynchronously; inspect its sync/health status and + the Deployment's rollout before declaring the release successful. + +7. After a successful cutover, remove obsolete blog repository secrets + `KUBE_CONFIG`, `NETBIRD_SETUP_KEY`, and `NETBIRD_URL` if no other workflow uses + them. If the kubeconfig had broad cluster access, revoke/rotate its credential + separately; deleting a CI secret does not revoke the underlying credential. + Remove no account-wide NetBird policies/keys without checking other users. + +## Concurrent publishing and rollback + +The blog workflow serializes its publishes. Other applications can still push to +this GitOps repository, so it retries non-fast-forward pushes with a rebase. +Conflicting changes fail visibly instead of force-pushing over someone else's +work. The digest updater refuses missing/duplicate image lines or a different +image repository. + +To roll back, revert the GitOps commit changing the blog's image digest, then +push that revert. Keep the old digest in the registry so it remains pullable. +Do not use `kubectl set image`: Argo CD's self-healing would put the Git version +back. A future source build will deliberately publish another desired-state +commit. + +> Checkpoint: if an image build succeeds but the GitOps push fails, should Argo +> deploy that image? No: the desired-state commit did not land, so it should keep +> the previous image. This separation is why CI no longer needs cluster access. diff --git a/docs/network.md b/docs/network.md new file mode 100644 index 0000000..a2f6da7 --- /dev/null +++ b/docs/network.md @@ -0,0 +1,119 @@ +# Network map + +Every real IP address this cluster hands out, and exactly what's reachable +there. This file isn't a template with the rest of the repo — it's a log +of *your* actual setup, meant to be updated every time an address is +assigned or changed, so you (or Claude) never have to reverse-engineer +"wait, what's running on .241 again?" from `kubectl` output. + +> **Keep this current.** If you're working with Claude and an IP changes — +> a new app gets a `LoadBalancer` address, the MetalLB pool is resized, +> Argo CD's address changes — ask it to update this file, or just point it +> here after any networking change. See `CLAUDE.md` for the standing +> instruction that tells Claude to do this on its own when it makes such a +> change itself. + +## The five nodes and the VIP + +None of these are from the MetalLB pool below — they're the machines +themselves and the shared address Talos elects between the three control-plane +nodes. Set in [`talos/README.md`](../talos/README.md); see +[`docs/02-talos-bootstrap.md`](02-talos-bootstrap.md) for what the VIP is +and why it's separate from any one node's address. Port 6443 is the Kubernetes +API on control-plane nodes; port 50000 is Talos's TLS-protected management API +on every node. + +| Address | What | Port(s) | URL(s) | +|---|---|---|---| +| `10.10.10.200` | `talos-455-i3b` — control plane (Raspberry Pi 4) | 6443, 50000 | `https://10.10.10.200:6443`, `https://10.10.10.200:50000` | +| `10.10.10.201` | `talos-9xy-iuk` — control plane (Raspberry Pi 4) | 6443, 50000 | `https://10.10.10.201:6443`, `https://10.10.10.201:50000` | +| `10.10.10.202` | `talos-k1b-7j5` — control plane (Raspberry Pi 4) | 6443, 50000 | `https://10.10.10.202:6443`, `https://10.10.10.202:50000` | +| `10.10.10.203` | `talos-rss-v5l` — worker01 (Proxmox VM on Gimli) | 50000 | `https://10.10.10.203:50000` | +| `10.10.10.204` | `talos-vba-ood` — worker02 (Proxmox VM on Titus) | 50000 | `https://10.10.10.204:50000` | +| `10.10.10.210` | Cluster VIP (use this one, not a node's own address) | 6443 | `https://10.10.10.210:6443` | + +## NetBird overlay + +These addresses exist only inside the private NetBird overlay and are subject +to NetBird access policies. All Talos nodes use the official +`siderolabs/netbird` system extension. + +| Address | What | Port(s) | URL(s) | +|---|---|---|---| +| `100.71.24.9` | `prd-vps-hel-01` — external Grafana and VictoriaMetrics host; internal name `prd-vps-hel-01.ahmza.com` (authoritative zone in Technitium) | 3000, 8428 | `http://prd-vps-hel-01.ahmza.com:3000`, `http://prd-vps-hel-01.ahmza.com:8428` | +| `100.71.236.121` | `ingress1` — Caddy HTTP origin gateway used by NetBird Reverse Proxy; the WireGuard overlay encrypts this leg | 80 | `http://100.71.236.121` | +| `100.71.121.67` | `talos-455-i3b` — control plane NetBird address | 50000 | `https://100.71.121.67:50000` | +| `100.71.157.217` | `talos-9xy-iuk` — control plane NetBird address | 50000 | `https://100.71.157.217:50000` | +| `100.71.242.40` | `talos-k1b-7j5` — control plane NetBird address | 50000 | `https://100.71.242.40:50000` | +| `100.71.3.210` | `talos-rss-v5l` — worker01 NetBird address | 50000 | `https://100.71.3.210:50000` | +| `100.71.29.51` | `talos-vba-ood` — worker02 NetBird address | 50000 | `https://100.71.29.51:50000` | + +## MetalLB pool + +This dedicated service range is advertised to the MikroTik at +`10.10.10.1` with BGP by the two workers; it is not an L2 or DHCP pool. + +| | | +|---|---| +| Range | `10.10.60.10-10.10.60.250` | +| Router / ASN | MikroTik RB5009 `10.10.10.1` / `64512` | +| Worker ASN | `64520` | +| Configured in | [`apps/metallb/templates/`](../apps/metallb/templates/) | + +## Allocated addresses (from the MetalLB pool) + +| Address | What | Namespace | Port(s) | URL | Source | +|---|---|---|---|---|---| +| `10.10.60.10` | Argo CD UI/API (`argocd-server`) | `argocd` | 443 | `https://10.10.60.10` | Patched to `LoadBalancer` by hand — see [`docs/05-metallb.md`](05-metallb.md) | +| `10.10.60.11` | Reserved for the Authentik LDAP Outpost used by Jellyfin | `authentik` | 636 | `ldaps://10.10.60.11:636` | Planned Authentik deployment; verify the address and TLS certificate before migrating Jellyfin | +| `10.10.60.12` | ingress-nginx (shared, routes by hostname) | `ingress-nginx` | 80, 443 | `http://10.10.60.12`, `https://10.10.60.12` | [`apps/ingress-nginx/values.yaml`](../apps/ingress-nginx/values.yaml) — see [`docs/11-ingress.md`](11-ingress.md) | + +Add a row here every time you add a new app with its own dedicated +`LoadBalancer` Service (see [`docs/08-adding-new-apps.md`](08-adding-new-apps.md)). +Most apps should route through ingress-nginx by hostname instead — see the +table below — and only need a dedicated address if they can't speak HTTP +(a raw TCP service, for example). Leave the URL column's port number off +only when it's the protocol default (80 for `http://`, 443 for `https://`) +— include it otherwise. + +## Hostnames routed through ingress-nginx + +Every real hostname routed to ingress-nginx's shared address +(`10.10.60.12`), and which app it reaches. Public DNS is hosted on +Cloudflare; internal overrides are served by Technitium where noted. + +| Hostname | App | Namespace | TLS | Source | +|---|---|---|---|---| +| `demo.ahmza.com` | Demo app | `demo` | Yes — `letsencrypt-prod`, real trusted cert (see [`docs/12-cert-manager.md`](12-cert-manager.md)) | [`apps/demo/ingress.yaml`](../apps/demo/ingress.yaml) | +| `auth.ahmza.com` | Authentik identity provider | `authentik` | Yes — `letsencrypt-prod`, trusted certificate and browser login verified; Technitium routes LAN clients to `10.10.60.12`. For public traffic, NetBird terminates public TLS and proxies HTTP through its encrypted WireGuard overlay to Caddy on `ingress1`; Caddy then uses certificate-validated HTTPS with `auth.ahmza.com` as SNI and the HTTP `Host` when connecting to ingress-nginx at `10.10.60.12:443` | [`apps/authentik/values.yaml`](../apps/authentik/values.yaml) | +| `mc.ahmza.com` | Packwiz modpack files | `mcpacks` | Yes — `letsencrypt-prod`, real trusted certificate | [`apps/mcpacks/ingress.yaml`](../apps/mcpacks/ingress.yaml) | +| `bazarr.ahmza.com` | Bazarr subtitle manager | `bazarr` | Yes — `letsencrypt-prod`, real trusted certificate; Technitium overrides Cloudflare's NetBird route on the LAN | [`apps/bazarr/ingress.yaml`](../apps/bazarr/ingress.yaml) | +| `soulsync.ahmza.com` | SoulSync media discovery | `soulsync` | Configured for `letsencrypt-prod`; verify the certificate after the first deployment; Technitium routes LAN clients to `10.10.60.12` | [`apps/soulsync/ingress.yaml`](../apps/soulsync/ingress.yaml) | + +Add a row here every time you point a new hostname at this cluster (see +[`docs/11-ingress.md`](11-ingress.md)). The DNS record itself lives in +Cloudflare, not in this repo — this table is just the log of what's real. + +## Planned hostname migrations (not yet verified live) + +- `https://blog.ahmza.com` → the `blog` Service in `blog-ahmza-com`, through + the existing ingress-nginx address `https://10.10.60.12`. Defined in + [`apps/blog-ahmza-com/ingress.yaml`](../apps/blog-ahmza-com/ingress.yaml). + Uses `letsencrypt-prod`; verify certificate issuance and update internal + DNS/external Caddy routing before cutover. No new MetalLB address is allocated. + Move this entry into the live hostname table only after verification. See + [`docs/13-blog-gitops.md`](13-blog-gitops.md) for the migration checklist. + +## Free in the pool + +Addresses from the range above that aren't spoken for yet. Update this +alongside the table above so it's obvious at a glance what's safe to hand +to the next app. + +- `10.10.60.13-10.10.60.250` + +(`whoami`'s former `10.10.60.12` was reclaimed here after that app was +removed as the docs/08 exercise — it now belongs to ingress-nginx instead. +`demo`'s former `10.10.60.11` was reclaimed the same way after that app +moved to routing through ingress-nginx by hostname instead — see +docs/11-ingress.md.)