User:Bstorm/snippits

Starting a page to collect random things that are useful that I'd rather not forget:

Toolforge Kubernetes

One way to find historical pods run in a namespace is to abuse prometheus: container_last_seen{namespace="$namespace", container_name="webservice"}

Fixing podpresets

In case I ever find that lots of these are messed up again, you can probably fix them with this kind of deal. This was to remove a mistaken "s" at the end of the dumps labstore1007 volume mount.

#!/bin/bash
# Run this script with your root/cluster admin account as appropriate.
# This will fix the dumps mounts for all existing tools.

set -Eeuo pipefail

declare -a namespaces
readarray -t namespaces < <(kubectl get ns -l tenancy=tool --no-headers=true -o custom-columns=:metadata.name)

for ns in "${namespaces[@]}"
do
    if [[ $(kubectl get podpreset -n "${ns}" --template='{{range .spec.volumeMounts}}{{ if eq .mountPath "/mnt/nfs/dumps-labstore1007.wikimedia.orgs" }}true{{end}}{{end}}' mount-toolforge-vols) == "true" ]]; then
        echo "Fixing ${ns}"
        kubectl -n "$ns" delete podpresets mount-toolforge-vols
        cat <<EOF | kubectl --namespace "$ns" apply -f -
apiVersion: settings.k8s.io/v1alpha1
kind: PodPreset
metadata:
  name: mount-toolforge-vols
  namespace: ${ns}
spec:
  env:
  - name: HOME
    value: /data/project/${ns:5}
  selector:
    matchLabels:
      toolforge: tool
  volumeMounts:
  - mountPath: /public/dumps
    name: dumps
    readOnly: true
  - mountPath: /mnt/nfs/dumps-labstore1007.wikimedia.org
    name: dumpsrc1
    readOnly: true
  - mountPath: /mnt/nfs/dumps-labstore1006.wikimedia.org
    name: dumpsrc2
    readOnly: true
  - mountPath: /data/project
    name: home
  - mountPath: /etc/wmcs-project
    name: wmcs-project
    readOnly: true
  - mountPath: /data/scratch
    name: scratch
  - mountPath: /etc/ldap.conf
    name: etcldap-conf
    readOnly: true
  - mountPath: /etc/ldap.yaml
    name: etcldap-yaml
    readOnly: true
  - mountPath: /etc/novaobserver.yaml
    name: etcnovaobserver-yaml
    readOnly: true
  - mountPath: /var/lib/sss/pipes
    name: sssd-pipes
  volumes:
  - hostPath:
      path: /public/dumps
      type: Directory
    name: dumps
  - hostPath:
      path: /mnt/nfs/dumps-labstore1007.wikimedia.org
      type: Directory
    name: dumpsrc1
  - hostPath:
      path: /mnt/nfs/dumps-labstore1006.wikimedia.org
      type: Directory
    name: dumpsrc2
  - hostPath:
      path: /data/project
      type: Directory
    name: home
  - hostPath:
      path: /etc/wmcs-project
      type: File
    name: wmcs-project
  - hostPath:
      path: /data/scratch
      type: Directory
    name: scratch
  - hostPath:
      path: /etc/ldap.conf
      type: File
    name: etcldap-conf
  - hostPath:
      path: /etc/ldap.yaml
      type: File
    name: etcldap-yaml
  - hostPath:
      path: /etc/novaobserver.yaml
      type: File
    name: etcnovaobserver-yaml
  - hostPath:
      path: /var/lib/sss/pipes
      type: Directory
    name: sssd-pipes
EOF
        echo "created new preset for $ns"
        echo "Finished $ns"
    fi
done

echo "*********************"
echo "Done!"