Lots of configuration for Slurm

slurm/manifests/configs/slurm.conf

@@ -38,7 +38,7 @@ ProctrackType=proctrack/linuxproc
 #PropagateResourceLimits=
 #PropagateResourceLimitsExcept=
 #RebootProgram=
-ReturnToService=1
+ReturnToService=2
 SlurmctldPidFile=/var/run/slurmctld.pid
 SlurmctldPort=6817
 SlurmdPidFile=/var/run/slurmd.pid
@@ -78,7 +78,7 @@ KillWait=30
 MinJobAge=300
 #OverTimeLimit=0
 SlurmctldTimeout=120
-SlurmdTimeout=300
+SlurmdTimeout=5
 #UnkillableStepTimeout=60
 #VSizeFactor=0
 Waittime=0
@@ -90,6 +90,7 @@ Waittime=0
 #SchedulerTimeSlice=30
 SchedulerType=sched/backfill
 SelectType=select/cons_tres # needed for supporting dynamic nodes
+#SelectType=select/cons_res
 #
 #
 # JOB PRIORITY
@@ -123,10 +124,11 @@ JobCompType=jobcomp/none
 #JobCompUser=
 #JobContainerType=job_container/none
 JobAcctGatherFrequency=30
-JobAcctGatherType=jobacct_gather/none
-SlurmctldDebug=info
+JobAcctGatherType=jobacct_gather/linux
+SlurmctldDebug=debug
 #SlurmctldLogFile=
-SlurmdDebug=info
+SlurmctldParameters=cloud_reg_addrs
+SlurmdDebug=debug
 #SlurmdLogFile=
 #SlurmSchedLogFile=
 #SlurmSchedLogLevel=
@@ -144,7 +146,7 @@ SlurmdDebug=info
 #SuspendRate=
 #SuspendTime=
 #
-#
-MaxNodeCount=100 # dynamic nodes
 # COMPUTE NODES
+MaxNodeCount=100
+#NodeName=compute-[0-10] CPUs=2 State=UNKNOWN
 PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP

slurm/manifests/kustomization.yaml

@@ -11,7 +11,10 @@ resources:
   - serviceaccount.yaml
   - clusterrolebinding.yaml
   - claim.yaml
+
   - servicemonitor.yaml
+  - scaledobject.yaml
+  - serviceaccount-slurm-metrics.yaml
 
 configMapGenerator:
 - name: slurm-conf

slurm/manifests/namespace.yaml

@@ -3,4 +3,4 @@ kind: Namespace
 metadata:
   name: slurm-system
   labels:
-    openshift.io/cluster-monitoring: 'true'
+    openshift.io/cluster-monitoring: 'true'

slurm/manifests/scaledobject.yaml

@@ -0,0 +1,50 @@
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  # annotations:
+  #   autoscaling.keda.sh/paused-replicas: "0" 
+  name: scaledobject
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: StatefulSet
+    name: compute
+  cooldownPeriod:  60
+  maxReplicaCount: 10
+  minReplicaCount: 0
+  fallback: 
+    failureThreshold: 3
+    replicas: 1
+  pollingInterval: 30
+  # advanced:
+  #   scalingModifiers:
+  #     activationTarget: "2"
+  #   horizontalPodAutoscalerConfig:
+  #     behavior:
+  #       scaleDown:
+  #         stabilizationWindowSeconds: 150
+  #         policies:
+  #           - type: Percent
+  #             value: 100
+  #             periodSeconds: 15
+  triggers:
+  - type: prometheus 
+    metadata:
+      serverAddress: https://thanos-querier.openshift-monitoring.svc.cluster.local:9092
+      namespace: slurm-system
+      query: slurm_queue_pending
+      threshold: '1'
+      authModes: bearer
+    authenticationRef:
+      name: prom-triggerauthentication
+      kind: TriggerAuthentication
+---
+apiVersion: keda.sh/v1alpha1
+kind: TriggerAuthentication
+metadata:
+  name: prom-triggerauthentication
+spec:
+  secretTargetRef:
+  - parameter: bearerToken
+    name: slurm-metrics-token-zkqtg
+    key: token

slurm/manifests/service.yaml

@@ -10,7 +10,6 @@ spec:
     name: slurmctld
   - port: 8080
     name: scrape
-  clusterIP: None
   selector:
     app: slurm
-    component: head
+    component: head

slurm/manifests/serviceaccount-slurm-metrics.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: slurm-metrics
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: thanos-metrics-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: slurm-metrics
+subjects:
+  - kind: ServiceAccount
+    name: slurm-metrics
+    namespace: slurm-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: thanos-metrics-reader

slurm/manifests/serviceaccount.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: slurm
+  name: slurm

slurm/manifests/servicemonitor.yaml

@@ -1,16 +1,17 @@
+# https://docs.openshift.com/container-platform/4.13/monitoring/managing-metrics.html
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: slurm
 spec:
   endpoints:
   - interval: 30s
+    scrapeTimeout: 30s
     port: scrape
     scheme: http
   selector:
     matchLabels:
       app: slurm
-      component: head
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

slurm/manifests/statefulset-compute.yaml

@@ -1,3 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: compute
+  labels:
+    app: slurm
+    component: compute
+spec:
+  ports:
+  - port: 6818
+    name: slurmd
+  selector:
+    app: slurm
+    component: compute
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -15,6 +29,7 @@ spec:
   serviceName: "compute"
   replicas: 1
   minReadySeconds: 10
+  terminationGracePeriodSeconds: 60
   template:
     metadata:
       labels:

slurm/manifests/statefulset-head.yaml

@@ -25,6 +25,7 @@ spec:
       dnsConfig:
         searches:
         - slurm.slurm-system.svc.cluster.local
+        - compute.slurm-system.svc.cluster.local
       enableServiceLinks: false
       terminationGracePeriodSeconds: 10
       serviceAccountName: slurm