I am trying to create a KEDA scaled job based on RabbitMQ queue trigger but encountered an issue when pods are not scaling at all.
I have created a following Scaled job and lined up messages in the queue but no pods are created. I see this message: Scaling is not performed because triggers are not active
What could be reason that pods are not scaling at all? Thanks for help.
And in Keda logs I see:
2021-12-29T13:50:19.738Z INFO scalemetrics Checking if ScaleJob Scalers are active {"ScaledJob": "celery-rabbitmq-scaledjob-2", "isActive": false, "maxValue": 0, "MultipleScalersCalculation": ""}
2021-12-29T13:50:19.738Z INFO scaleexecutor Scaling Jobs {"scaledJob.Name": "celery-rabbitmq-scaledjob-2", "scaledJob.Namespace": "sandbox-dev", "Number of running Jobs": 0}
2021-12-29T13:50:19.738Z INFO scaleexecutor Scaling Jobs {"scaledJob.Name": "celery-rabbitmq-scaledjob-2", "scaledJob.Namespace": "sandbox-dev", "Number of pending Jobs ": 0}
--
apiVersion: keda.sh/v1alpha1
kind: ScaledJob
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: >
{"apiVersion":"keda.sh/v1alpha1","kind":"ScaledJob","metadata":{"annotations":{},"name":"celery-rabbitmq-scaledjob-2","namespace":"sandbox-dev"},"spec":{"failedJobsHistoryLimit":5,"jobTargetRef":{"activeDeadlineSeconds":3600,"backoffLimit":6,"completions":1,"parallelism":5,"template":{"spec":{"containers":[{"command":["/bin/bash","-c","CELERY_BROKER_URL=amqp://$RABBITMQ_USERNAME:[email protected]:5672
celery worker -A test_data.tasks.all --loglevel=info -c 1 -n
worker.all"],"env":[{"name":"APP_CFG","value":"test_data.config.dev"},{"name":"C_FORCE_ROOT","value":"true"},{"name":"RABBITMQ_USERNAME","valueFrom":{"secretKeyRef":{"key":"rabbitmq-user","name":"develop"}}},{"name":"RABBITMQ_PASSWORD","valueFrom":{"secretKeyRef":{"key":"rabbitmq-pass","name":"develop"}}}],"image":"767487149142.dkr.ecr.us-east-1.amazonaws.com/test-data-celery:DEV-2021.12.27.0","imagePullPolicy":"IfNotPresent","lifecycle":{"postStart":{"exec":{"command":["/bin/sh","-c","echo
startup \u003e\u003e
/tmp/startup.log"]}},"preStop":{"exec":{"command":["/bin/sh","-c","echo
shutdown \u003e\u003e
/tmp/shutdown.log"]}}},"name":"celery-backend","resources":{"limits":{"cpu":"1700m","memory":"3328599654400m"},"requests":{"cpu":"1600m","memory":"3Gi"}},"securityContext":{"allowPrivilegeEscalation":false,"privileged":false,"readOnlyRootFilesystem":false},"terminationMessagePath":"/tmp/termmsg.log","terminationMessagePolicy":"File","volumeMounts":[{"mountPath":"/tmp","name":"temp"},{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount","name":"default-token-s7vl6","readOnly":true}]}]}}},"maxReplicaCount":100,"pollingInterval":5,"rolloutStrategy":"gradual","successfulJobsHistoryLimit":5,"triggers":[{"metadata":{"host":"amqp://guest:[email protected]:5672/vhost","mode":"QueueLength","queueName":"celery","value":"1"},"type":"rabbitmq"}]}}
creationTimestamp: '2021-12-29T13:11:15Z'
finalizers:
- finalizer.keda.sh
generation: 3
managedFields:
- apiVersion: keda.sh/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:finalizers: {}
f:spec:
f:jobTargetRef:
f:template:
f:metadata:
.: {}
f:creationTimestamp: {}
f:scalingStrategy: {}
f:status:
.: {}
f:conditions: {}
manager: keda
operation: Update
time: '2021-12-29T13:11:15Z'
- apiVersion: keda.sh/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:kubectl.kubernetes.io/last-applied-configuration: {}
f:spec:
.: {}
f:failedJobsHistoryLimit: {}
f:jobTargetRef:
.: {}
f:activeDeadlineSeconds: {}
f:backoffLimit: {}
f:completions: {}
f:template:
.: {}
f:spec:
.: {}
f:containers: {}
f:maxReplicaCount: {}
f:pollingInterval: {}
f:rolloutStrategy: {}
f:successfulJobsHistoryLimit: {}
f:triggers: {}
manager: kubectl-client-side-apply
operation: Update
time: '2021-12-29T13:11:15Z'
- apiVersion: keda.sh/v1alpha1
fieldsType: FieldsV1
fieldsV1:
f:spec:
f:jobTargetRef:
f:parallelism: {}
manager: node-fetch
operation: Update
time: '2021-12-29T13:37:11Z'
name: celery-rabbitmq-scaledjob-2
namespace: sandbox-dev
resourceVersion: '222981509'
selfLink: >-
/apis/keda.sh/v1alpha1/namespaces/sandbox-dev/scaledjobs/celery-rabbitmq-scaledjob-2
uid: 9013295a-6ace-48ba-96d3-8810efde1b35
status:
conditions:
- message: ScaledJob is defined correctly and is ready to scaling
reason: ScaledJobReady
status: 'True'
type: Ready
- message: Scaling is not performed because triggers are not active
reason: ScalerNotActive
status: 'False'
type: Active
- status: Unknown
type: Fallback
spec:
failedJobsHistoryLimit: 5
jobTargetRef:
activeDeadlineSeconds: 3600
backoffLimit: 6
completions: 1
parallelism: 1
template:
metadata:
creationTimestamp: null
spec:
containers:
- command:
- /bin/bash
- '-c'
- >-
CELERY_BROKER_URL=amqp://$RABBITMQ_USERNAME:[email protected]:5672
celery worker -A test_data.tasks.all --loglevel=info -c 1
-n worker.all
env:
- name: APP_CFG
value: test_data.config.dev
- name: C_FORCE_ROOT
value: 'true'
- name: RABBITMQ_USERNAME
valueFrom:
secretKeyRef:
key: rabbitmq-user
name: develop
- name: RABBITMQ_PASSWORD
valueFrom:
secretKeyRef:
key: rabbitmq-pass
name: develop
image: >-
111.dkr.ecr.us-east-1.amazonaws.com/test-data-celery:DEV-2021.12.27.0
imagePullPolicy: IfNotPresent
lifecycle:
postStart:
exec:
command:
- /bin/sh
- '-c'
- echo startup >> /tmp/startup.log
preStop:
exec:
command:
- /bin/sh
- '-c'
- echo shutdown >> /tmp/shutdown.log
name: celery-backend
resources:
limits:
cpu: 1700m
memory: 3328599654400m
requests:
cpu: 1600m
memory: 3Gi
securityContext:
allowPrivilegeEscalation: false
privileged: false
readOnlyRootFilesystem: false
terminationMessagePath: /tmp/termmsg.log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /tmp
name: temp
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: default-token-s7vl6
readOnly: true
maxReplicaCount: 100
pollingInterval: 5
rolloutStrategy: gradual
scalingStrategy: {}
successfulJobsHistoryLimit: 5
triggers:
- metadata:
host: amqp://guest:[email protected]:5672/vhost
mode: QueueLength
queueName: celery
value: '1'
type: rabbitmq
CodePudding user response：
mode may not work in some cases.
Try changing
- metadata:
host: amqp://guest:[email protected]:5672/vhost
mode: QueueLength
queueName: celery
value: '1'
to
- metadata:
host: amqp://guest:[email protected]:5672/vhost
queueName: files
queueLength: '1'