From fafa046612de1c93b5ace93371915e8dd51b28f0 Mon Sep 17 00:00:00 2001 From: Francisco Araujo <3943185+araujofrancisco@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:29:17 -0400 Subject: [PATCH 1/4] Upgrade packages versions. Fixed issue with flower argument on monitor. Changed user and group for celery execution to avoid using root. --- api/Dockerfile | 4 ++-- api/Dockerfile.dev | 4 ++-- api/requirements.txt | 6 +++--- api/worker.py | 9 ++++++++- celery-queue/Dockerfile | 4 ++-- celery-queue/Dockerfile.dev | 6 +++--- celery-queue/requirements.txt | 9 ++++----- celery-queue/tasks.py | 9 ++++++++- docker-compose.development.yml | 4 ++-- docker-compose.yml | 4 ++-- 10 files changed, 36 insertions(+), 23 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index e158921..4120e25 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -ENV C_FORCE_ROOT true +#ENV C_FORCE_ROOT true ENV HOST 0.0.0.0 ENV PORT 5001 diff --git a/api/Dockerfile.dev b/api/Dockerfile.dev index b0fd2e3..b6b133d 100644 --- a/api/Dockerfile.dev +++ b/api/Dockerfile.dev @@ -1,8 +1,8 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -ENV C_FORCE_ROOT true +#ENV C_FORCE_ROOT true ENV HOST 0.0.0.0 ENV PORT 5001 diff --git a/api/requirements.txt b/api/requirements.txt index d0b24fd..900d1cf 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,3 +1,3 @@ -Flask==2.0.1 -celery==5.1.2 -redis==3.5.3 \ No newline at end of file +Flask==2.3.2 +celery==5.3.1 +redis==4.6.0 \ No newline at end of file diff --git a/api/worker.py b/api/worker.py index 0ed39d7..c5bd636 100644 --- a/api/worker.py +++ b/api/worker.py @@ -6,4 +6,11 @@ CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') -celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND) +#celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND) +celery = Celery(__name__, + broker=CELERY_BROKER_URL, + backend=CELERY_RESULT_BACKEND, + broker_connection_retry=True, + broker_connection_retry_on_startup=True, + broker_connection_max_retries=10, + ) diff --git a/celery-queue/Dockerfile b/celery-queue/Dockerfile index 3d3621b..9b4f537 100644 --- a/celery-queue/Dockerfile +++ b/celery-queue/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -ENV C_FORCE_ROOT true +#ENV C_FORCE_ROOT true COPY . /queue WORKDIR /queue diff --git a/celery-queue/Dockerfile.dev b/celery-queue/Dockerfile.dev index e3ab9ab..349c467 100644 --- a/celery-queue/Dockerfile.dev +++ b/celery-queue/Dockerfile.dev @@ -1,8 +1,8 @@ -FROM python:3.9-alpine +FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -ENV C_FORCE_ROOT true +#ENV C_FORCE_ROOT true COPY . /queue WORKDIR /queue @@ -11,4 +11,4 @@ RUN pip install -U setuptools pip RUN pip install -r requirements.txt # hot code reloading -CMD watchmedo auto-restart --directory=./ --pattern=*.py --recursive -- celery -A tasks worker --concurrency=1 --loglevel=INFO -E +CMD watchmedo auto-restart --directory=./ --pattern=*.py --recursive -- celery -A tasks worker --concurrency=1 --loglevel=INFO --uid=nobody --gid=nogroup -E diff --git a/celery-queue/requirements.txt b/celery-queue/requirements.txt index edd1ecf..9f06c78 100644 --- a/celery-queue/requirements.txt +++ b/celery-queue/requirements.txt @@ -1,5 +1,4 @@ -celery==5.1.2 -flower==1.0.0 -redis==3.5.3 -watchdog==2.1.5 - +celery==5.3.1 +flower==2.0.0 +redis==4.6.0 +watchdog==3.0.0 \ No newline at end of file diff --git a/celery-queue/tasks.py b/celery-queue/tasks.py index 987cee6..ddfd86b 100644 --- a/celery-queue/tasks.py +++ b/celery-queue/tasks.py @@ -6,7 +6,14 @@ CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'), CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') -celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND) +#celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND) +celery = Celery(__name__, + broker=CELERY_BROKER_URL, + backend=CELERY_RESULT_BACKEND, + broker_connection_retry=True, + broker_connection_retry_on_startup=True, + broker_connection_max_retries=10, + ) @celery.task(name='tasks.add') diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 9aa0265..237db6a 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -17,7 +17,7 @@ services: build: context: ./celery-queue dockerfile: Dockerfile.dev - command: celery -A tasks worker -l info -E + command: celery -A tasks worker -l info --uid=nobody --gid=nogroup -E environment: CELERY_BROKER_URL: redis://redis CELERY_RESULT_BACKEND: redis://redis @@ -30,7 +30,7 @@ services: dockerfile: Dockerfile.dev ports: - "5555:5555" - command: ['celery', 'flower', '-A', 'tasks'] + command: ['celery', '-A', 'tasks', 'flower'] environment: CELERY_BROKER_URL: redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0 diff --git a/docker-compose.yml b/docker-compose.yml index 17ead9e..77eaed8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: build: context: ./celery-queue dockerfile: Dockerfile - command: celery -A tasks worker -l info -E + command: celery -A tasks worker -l info --uid=nobody --gid=nogroup -E environment: CELERY_BROKER_URL: redis://redis CELERY_RESULT_BACKEND: redis://redis @@ -27,7 +27,7 @@ services: dockerfile: Dockerfile ports: - "5555:5555" - command: ['celery', 'flower', '-A', 'tasks'] + command: ['celery', '-A', 'tasks', 'flower'] environment: CELERY_BROKER_URL: redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0 From 40e24ec8de3b691e94775c0cd96e44283520f084 Mon Sep 17 00:00:00 2001 From: Francisco Araujo <3943185+araujofrancisco@users.noreply.github.com> Date: Fri, 14 Jul 2023 09:53:07 -0400 Subject: [PATCH 2/4] Notes on settings to avoid redis memory issues. --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index cd26d1c..c24b19d 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,17 @@ To change the endpoints, update the code in [api/app.py](api/app.py) Task changes should happen in [celery-queue/tasks.py](celery-queue/tasks.py) + +### Setup considerations + +From: [Redis administration](https://redis.io/docs/management/admin/) + +``` +Make sure to set Linux kernel overcommit memory setting to 1. + +This can be done by adding **vm.overcommit_memory=1** to **/etc/sysctl.conf**. Then, reboot or run the command **sysctl vm.overcommit_memory=1** to activate the setting. +``` + --- adapted from [https://github.com/itsrifat/flask-celery-docker-scale](https://github.com/itsrifat/flask-celery-docker-scale) From 701451d94575f0e89ce384b76a7290f0e4b8bb07 Mon Sep 17 00:00:00 2001 From: Francisco Araujo <3943185+araujofrancisco@users.noreply.github.com> Date: Fri, 14 Jul 2023 09:54:40 -0400 Subject: [PATCH 3/4] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c24b19d..a34b8db 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ From: [Redis administration](https://redis.io/docs/management/admin/) ``` Make sure to set Linux kernel overcommit memory setting to 1. -This can be done by adding **vm.overcommit_memory=1** to **/etc/sysctl.conf**. Then, reboot or run the command **sysctl vm.overcommit_memory=1** to activate the setting. +This can be done by adding vm.overcommit_memory=1 to /etc/sysctl.conf. Then, reboot or run the command sysctl vm.overcommit_memory=1 to activate the setting. ``` --- From fedd6641c94fdb4eda4b2703b3029135e0a75bb1 Mon Sep 17 00:00:00 2001 From: Francisco Araujo <3943185+araujofrancisco@users.noreply.github.com> Date: Fri, 14 Jul 2023 11:26:29 -0400 Subject: [PATCH 4/4] Remove commented code. Clean up dockerfiles. --- api/Dockerfile | 1 - api/Dockerfile.dev | 1 - api/worker.py | 2 -- celery-queue/Dockerfile | 1 - celery-queue/Dockerfile.dev | 1 - celery-queue/tasks.py | 1 - 6 files changed, 7 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 4120e25..ba2ceab 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -2,7 +2,6 @@ FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -#ENV C_FORCE_ROOT true ENV HOST 0.0.0.0 ENV PORT 5001 diff --git a/api/Dockerfile.dev b/api/Dockerfile.dev index b6b133d..3f0e7df 100644 --- a/api/Dockerfile.dev +++ b/api/Dockerfile.dev @@ -2,7 +2,6 @@ FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -#ENV C_FORCE_ROOT true ENV HOST 0.0.0.0 ENV PORT 5001 diff --git a/api/worker.py b/api/worker.py index c5bd636..ffd7a06 100644 --- a/api/worker.py +++ b/api/worker.py @@ -5,8 +5,6 @@ CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'), CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') - -#celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND) celery = Celery(__name__, broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND, diff --git a/celery-queue/Dockerfile b/celery-queue/Dockerfile index 9b4f537..a1bed50 100644 --- a/celery-queue/Dockerfile +++ b/celery-queue/Dockerfile @@ -2,7 +2,6 @@ FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -#ENV C_FORCE_ROOT true COPY . /queue WORKDIR /queue diff --git a/celery-queue/Dockerfile.dev b/celery-queue/Dockerfile.dev index 349c467..45bc42f 100644 --- a/celery-queue/Dockerfile.dev +++ b/celery-queue/Dockerfile.dev @@ -2,7 +2,6 @@ FROM python:3.10-alpine ENV CELERY_BROKER_URL redis://redis:6379/0 ENV CELERY_RESULT_BACKEND redis://redis:6379/0 -#ENV C_FORCE_ROOT true COPY . /queue WORKDIR /queue diff --git a/celery-queue/tasks.py b/celery-queue/tasks.py index ddfd86b..579e3d8 100644 --- a/celery-queue/tasks.py +++ b/celery-queue/tasks.py @@ -6,7 +6,6 @@ CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'), CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') -#celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND) celery = Celery(__name__, broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND,