Bump python version to 3.10 and libraries to more recent versions

README.md

@@ -38,6 +38,17 @@ To change the endpoints, update the code in [api/app.py](api/app.py)
 
 Task changes should happen in [celery-queue/tasks.py](celery-queue/tasks.py) 
 
+
+### Setup considerations
+
+From: [Redis administration](https://redis.io/docs/management/admin/)
+
+```
+Make sure to set Linux kernel overcommit memory setting to 1.
+
+This can be done by adding vm.overcommit_memory=1 to /etc/sysctl.conf. Then, reboot or run the command sysctl vm.overcommit_memory=1 to activate the setting.
+```
+
 ---
 
 adapted from [https://github.com/itsrifat/flask-celery-docker-scale](https://github.com/itsrifat/flask-celery-docker-scale)

api/Dockerfile

@@ -1,8 +1,7 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
 
 ENV CELERY_BROKER_URL redis://redis:6379/0
 ENV CELERY_RESULT_BACKEND redis://redis:6379/0
-ENV C_FORCE_ROOT true
 
 ENV HOST 0.0.0.0
 ENV PORT 5001

api/Dockerfile.dev

@@ -1,8 +1,7 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
 
 ENV CELERY_BROKER_URL redis://redis:6379/0
 ENV CELERY_RESULT_BACKEND redis://redis:6379/0
-ENV C_FORCE_ROOT true
 
 ENV HOST 0.0.0.0
 ENV PORT 5001

api/requirements.txt

@@ -1,3 +1,3 @@
-Flask==2.0.1
-celery==5.1.2
-redis==3.5.3
+Flask==2.3.2
+celery==5.3.1
+redis==4.6.0

api/worker.py

@@ -5,5 +5,10 @@
 CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'),
 CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379')
 
-
-celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND)
+celery = Celery(__name__, 
+                broker=CELERY_BROKER_URL, 
+                backend=CELERY_RESULT_BACKEND,
+                broker_connection_retry=True,
+                broker_connection_retry_on_startup=True,
+                broker_connection_max_retries=10,
+                )

celery-queue/Dockerfile

@@ -1,8 +1,7 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
 
 ENV CELERY_BROKER_URL redis://redis:6379/0
 ENV CELERY_RESULT_BACKEND redis://redis:6379/0
-ENV C_FORCE_ROOT true
 
 COPY . /queue
 WORKDIR /queue

celery-queue/Dockerfile.dev

@@ -1,8 +1,7 @@
-FROM python:3.9-alpine
+FROM python:3.10-alpine
 
 ENV CELERY_BROKER_URL redis://redis:6379/0
 ENV CELERY_RESULT_BACKEND redis://redis:6379/0
-ENV C_FORCE_ROOT true
 
 COPY . /queue
 WORKDIR /queue
@@ -11,4 +10,4 @@ RUN pip install -U setuptools pip
 RUN pip install -r requirements.txt
 
 # hot code reloading
-CMD watchmedo auto-restart --directory=./ --pattern=*.py --recursive -- celery -A tasks worker --concurrency=1 --loglevel=INFO -E
+CMD watchmedo auto-restart --directory=./ --pattern=*.py --recursive -- celery -A tasks worker --concurrency=1 --loglevel=INFO --uid=nobody --gid=nogroup -E

celery-queue/requirements.txt

@@ -1,5 +1,4 @@
-celery==5.1.2
-flower==1.0.0
-redis==3.5.3
-watchdog==2.1.5
-
+celery==5.3.1
+flower==2.0.0
+redis==4.6.0
+watchdog==3.0.0

celery-queue/tasks.py

@@ -6,7 +6,13 @@
 CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'),
 CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379')
 
-celery = Celery('tasks', broker=CELERY_BROKER_URL, backend=CELERY_RESULT_BACKEND)
+celery = Celery(__name__, 
+                broker=CELERY_BROKER_URL, 
+                backend=CELERY_RESULT_BACKEND,
+                broker_connection_retry=True,
+                broker_connection_retry_on_startup=True,
+                broker_connection_max_retries=10,
+                )
 
 
 @celery.task(name='tasks.add')

docker-compose.development.yml

@@ -17,7 +17,7 @@ services:
     build:
       context: ./celery-queue
       dockerfile: Dockerfile.dev
-    command: celery -A tasks worker -l info -E
+    command: celery -A tasks worker -l info --uid=nobody --gid=nogroup -E
     environment:
       CELERY_BROKER_URL: redis://redis
       CELERY_RESULT_BACKEND: redis://redis
@@ -30,7 +30,7 @@ services:
       dockerfile: Dockerfile.dev
     ports:
      - "5555:5555"
-    command:  ['celery', 'flower', '-A', 'tasks']
+    command:  ['celery', '-A', 'tasks', 'flower']
     environment:
       CELERY_BROKER_URL: redis://redis:6379/0
       CELERY_RESULT_BACKEND: redis://redis:6379/0

docker-compose.yml

@@ -14,7 +14,7 @@ services:
     build:
       context: ./celery-queue
       dockerfile: Dockerfile
-    command: celery -A tasks worker -l info -E
+    command: celery -A tasks worker -l info --uid=nobody --gid=nogroup -E
     environment:
       CELERY_BROKER_URL: redis://redis
       CELERY_RESULT_BACKEND: redis://redis
@@ -27,7 +27,7 @@ services:
       dockerfile: Dockerfile
     ports:
      - "5555:5555"
-    command:  ['celery', 'flower', '-A', 'tasks']
+    command:  ['celery', '-A', 'tasks', 'flower']
     environment:
       CELERY_BROKER_URL: redis://redis:6379/0
       CELERY_RESULT_BACKEND: redis://redis:6379/0