django - 如何让 celerybeat cron 任务在 django 应用程序的 docker 容器中运行？

Question

我正在尝试在我的 django/nuxt 应用程序中运行 celery beat 任务 我有单独的前端和后端目录，我正在使用 docker-compose 来构建和运行我的应用程序。 (pycharm professional-mac-oS 系列

我可以在不在本地使用 docker 容器的情况下完美地运行我的任务，但是当我尝试容器化运行时，celery 和 celery-beat 都不会继续运行。我发现在线文档非常差，甚至在 celery 或 dockers 官方文档中也没有提到运行 celery & beat with docker 谁能告诉我我的配置是否错误或我需要做什么才能让我的应用程序接受我的 settings.py 中的 cronjobs？

我使用 docker-compose up-d 构建我的容器 并使用 docker-compose exec django bash

运行我的应用

谁能指出我正确的方向？

设置.py

    import os
    from configurations import Configuration, values
    from datetime import timedelta
    #print(os.environ)


    class Base(Configuration):

        DEBUG = values.BooleanValue(False)
        SECRET_KEY = '2pj=b#ywty7ojkv_gd#!$!vzywakop1azlxiqxrl^r50i(nf-^'
        BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
        DATABASES = values.DatabaseURLValue()
        ALLOWED_HOSTS = []

        INSTALLED_APPS = [
            "django.contrib.admin",
            "django.contrib.auth",
            "django.contrib.contenttypes",
            "django.contrib.sessions",
            "django.contrib.messages",
            "django.contrib.staticfiles",
            "rest_framework",
            "rest_framework.authtoken",
            "django_celery_results" ,
            "django_celery_beat" ,
            "corsheaders",
            "djoser",
            "accounts",
            "posts",
            "comments",
            "events",
        ]

        MIDDLEWARE = [
            "django.middleware.security.SecurityMiddleware",
            "django.contrib.sessions.middleware.SessionMiddleware",
            "corsheaders.middleware.CorsMiddleware",
            "django.middleware.common.CommonMiddleware",
            "django.middleware.csrf.CsrfViewMiddleware",
            "django.contrib.auth.middleware.AuthenticationMiddleware",
            "django.contrib.messages.middleware.MessageMiddleware",
            "django.middleware.clickjacking.XFrameOptionsMiddleware",
        ]

        ROOT_URLCONF = "spacenews.urls"
        TEMPLATES = [
            {
                "BACKEND": "django.template.backends.django.DjangoTemplates",
                "DIRS": [],
                "APP_DIRS": True,
                "OPTIONS": {
                    "context_processors": [
                        "django.template.context_processors.debug",
                        "django.template.context_processors.request",
                        "django.contrib.auth.context_processors.auth",
                        "django.contrib.messages.context_processors.messages",
                    ]
                },
            }
        ]

        # WSGI
        WSGI_APPLICATION = "spacenews.wsgi.application"
        # Password validators
        AUTH_PASSWORD_VALIDATORS = [
            {
                "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"  # noqa
            },
            {
                "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"
            },
            {
                "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"
            },
            {
                "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"
            },
        ]
        AUTH_USER_MODEL = "accounts.User"

        # Internationalization
        LANGUAGE_CODE = "en-us"
        TIME_ZONE = "UTC"
        USE_I18N = True
        USE_L10N = True
        USE_TZ = True

        # Static files

        STATIC_URL = "/static/"
        # REST
        REST_FRAMEWORK = {
            "DEFAULT_AUTHENTICATION_CLASSES": (
                "rest_framework.authentication.BasicAuthentication",
                "rest_framework.authentication.SessionAuthentication",
                "rest_framework.authentication.TokenAuthentication",
            ),
            "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
            "PAGE_SIZE": 100,
        }


        CELERY_BROKER_URL = 'redis://localhost:6379/'
        CELERY_RESULT_BACKEND = 'redis://localhost:6379/'
        #CELERYD_HIJACK_ROOT_LOGGER = False
        # use json format for everything
        CELERY_ACCEPT_CONTENT = ['application/json']
        CELERY_TASK_SERIALIZER = 'json'
        CELERY_RESULT_SERIALIZER = 'json'
        CELERY_TIMEZONE = 'UTC'


        CELERY_BEAT_SCHEDULE = {


            'login_mb': {
                'task': 'events.tasks.login_mb',
                'schedule': timedelta(seconds=10),

            } ,
            'mb_get_events': {
                'task': 'events.tasks.mb_get_events' ,
                'schedule': timedelta(seconds=10) ,

            } ,
        }



    class Development(Base):
        DEBUG = values.BooleanValue(True)
        CORS_ORIGIN_ALLOW_ALL = True
        ALLOWED_HOSTS = ["localhost", "django","postgres","redis"]


    class Production(Base):
        pass


    class Testing(Base):
        pass

docker 文件

    FROM python:3.6
    ENV PYTHONUNBUFFERED 1
    #ENV C_FORCE_ROOT true
    RUN apt-get update && apt-get install -y postgresql-client
    ADD . /app
    WORKDIR /app
    COPY requirements.txt /app/requirements.txt
    RUN pip install -r requirements.txt
    EXPOSE 8000

docker-compose.yaml

  version: '2'

  services:

  db:
    restart: always
    image: postgres

  redis:
    restart: always
    image: redis

  api:
    build:
      context: ./backend
    environment:
       - DATABASE_URL=postgres://postgres@db:5432/postgres
       - CELERY_BROKER_URL=redis://localhost:6379/
       - CELERY_RESULT_BACKEND =redis://localhost:6379/
       - DJANGO_SECRET_KEY=seekret
    volumes:
       - ./backend:/app

  celery:
    extends:
      service: api
    command:
      bash -c "cd spacenews && celery -A spacenews worker -B --loglevel=info"
    depends_on:
      - db
      - redis

  django:
    extends:
      service: api
    command: ./wait-for-it.sh db:5432 -- ./spacenews/manage.py runserver 0.0.0.0:8000
    ports:
      - "8000:8000"
    volumes:
      - ./backend:/app
    depends_on:
      - db
      - redis
      - celery
      - celery-beat

  celery-beat:
    extends:
      service: api

    command:
        bash -c "cd backend/spacenews && celery -A spacenews beat -B --loglevel=info"
    volumes:
      - ./backend:/app
    depends_on:
      - db
      - redis
      - celery
  nuxt:
    build:
      context: ./frontend
    environment:
      - API_URI=http://django:8000/api
    command: bash -c "npm install && npm run dev"
    volumes:
      - ./frontend:/app
    ports:
      - "3000:3000"
    depends_on:
      - django
      - redis

  volumes:
  pgdata:
  redisdata:

堆栈跟踪 celery 容器

    Traceback (most recent call last):
      File "/usr/local/lib/python3.6/site-packages/kombu/utils/objects.py", line 42, in __get__
        return obj.__dict__[self.__name__]
    KeyError: 'data'

    During handling of the above exception, another exception occurred:

    Traceback (most recent call last):
      File "/usr/local/bin/celery", line 11, in <module>
        sys.exit(main())
      File "/usr/local/lib/python3.6/site-packages/celery/__main__.py", line 16, in main
        _main()
      File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 322, in main
        cmd.execute_from_commandline(argv)
      File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 496, in execute_from_commandline
        super(CeleryCommand, self).execute_from_commandline(argv)))
      File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 275, in execute_from_commandline
        return self.handle_argv(self.prog_name, argv[1:])
      File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 488, in handle_argv
        return self.execute(command, argv)
      File "/usr/local/lib/python3.6/site-packages/celery/bin/celery.py", line 420, in execute
        ).run_from_argv(self.prog_name, argv[1:], command=argv[0])
      File "/usr/local/lib/python3.6/site-packages/celery/bin/worker.py", line 221, in run_from_argv
        *self.parse_options(prog_name, argv, command))
      File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 398, in parse_options
        self.parser = self.create_parser(prog_name, command)
      File "/usr/local/lib/python3.6/site-packages/celery/bin/base.py", line 414, in create_parser
        self.add_arguments(parser)
      File "/usr/local/lib/python3.6/site-packages/celery/bin/worker.py", line 277, in add_arguments
        default=conf.worker_state_db,
      File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 126, in __getattr__
        return self[k]
      File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 429, in __getitem__
        return getitem(k)
      File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 278, in __getitem__
        return mapping[_key]
      File "/usr/local/lib/python3.6/collections/__init__.py", line 987, in __getitem__
        if key in self.data:
      File "/usr/local/lib/python3.6/site-packages/kombu/utils/objects.py", line 44, in __get__
        value = obj.__dict__[self.__name__] = self.__get(obj)
      File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 141, in data
        return self.callback()
      File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 924, in _finalize_pending_conf
        conf = self._conf = self._load_config()
      File "/usr/local/lib/python3.6/site-packages/celery/app/base.py", line 934, in _load_config
        self.loader.config_from_object(self._config_source)
      File "/usr/local/lib/python3.6/site-packages/celery/loaders/base.py", line 131, in config_from_object
        self._conf = force_mapping(obj)
      File "/usr/local/lib/python3.6/site-packages/celery/utils/collections.py", line 46, in force_mapping
        if isinstance(m, (LazyObject, LazySettings)):
      File "/usr/local/lib/python3.6/site-packages/django/utils/functional.py", line 215, in inner
        self._setup()
      File "/usr/local/lib/python3.6/site-packages/django/conf/__init__.py", line 43, in _setup
        self._wrapped = Settings(settings_module)
      File "/usr/local/lib/python3.6/site-packages/django/conf/__init__.py", line 106, in __init__
        mod = importlib.import_module(self.SETTINGS_MODULE)
      File "/usr/local/lib/python3.6/importlib/__init__.py", line 126, in import_module
        return _bootstrap._gcd_import(name[level:], package, level)
      File "<frozen importlib._bootstrap>", line 994, in _gcd_import
      File "<frozen importlib._bootstrap>", line 971, in _find_and_load
      File "<frozen importlib._bootstrap>", line 955, in _find_and_load_unlocked
      File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
      File "<frozen importlib._bootstrap_external>", line 678, in exec_module
      File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
      File "/app/spacenews/spacenews/settings.py", line 7, in <module>
        class Base(Configuration):

celery .py

from __future__ import absolute_import
import os
from celery import Celery

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'spacenews.settings')



app = Celery('spacenews')

# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()


@app.task(bind=True)
def debug_task(self):
    print('Request: {0!r}'.format(self.request))

Answer 1

您的 settings.py 文件看起来对 celery 无效。通常设置文件如下所示:

DEBUG = True

<b>BROKER_URL = 'redis://localhost:6379/'</b>
CELERY_RESULT_BACKEND = 'redis://localhost:6379/'
#CELERYD_HIJACK_ROOT_LOGGER = False
# use json format for everything
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'UTC'


CELERY_BEAT_SCHEDULE = {


    'login_mb': {
        'task': 'events.tasks.login_mb',
        'schedule': timedelta(seconds=10),

    } ,
    'mb_get_events': {
        'task': 'events.tasks.mb_get_events' ,
        'schedule': timedelta(seconds=10) ,

    } ,
}

不使用 Base、Production 或 Dev 等对象。如果你想像你一样使用对象，你必须做额外的工作来将它们公开为 django 设置对象(用于配置 celery)的设置。

或者，您可以向 config_from_object 提供包含 celery 配置的类实例。

最后，请注意您的 broker_url 设置应该是 BROKER_URL 而不是 CELERY_BROKER_URL。