Added kafka integration. (#105)

Signed-off-by: Pavel Kirilin <win10@list.ru>

Author: s3rius

fastapi_template/cli.py

@@ -94,6 +94,13 @@ def parse_args():
         default=None,
         dest="enable_rmq",
     )
+    parser.add_argument(
+        "--kafka",
+        help="Add Kafka support",
+        action="store_true",
+        default=None,
+        dest="enable_kafka",
+    )
     parser.add_argument(
         "--migrations",
         help="Add migrations support",
@@ -184,62 +191,31 @@ def parse_args():
 
 
 def ask_features(current_context: BuilderContext) -> BuilderContext:
+    # List of features and associated cookiecutter values.
     features = {
-        "Redis support": {
-            "name": "enable_redis",
-            "value": current_context.enable_redis,
-        },
-        "Kubernetes config (deprecated)": {
-            "name": "enable_kube",
-            "value": current_context.enable_kube,
-        },
-        "Demo routers": {
-            "name": "enable_routers",
-            "value": current_context.enable_routers,
-        },
-        "Self-hosted swagger": {
-            "name": "self_hosted_swagger",
-            "value": current_context.self_hosted_swagger,
-        },
-        "RabbitMQ integration": {
-            "name": "enable_rmq",
-            "value": current_context.enable_rmq,
-        },
-        "Prometheus integration": {
-            "name": "prometheus_enabled",
-            "value": current_context.prometheus_enabled,
-        },
-        "Sentry integration": {
-            "name": "sentry_enabled",
-            "value": current_context.sentry_enabled,
-        },
-        "Opentelemetry integration": {
-            "name": "otlp_enabled",
-            "value": current_context.otlp_enabled,
-        },
-        "Loguru logger": {
-            "name": "enable_loguru",
-            "value": current_context.enable_loguru,
-        },
-        "Traefik labels for docker": {
-            "name": "traefik_labels",
-            "value": current_context.traefik_labels,
-        },
+        "Redis support": "enable_redis",
+        "Kubernetes config (deprecated)": "enable_kube",
+        "Demo routers": "enable_routers",
+        "Self-hosted swagger": "self_hosted_swagger",
+        "RabbitMQ integration": "enable_rmq",
+        "Kafka integration": "enable_kafka",
+        "Prometheus integration": "prometheus_enabled",
+        "Sentry integration": "sentry_enabled",
+        "Opentelemetry integration": "otlp_enabled",
+        "Loguru logger": "enable_loguru",
+        "Traefik labels for docker": "traefik_labels",
     }
     if current_context.db != DatabaseType.none:
-        features["Migrations support"] = {
-            "name": "enable_migrations",
-            "value": current_context.enable_migrations,
-        }
-        features["Add dummy model"] = {
-            "name": "add_dummy",
-            "value": current_context.add_dummy,
-        }
+        features.update({
+            "Migrations support": "enable_migrations",
+            "Add dummy model": "add_dummy"
+        })
     checkbox_values = []
-    for feature_name, feature in features.items():
-        if feature["value"] is None:
-            setattr(current_context, feature["name"], False)
-            checkbox_values.append((feature["name"], feature_name))
+    for feature_display_name, feature_key in features.items():
+        value = getattr(current_context, feature_key, None)
+        if value is None:
+            setattr(current_context, feature_key, False)
+            checkbox_values.append((feature_key, feature_display_name))
     if checkbox_values and not current_context.quite:
         results = checkboxlist_dialog(
             title="Features",
@@ -248,8 +224,8 @@ def ask_features(current_context: BuilderContext) -> BuilderContext:
         ).run()
         if results is None:
             raise KeyboardInterrupt()
-        for feature in results:
-            setattr(current_context, feature, True)
+        for feature_key in results:
+            setattr(current_context, feature_key, True)
     return current_context
 
 

fastapi_template/input_model.py

@@ -123,6 +123,7 @@ class BuilderContext(BaseModel):
     enable_rmq: Optional[bool]
     enable_loguru: Optional[bool]
     traefik_labels: Optional[bool]
+    enable_kafka: Optional[bool]
     force: bool = False
     quite: bool = False
 

fastapi_template/template/cookiecutter.json

@@ -32,6 +32,9 @@
     "enable_routers": {
         "type": "bool"
     },
+    "enable_kafka": {
+        "type": "bool"
+    },
     "enable_loguru": {
         "type": "bool"
     },

fastapi_template/template/{{cookiecutter.project_name}}/.github/workflows/tests.yml

@@ -48,9 +48,11 @@ jobs:
   pytest:
     runs-on: ubuntu-latest
     {%- if ((cookiecutter.db_info.name != "none" and cookiecutter.db_info.name != "sqlite") or
-          (cookiecutter.enable_rmq == "True")) %}
+          (cookiecutter.enable_rmq == "True") or
+          (cookiecutter.enable_kafka == "True")) %}
     services:
       {%- if cookiecutter.db_info.name != "none" and cookiecutter.db_info.name != "sqlite" %}
+
       {{cookiecutter.project_name}}-db:
         image: {{ cookiecutter.db_info.image }}
         env:
@@ -66,24 +68,66 @@ jobs:
           ALLOW_EMPTY_PASSWORD: yes
           {%- endif %}
         {%- if cookiecutter.db_info.name == "mysql" %}
-        options: --health-cmd="mysqladmin ping -u root" --health-interval=15s --health-timeout=5s --health-retries=5
+        options: >-
+          --health-cmd="mysqladmin ping -u root"
+          --health-interval=15s
+          --health-timeout=5s
+          --health-retries=5
         {%- endif %}
         {%- if cookiecutter.db_info.name == "postgresql" %}
-        options: --health-cmd="pg_isready" --health-interval=10s --health-timeout=5s --health-retries=5
+        options: >-
+          --health-cmd="pg_isready"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
         {%- endif %}
         ports:
-          - {{ cookiecutter.db_info.port }}
+          - {{ cookiecutter.db_info.port }}:{{ cookiecutter.db_info.port }}
         {%- endif %}
       {%- if cookiecutter.enable_rmq == "True" %}
+
       {{cookiecutter.project_name}}-rmq:
         image: rabbitmq:3.9.16-alpine
         env:
           RABBITMQ_DEFAULT_USER: "guest"
           RABBITMQ_DEFAULT_PASS: "guest"
           RABBITMQ_DEFAULT_VHOST: "/"
-        options: --health-cmd="rabbitmq-diagnostics check_running -q" --health-interval=10s --health-timeout=5s --health-retries=8
+        options: >-
+          --health-cmd="rabbitmq-diagnostics check_running -q"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=8
+        ports:
+          - 5672:5672
+      {%- endif %}
+      {%- if cookiecutter.enable_kafka == "True" %}
+
+      {{cookiecutter.project_name}}-zookeeper:
+        image: "bitnami/zookeeper:3.7.1"
+        env:
+          ALLOW_ANONYMOUS_LOGIN: "yes"
+          ZOO_LOG_LEVEL: "ERROR"
+        options: >-
+          --health-cmd="zkServer.sh status"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=8
+
+      {{cookiecutter.project_name}}-kafka:
+        image: bitnami/kafka:3.2.0
+        env:
+          KAFKA_BROKER_ID: "1"
+          ALLOW_PLAINTEXT_LISTENER: "yes"
+          KAFKA_CFG_LISTENERS: "PLAINTEXT://0.0.0.0:9092"
+          KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://localhost:9092"
+          KAFKA_CFG_ZOOKEEPER_CONNECT: "{{cookiecutter.project_name}}-zookeeper:2181"
+        options: >-
+          --health-cmd="kafka-topics.sh --list --bootstrap-server localhost:9092"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=8
         ports:
-          - 5672
+          - 9092:9092
       {%- endif %}
     {%- endif %}
     steps:
@@ -103,10 +147,12 @@ jobs:
         {%- if cookiecutter.db_info.name != "none" %}
         {%- if cookiecutter.db_info.name != "sqlite" %}
           {{ cookiecutter.project_name | upper }}_DB_HOST: localhost
-          {{ cookiecutter.project_name | upper }}_DB_PORT: {{'${{'}}job.services.{{cookiecutter.project_name}}-db.ports['{{cookiecutter.db_info.port}}']{{'}}'}}
         {%- endif %}
         {%- endif %}
         {%- if cookiecutter.enable_rmq == "True" %}
           {{ cookiecutter.project_name | upper }}_RABBIT_HOST: localhost
-          {{ cookiecutter.project_name | upper }}_RABBIT_PORT: {{'${{'}}job.services.{{cookiecutter.project_name}}-rmq.ports['5672']{{'}}'}}
         {%- endif %}
+        {%- if cookiecutter.enable_kafka == "True" %}
+          {{ cookiecutter.project_name | upper }}_KAFKA_BOOTSTRAP_SERVERS: '["localhost:9092"]'
+        {%- endif %}
+

fastapi_template/template/{{cookiecutter.project_name}}/.gitlab-ci.yml

@@ -36,7 +36,8 @@ pytest:
   extends:
     - .test-template
   {%- if ((cookiecutter.db_info.name != "none" and cookiecutter.db_info.name != "sqlite") or
-          (cookiecutter.enable_rmq == "True")) %}
+          (cookiecutter.enable_rmq == "True") or
+          (cookiecutter.enable_kafka == "True")) %}
   services:
     {%- if cookiecutter.db_info.name != "none" and cookiecutter.db_info.name != "sqlite" %}
     - name: {{ cookiecutter.db_info.image }}
@@ -46,26 +47,49 @@ pytest:
     - name: rabbitmq:3.9.16-alpine
       alias: rmq
     {%- endif %}
+    {%- if cookiecutter.enable_kafka == "True" %}
+    - name: bitnami/kafka:3.2.0
+      alias: kafka
+    {%- endif %}
   variables:
     {%- if cookiecutter.db_info.name == "postgresql" %}
+
+    # Postgresql variables
     {{ cookiecutter.project_name | upper }}_DB_HOST: database
     POSTGRES_PASSWORD: {{ cookiecutter.project_name }}
     POSTGRES_USER: {{ cookiecutter.project_name }}
     POSTGRES_DB: {{ cookiecutter.project_name }}
     {%- endif %}
     {%- if cookiecutter.db_info.name == "mysql" %}
+
+    # MySQL variables
     {{ cookiecutter.project_name | upper }}_DB_HOST: database
     MYSQL_PASSWORD: {{ cookiecutter.project_name }}
     MYSQL_USER: {{ cookiecutter.project_name }}
     MYSQL_DATABASE: {{ cookiecutter.project_name }}
     ALLOW_EMPTY_PASSWORD: yes
     {%- endif %}
     {%- if cookiecutter.enable_rmq == "True" %}
+
+    # Rabbitmq variables
     RABBITMQ_DEFAULT_USER: "guest"
     RABBITMQ_DEFAULT_PASS: "guest"
     RABBITMQ_DEFAULT_VHOST: "/"
     {{ cookiecutter.project_name | upper }}_RABBIT_HOST: rmq
     {%- endif %}
+    {%- if cookiecutter.enable_kafka == "True" %}
+
+    # Kafka variables
+    KAFKA_BROKER_ID: "1"
+    KAFKA_ENABLE_KRAFT: "yes"
+    ALLOW_PLAINTEXT_LISTENER: "yes"
+    KAFKA_CFG_PROCESS_ROLES: "broker,controller"
+    KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
+    KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
+    KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093"
+    KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@127.0.0.1:9093"
+    {{ cookiecutter.project_name | upper }}_KAFKA_BOOTSTRAP_SERVERS: '["kafka:9092"]'
+    {%- endif %}
   {%- endif %}
   script:
     {%- if cookiecutter.db_info.name != "none" %}

fastapi_template/template/{{cookiecutter.project_name}}/conditional_files.json

@@ -11,7 +11,8 @@
             "{{cookiecutter.project_name}}/web/api/rabbit",
             "{{cookiecutter.project_name}}/web/api/dummy",
             "{{cookiecutter.project_name}}/web/api/echo",
-            "{{cookiecutter.project_name}}/web/api/redis"
+            "{{cookiecutter.project_name}}/web/api/redis",
+            "{{cookiecutter.project_name}}/web/api/kafka"
         ]
     },
     "Redis": {
@@ -34,6 +35,15 @@
             "deploy/kube/redis.yml"
         ]
     },
+    "Kafka support": {
+        "enabled": "{{cookiecutter.enable_kafka}}",
+        "resources": [
+            "{{cookiecutter.project_name}}/web/api/kafka",
+            "{{cookiecutter.project_name}}/web/gql/kafka",
+            "{{cookiecutter.project_name}}/services/kafka",
+            "{{cookiecutter.project_name}}/tests/test_kafka.py"
+        ]
+    },
     "Kubernetes": {
         "enabled": "{{cookiecutter.enable_kube}}",
         "resources": [
@@ -102,6 +112,8 @@
             "{{cookiecutter.project_name}}/web/gql/dummy",
             "{{cookiecutter.project_name}}/web/api/redis",
             "{{cookiecutter.project_name}}/web/gql/redis",
+            "{{cookiecutter.project_name}}/web/api/kafka",
+            "{{cookiecutter.project_name}}/web/gql/kafka",
             "{{cookiecutter.project_name}}/tests/test_echo.py",
             "{{cookiecutter.project_name}}/tests/test_dummy.py",
             "{{cookiecutter.project_name}}/tests/test_redis.py"