demos/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml at 6e0f2e698232a543c0458ec057b6d4dd4b8fc75e · stackabletech/demos · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
---
apiVersion: airflow.stackable.tech/v1alpha1
kind: AirflowCluster
metadata:
  name: airflow
  labels:
    stackable.tech/vendor: Stackable
    stackable.tech/demo: {{ DEMO }}
spec:
  image:
    productVersion: 3.1.6
    pullPolicy: IfNotPresent
  clusterConfig:
    loadExamples: false
    exposeConfig: false
    credentialsSecret: airflow-credentials
    # NOTE: This cannot be parameterized via stackablectl since it is synced via ArgoCD.
    #
    # In terms of forking the repository, this should not be a problem since the DAGs are the same.
    # You can still change this locally and point to the fork in case of any changes to DAGs.
    #
    # TODO(@maltesander): In a Stackable release, this branch should point to the release-YY.MM branch!
    dagsGitSync:
      - repo: https://github.com/stackabletech/demos/
        branch: main
        gitFolder: "demos/argo-cd-git-ops/dags"
        depth: 2
    volumes:
      - name: minio-tls
        ephemeral:
          volumeClaimTemplate:
            metadata:
              annotations:
                secrets.stackable.tech/class: tls
                secrets.stackable.tech/scope: pod
            spec:
              accessModes:
                - ReadWriteOnce
              resources:
                requests:
                  storage: "1"
              storageClassName: secrets.stackable.tech
    volumeMounts:
      - name: minio-tls
        mountPath: /stackable/minio-tls
  webservers:
    roleConfig:
      listenerClass: external-stable
    envOverrides: &envOverrides
      AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D"
      # Via sealed secrets and pod overrides, just kept for reference here
      # AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000"
      AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt"
      AIRFLOW__LOGGING__REMOTE_LOGGING: "True"
      AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/
      AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio
      # 20 seconds to reload the DAGS folder (default: 300)
      AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "20"
    podOverrides: &podOverrides
      spec:
        containers:
          - name: airflow
            env:
              - name: AIRFLOW_CONN_MINIO
                valueFrom:
                  secretKeyRef:
                    name: airflow-minio-connection
                    key: airflow-minio-connection
          - name: git-sync-0
            resources:
              limits:
                memory: "128Mi"
              requests:
                memory: "128Mi"
    config:
      gracefulShutdownTimeout: 30s
      resources:
        cpu:
          min: 400m
          max: "1"
        memory:
          limit: 2Gi
    roleGroups:
      default:
        # NOTE: In order to properly show ArgoCD in action - syncing changes via Git - do the following:
        # - Fork this repository
        # - Use the demo parameters `customGitUrl` and `customGitBranch` to point the
        #   the repository to your fork and branch
        # - Change the replicas here and push that to the provided `customGitBranch` in your fork
        replicas: 1
  kubernetesExecutors:
    envOverrides: *envOverrides
    podOverrides:
      spec:
        containers:
          - name: base
            env:
              - name: AIRFLOW_CONN_MINIO
                valueFrom:
                  secretKeyRef:
                    name: airflow-minio-connection
                    key: airflow-minio-connection
        initContainers:
          - name: git-sync-0-init
            resources:
              limits:
                memory: "128Mi"
              requests:
                memory: "128Mi"
  schedulers:
    envOverrides: *envOverrides
    podOverrides: *podOverrides
    config:
      gracefulShutdownTimeout: 30s
      resources:
        cpu:
          min: 400m
          max: "1"
        memory:
          limit: 1Gi
    roleGroups:
      default:
        replicas: 1