Airflow/airflow-custom.yaml at main · AllStackProject/Airflow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
executor: "KubernetesExecutor"

env:
  - name: AIRFLOW__API__AUTH_BACKENDS
    value: 'airflow.api.auth.backend.basic_auth'

dags:
  gitSync:
    enabled: true
    repo: https://github.com/AllStackProject/Airflow.git
    branch: main
    subPath: ""
    credentialsSecret: airflow-git-secret
  persistence:
    enabled: false

scheduler:
  startupProbe:
    initialDelaySeconds: 300
    failureThreshold: 20
    periodSeconds: 10
    timeoutSeconds: 30
  livenessProbe:
    initialDelaySeconds: 10
    timeoutSeconds: 120
    failureThreshold: 20
    periodSeconds: 60

dagProcessor:
  enabled: true
  livenessProbe:
    initialDelaySeconds: 60
    timeoutSeconds: 20
    periodSeconds: 60
    failureThreshold: 10

logs:
  persistence:
    enabled: true
    existingClaim: pvc-hdd-airflow-logs

triggerer:
  enabled: false

statsd:
  enabled: true

redis:
  enabled: false

postgresql:
  enabled: false

pgbouncer:
  enabled: true
  metadataPoolSize: 40
  resultBackendPoolSize: 20

data:
  metadataConnection:
    user: airflow
    pass: airflow
    host: 172.16.2.110
    protocol: postgresql
    port: 5040
    db: airflow

apiServer:
  service:
    type: NodePort
    ports:
      - name: airflow-web
        port: 8080
        nodePort: 30180
        protocol: TCP
  startupProbe:
    initialDelaySeconds: 600
    timeoutSeconds: 60
    periodSeconds: 10
    failureThreshold: 20

resources:
  limits:
    cpu: 2Gi
    memory: 2Gi

secret:
  - envName: AWS_ACCESS_KEY_ID
    secretName: airflow-aws
    secretKey: AWS_ACCESS_KEY_ID

  - envName: AWS_SECRET_ACCESS_KEY
    secretName: airflow-aws
    secretKey: AWS_SECRET_ACCESS_KEY

  - envName: AWS_DEFAULT_REGION
    secretName: airflow-aws
    secretKey: AWS_DEFAULT_REGION