MWAA

export EKS_CLUSTER_NAME=<CLUSTER_NAME> export REGION_CODE=ap-northeast-2 export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) export MWAA_ENV_NAME=<MWAA_ENV_NAME> export MWAA_S3_BUCKET=<MWAA_BUCKET_NAME>

aws s3 mb s3://$MWAA_S3_BUCKET --region $REGION_CODE

eksctl utils associate-iam-oidc-provider \ --region $REGION_CODE \ --cluster $EKS_CLUSTER_NAME \ --approve

kubectl create namespace mwaa

cat << EOF | kubectl apply -f - -n mwaa kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: mwaa-role rules: - apiGroups: - "" - "apps" - "batch" - "extensions" resources: - "jobs" - "pods" - "pods/attach" - "pods/exec" - "pods/log" - "pods/portforward" - "secrets" - "services" verbs: - "create" - "delete" - "describe" - "get" - "list" - "patch" - "update" --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: mwaa-role-binding subjects: - kind: User name: mwaa-service roleRef: kind: Role name: mwaa-role apiGroup: rbac.authorization.k8s.io EOF

kubectl get pods -n mwaa --as mwaa-service

> 결과값 No resources found in mwaa namespace.

cat << EOF > mwaa-policy.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": "airflow:PublishMetrics", "Resource": "arn:aws:airflow:${REGION_CODE}:${AWS_ACCOUNT_ID}:environment/${MWAA_ENV_NAME}" }, { "Effect": "Deny", "Action": "s3:ListAllMyBuckets", "Resource": [ "arn:aws:s3:::$MWAA_S3_BUCKET", "arn:aws:s3:::$MWAA_S3_BUCKET/*" ] }, { "Effect": "Allow", "Action": ["s3:GetObject*", "s3:GetBucket*", "s3:List*"], "Resource": [ "arn:aws:s3:::$MWAA_S3_BUCKET", "arn:aws:s3:::$MWAA_S3_BUCKET/*" ] }, { "Effect": "Allow", "Action": [ "logs:CreateLogStream", "logs:CreateLogGroup", "logs:PutLogEvents", "logs:GetLogEvents", "logs:GetLogRecord", "logs:GetLogGroupFields", "logs:GetQueryResults" ], "Resource": [ "arn:aws:logs:${REGION_CODE}:${AWS_ACCOUNT_ID}:log-group:airflow-${MWAA_ENV_NAME}-*" ] }, { "Effect": "Allow", "Action": ["logs:DescribeLogGroups"], "Resource": ["*"] }, { "Effect": "Allow", "Action": "cloudwatch:PutMetricData", "Resource": "*" }, { "Effect": "Allow", "Action": [ "sqs:ChangeMessageVisibility", "sqs:DeleteMessage", "sqs:GetQueueAttributes", "sqs:GetQueueUrl", "sqs:ReceiveMessage", "sqs:SendMessage" ], "Resource": "arn:aws:sqs:${REGION_CODE}:*:airflow-celery-*" }, { "Effect": "Allow", "Action": [ "kms:Decrypt", "kms:DescribeKey", "kms:GenerateDataKey*", "kms:Encrypt" ], "NotResource": "arn:aws:kms:*:${AWS_ACCOUNT_ID}:key/*", "Condition": { "StringLike": { "kms:ViaService": ["sqs.${REGION_CODE}.amazonaws.com"] } } }, { "Effect": "Allow", "Action": ["eks:DescribeCluster"], "Resource": "arn:aws:eks:${REGION_CODE}:${AWS_ACCOUNT_ID}:cluster/${EKS_CLUSTER_NAME}" } ] } EOF

cat << EOF > mwaa-role-trust-policy.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": [ "airflow.amazonaws.com", "airflow-env.amazonaws.com" ] }, "Action": "sts:AssumeRole" } ] } EOF

aws iam create-policy \ --policy-name mwaa-policy \ --policy-document file://mwaa-policy.json

aws iam create-role \ --role-name mwaa-execution-role \ --assume-role-policy-document file://mwaa-role-trust-policy.json

aws iam attach-role-policy \ --role-name mwaa-execution-role \ --policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/mwaa-policy

eksctl create iamidentitymapping \ --region $REGION_CODE \ --cluster $EKS_CLUSTER_NAME \ --arn arn:aws:iam::$AWS_ACCOUNT_ID:role/mwaa-execution-role \ --username mwaa-service

aws eks update-kubeconfig \ --region $REGION_CODE \ --kubeconfig ./kube_config.yaml \ --name $EKS_CLUSTER_NAME \ --alias aws

. ├─dags │ ├─example-eks.py │ └─kube_config.yaml └─requirements └─requirements.txt

apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJZEVVVzNtbHQ0VU13RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TlRBeE1qY3dOalV3TlRsYUZ3MHpOVEF4TWpVd05qVTFOVGxhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUMrcEpiaEdtbDZlL1dKTTFSOEdZZGN5L1lQU0s1Nnl1OTg1M0JXc0lVWjhKT25qRE5INzZYdzJVWXgKWXk2MXExSUFESGVmK0pxRjNjTm95Rm9nZXBNcW9WSDlxSnZxTU4wbEZhLzhYZE9Kd0dQb2g0NHB5aW1pV1FkdQpWQWJQNlFyUXpvZk9IdFZxVVVvUm1jb1NuQm9kREV5VzZKbDFMTWZCdDh3eEhDV0ZCYUI2TEVWTlFjV1JtREdsCjhERGcwZEdyNTNJZWVndGxWU0tGVmllS2lIb0dvTGhSZWlZM1A4V3REVHpzNnd6ay9BU3ByT3ZTQVgzYTdTS1YKblV5TWV2SGZxMGNiMnFvZGZ4RW43NjBGN2RnRHpobEtuWWl6aG9JU0c4SDgwL3NuNzFMNkRiRERKUFZGSmg5cQppQlJnQlhNUzJzWWVlMDcyRTJtS3ZtME4zczRiQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJRUEllU25jUEp3QW9QeE83K3hNVmdpaFRlWGNqQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQkI3RUdiNm8rbgpPQzRDUWdMN0IrYjNFa1IzSW03aXNxTWluVVY1Syt5c21SaXZFY3NiMnQ3VTZ2YW5ZNEc4QUhsdkcyV3FJTnRXClk1Q0R3NXlIeXZtTnoxenRoejA4dzZYK1ZsdzVuQmNCYnNPa1JwbGhtUG5tbWUwUUd3REVRREV5b1I4a1dmR0EKNWF1MTlOK3dYUlJTeWR1ZmFsRlRCZExkZjJNN05QakszSW8wWnNnY2Z6QmV0Y3BvTnVuTDYxRzRvSExUVXVtVgpKMThHUFNnQVBIYWhNdGhTNllEaEsvd3dQVFEvcjJXNTQ1N24wcjNVZVQ2MGVHVENwQ2JYK2hvOE9ObnF6TVQ1Ck5FSlBKSTJUVlp3dXkxeWVwK3hMeDdtVDJ6djJHbUdCNGFNYkxXeHhZOWYvbjJ3c3JBekhtT2hSbFE2eGV4dDIKZkM1QUVsdEFWNGlHCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://D1C3A827859FF35F029846CB204F07FA.yl4.ap-northeast-2.eks.amazonaws.com name: arn:aws:eks:ap-northeast-2:362708816803:cluster/skills-eks-cluster contexts: - context: cluster: arn:aws:eks:ap-northeast-2:362708816803:cluster/skills-eks-cluster user: arn:aws:eks:ap-northeast-2:362708816803:cluster/skills-eks-cluster name: aws current-context: aws kind: Config preferences: {} users: - name: arn:aws:eks:ap-northeast-2:362708816803:cluster/skills-eks-cluster user: exec: apiVersion: client.authentication.k8s.io/v1beta1 args: - --region - ap-northeast-2 - eks - get-token - --cluster-name - skills-eks-cluster - --output - json command: aws

from airflow import DAG from datetime import datetime from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator default_args = { 'owner': 'aws', 'depends_on_past': False, 'start_date': datetime(2019, 2, 20), 'provide_context': True } dag = DAG( 'kubernetes_pod_example', default_args=default_args, schedule_interval=None, catchup=False ) kube_config_path = '/usr/local/airflow/dags/kube_config.yaml' podRun = KubernetesPodOperator( namespace="mwaa", image="ubuntu:18.04", cmds=["bash"], arguments=["-c", "ls"], labels={"foo": "bar"}, name="mwaa-pod-test", task_id="pod-task", get_logs=True, dag=dag, is_delete_operator_pod=False, config_file=kube_config_path, in_cluster=False, cluster_context='aws' )

apache-airflow-providers-cncf-kubernetes kubernetes

kubectl get pods -n mwaa kubectl logs -n mwaa <pod-name>