EFK

export EKS_CLUSTER_NAME=<CLUSTER_NAME> export ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) export AWS_REGION=ap-northeast-2 export ES_DOMAIN_NAME="skills-logging" # Elasticsearch domain name export ES_VERSION="7.10" # Elasticsearch version export ES_DOMAIN_USER="admin" # kibana admin user export ES_DOMAIN_PASSWORD="$(openssl rand -base64 12)_Ek1$"

eksctl utils associate-iam-oidc-provider \ --cluster $EKS_CLUSTER_NAME \ --approve

cat << EOF > fluent-bit-policy.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "es:ESHttp*" ], "Resource": "arn:aws:es:${AWS_REGION}:${ACCOUNT_ID}:domain/${ES_DOMAIN_NAME}" } ] } EOF

aws iam create-policy \ --policy-name fluent-bit-policy \ --policy-document file://fluent-bit-policy.json

kubectl create namespace logging

eksctl create iamserviceaccount \ --name fluent-bit \ --namespace logging \ --cluster $EKS_CLUSTER_NAME \ --attach-policy-arn "arn:aws:iam::${ACCOUNT_ID}:policy/fluent-bit-policy" \ --override-existing-serviceaccounts \ --approve

kubectl -n logging describe serviceaccounts fluent-bit

cat << EOF > es_domain.json { "DomainName": "${ES_DOMAIN_NAME}", "ElasticsearchVersion": "${ES_VERSION}", "ElasticsearchClusterConfig": { "InstanceType": "r5.large.elasticsearch", "InstanceCount": 1, "DedicatedMasterEnabled": false, "ZoneAwarenessEnabled": false, "WarmEnabled": false }, "EBSOptions": { "EBSEnabled": true, "VolumeType": "gp2", "VolumeSize": 100 }, "AccessPolicies": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"*\"},\"Action\":\"es:ESHttp*\",\"Resource\":\"arn:aws:es:${AWS_REGION}:${ACCOUNT_ID}:domain/${ES_DOMAIN_NAME}/*\"}]}", "SnapshotOptions": {}, "CognitoOptions": { "Enabled": false }, "EncryptionAtRestOptions": { "Enabled": true }, "NodeToNodeEncryptionOptions": { "Enabled": true }, "DomainEndpointOptions": { "EnforceHTTPS": true, "TLSSecurityPolicy": "Policy-Min-TLS-1-0-2019-07" }, "AdvancedSecurityOptions": { "Enabled": true, "InternalUserDatabaseEnabled": true, "MasterUserOptions": { "MasterUserName": "${ES_DOMAIN_USER}", "MasterUserPassword": "${ES_DOMAIN_PASSWORD}" } } } EOF

aws es create-elasticsearch-domain \ --cli-input-json file://es_domain.json

export FLUENTBIT_ROLE=$(eksctl get iamserviceaccount --cluster $EKS_CLUSTER_NAME --namespace logging -o json | jq '.[].status.roleARN' -r) export ES_ENDPOINT=$(aws es describe-elasticsearch-domain --domain-name ${ES_DOMAIN_NAME} --output text --query "DomainStatus.Endpoint")

curl -sS -u "${ES_DOMAIN_USER}:${ES_DOMAIN_PASSWORD}" \ -X PATCH \ https://${ES_ENDPOINT}/_opendistro/_security/api/rolesmapping/all_access?pretty \ -H 'Content-Type: application/json' \ -d' [ { "op": "add", "path": "/backend_roles", "value": ["'${FLUENTBIT_ROLE}'"] } ] '

--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: fluent-bit-read rules: - apiGroups: [""] resources: - namespaces - pods verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: fluent-bit-read roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: fluent-bit-read subjects: - kind: ServiceAccount name: fluent-bit namespace: logging --- apiVersion: v1 kind: ConfigMap metadata: name: fluent-bit-config namespace: logging labels: k8s-app: fluent-bit data: # Configuration files: server, input, filters and output # ====================================================== fluent-bit.conf: | [SERVICE] Flush 1 Log_Level info Daemon off Parsers_File parsers.conf HTTP_Server On HTTP_Listen 0.0.0.0 HTTP_Port 2020 @INCLUDE input-kubernetes.conf @INCLUDE filter-kubernetes.conf @INCLUDE output-elasticsearch.conf input-kubernetes.conf: | [INPUT] Name tail Tag kube.* Path /var/log/containers/*.log Parser docker DB /var/log/flb_kube.db Mem_Buf_Limit 50MB Skip_Long_Lines On Refresh_Interval 10 filter-kubernetes.conf: | [FILTER] Name kubernetes Match kube.* Kube_URL https://kubernetes.default.svc:443 Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token Kube_Tag_Prefix kube.var.log.containers. Merge_Log On Merge_Log_Key log_processed K8S-Logging.Parser On K8S-Logging.Exclude Off output-elasticsearch.conf: | [OUTPUT] Name es Match * Host ${ES_ENDPOINT} Port 443 TLS On AWS_Auth On AWS_Region ap-northeast-2 Retry_Limit 6 parsers.conf: | [PARSER] Name apache Format regex Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ Time_Key time Time_Format %d/%b/%Y:%H:%M:%S %z [PARSER] Name apache2 Format regex Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ Time_Key time Time_Format %d/%b/%Y:%H:%M:%S %z [PARSER] Name apache_error Format regex Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$ [PARSER] Name nginx Format regex Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$ Time_Key time Time_Format %d/%b/%Y:%H:%M:%S %z [PARSER] Name json Format json Time_Key time Time_Format %d/%b/%Y:%H:%M:%S %z [PARSER] Name docker Format json Time_Key time Time_Format %Y-%m-%dT%H:%M:%S.%L Time_Keep On [PARSER] Name syslog Format regex Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$ Time_Key time Time_Format %b %d %H:%M:%S --- apiVersion: apps/v1 kind: DaemonSet metadata: name: fluent-bit namespace: logging labels: k8s-app: fluent-bit-logging version: v1 kubernetes.io/cluster-service: "true" spec: selector: matchLabels: k8s-app: fluent-bit-logging template: metadata: labels: k8s-app: fluent-bit-logging version: v1 kubernetes.io/cluster-service: "true" annotations: prometheus.io/scrape: "true" prometheus.io/port: "2020" prometheus.io/path: /api/v1/metrics/prometheus spec: containers: - name: fluent-bit image: amazon/aws-for-fluent-bit:latest imagePullPolicy: Always ports: - containerPort: 2020 volumeMounts: - name: varlog mountPath: /var/log - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true - name: fluent-bit-config mountPath: /fluent-bit/etc/ terminationGracePeriodSeconds: 10 volumes: - name: varlog hostPath: path: /var/log - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - name: fluent-bit-config configMap: name: fluent-bit-config serviceAccountName: fluent-bit

kubectl apply -f fluentbit.yaml

kubectl -n logging get pods -o wide

echo "Kibana URL: https://${ES_ENDPOINT}/_plugin/kibana/ Kibana user: ${ES_DOMAIN_USER} Kibana password: ${ES_DOMAIN_PASSWORD}"