Fluentd

EKS_CLUSTER_NAME="<Cluster Name>" REGION_CODE="ap-northeast-2"

apiVersion: v1 kind: Namespace metadata: name: fluentd labels: name: amazon-cloudwatch

kubectl apply -f ns.yaml kubectl create ns skills

eksctl create iamserviceaccount \ --name fluentd \ --region=$REGION_CODE \ --cluster $EKS_CLUSTER_NAME \ --namespace=fluentd \ --attach-policy-arn arn:aws:iam::aws:policy/CloudWatchFullAccess \ --override-existing-serviceaccounts \ --approve

kubectl create configmap cluster-info --from-literal=cluster.name=$EKS_CLUSTER_NAME --from-literal=logs.region=$REGION_CODE -n fluentd

apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: fluentd-role rules: - apiGroups: [""] resources: - namespaces - pods - pods/logs verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: fluentd-role-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: fluentd-role subjects: - kind: ServiceAccount name: fluentd namespace: fluentd --- apiVersion: v1 kind: ConfigMap metadata: name: fluentd-config namespace: fluentd labels: k8s-app: fluentd-cloudwatch data: kubernetes.conf: | kubernetes.conf fluent.conf: | @include containers.conf <match fluent.**> @type null </match> containers.conf: | <source> @type tail path /var/log/containers/*_skills_*.log pos_file /var/log/fluentd-containers.log.pos tag kubernetes.containers read_from_head true <parse> @type none </parse> </source> <filter kubernetes.containers> @type kubernetes_metadata @id filter_kube_metadata_containers </filter> <match kubernetes.containers> @type cloudwatch_logs log_group_name /eks/application/logs log_stream_name containers-logs auto_create_stream true <buffer tag> @type memory flush_interval 30s flush_thread_count 2 chunk_limit_size 2M queue_limit_length 32 retry_forever true retry_max_interval 30 overflow_action block </buffer> </match> --- apiVersion: apps/v1 kind: DaemonSet metadata: name: fluentd-cloudwatch namespace: fluentd spec: selector: matchLabels: k8s-app: fluentd-cloudwatch template: metadata: labels: k8s-app: fluentd-cloudwatch annotations: configHash: 8915de4cf9c3551a8dc74c0137a3e83569d28c71044b0359c2578d2e0461825 spec: serviceAccountName: fluentd terminationGracePeriodSeconds: 30 initContainers: - name: copy-fluentd-config image: busybox command: ["sh", "-c", "cp /config-volume/..data/* /fluentd/etc"] volumeMounts: - name: config-volume mountPath: /config-volume - name: fluentdconf mountPath: /fluentd/etc containers: - name: fluentd-cloudwatch image: fluent/fluentd-kubernetes-daemonset:v1.10.3-debian-cloudwatch-1.0 env: - name: AWS_REGION valueFrom: configMapKeyRef: name: cluster-info key: logs.region - name: CLUSTER_NAME valueFrom: configMapKeyRef: name: cluster-info key: cluster.name - name: CI_VERSION value: "k8s/1.3.24" resources: limits: memory: 400Mi requests: cpu: 100m memory: 200Mi volumeMounts: - name: config-volume mountPath: /config-volume - name: fluentdconf mountPath: /fluentd/etc - name: varlog mountPath: /var/log - name: varlibdockercontainers mountPath: /var/lib/docker/containers readOnly: true - name: runlogjournal mountPath: /run/log/journal readOnly: true - name: dmesg mountPath: /var/log/dmesg readOnly: true volumes: - name: config-volume configMap: name: fluentd-config - name: fluentdconf emptyDir: {} - name: varlog hostPath: path: /var/log - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers - name: runlogjournal hostPath: path: /run/log/journal - name: dmesg hostPath: path: /var/log/dmesg

kubectl apply -f fluentd.yaml

apiVersion: apps/v1 kind: Deployment metadata: name: skills-app-deployment namespace: skills spec: selector: matchLabels: app: skills-app replicas: 2 template: metadata: labels: app: skills-app spec: nodeSelector: eks.amazonaws.com/nodegroup: skills-app-nodegroup containers: - name: skills-app image: 362708816803.dkr.ecr.ap-northeast-2.amazonaws.com/service-a:latest ports: - containerPort: 8080 imagePullPolicy: Always

kubectl apply -f deployment.yaml

kubectl exec -it -n skills deployment.apps/skills-app-deployment -- curl localhost:8080 > /dev/null 2>&1

kubectl logs -n skills deployment.apps/skills-app-deployment