File Cache CSI Driver

export CLUSTER_NAME=skills-eks-cluster export REGION_CODE=ap-northeast-2 export NODE_GROUP_NAME=skills-app-nodegroup

eksctl create iamserviceaccount \ --name file-cache-csi-controller-sa \ --namespace kube-system \ --cluster $CLUSTER_NAME \ --attach-policy-arn arn:aws:iam::aws:policy/AmazonFSxFullAccess \ --approve \ --role-name AmazonEKSFileCacheCSIDriverFullAccess \ --region $REGION_CODE

cat << EOF > file-cache-csi-policy.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "ds:DescribeDirectories", "fsx:*" ], "Resource": "*" }, { "Effect": "Allow", "Action": "iam:CreateServiceLinkedRole", "Resource": "*", "Condition": { "StringEquals": { "iam:AWSServiceName": [ "fsx.amazonaws.com" ] } } }, { "Effect": "Allow", "Action": "iam:CreateServiceLinkedRole", "Resource": "*", "Condition": { "StringEquals": { "iam:AWSServiceName": [ "s3.data-source.lustre.fsx.amazonaws.com" ] } } }, { "Effect": "Allow", "Action": [ "logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents" ], "Resource": [ "arn:aws:logs:*:*:log-group:/aws/fsx/*" ] }, { "Effect": "Allow", "Action": [ "firehose:PutRecord" ], "Resource": [ "arn:aws:firehose:*:*:deliverystream/aws-fsx-*" ] }, { "Effect": "Allow", "Action": [ "ec2:CreateTags" ], "Resource": [ "arn:aws:ec2:*:*:route-table/*" ], "Condition": { "StringEquals": { "aws:RequestTag/AmazonFSx": "ManagedByAmazonFSx" }, "ForAnyValue:StringEquals": { "aws:CalledVia": [ "fsx.amazonaws.com" ] } } }, { "Effect": "Allow", "Action": [ "ec2:DescribeSecurityGroups", "ec2:DescribeSubnets", "ec2:DescribeVpcs" ], "Resource": "*", "Condition": { "ForAnyValue:StringEquals": { "aws:CalledVia": [ "fsx.amazonaws.com" ] } } } ] } EOF

aws iam create-policy --policy-name FileCacheCSIPolicy --policy-document file://file-cache-csi-policy.json

NODEGROUP_ROLE_NAME=$(aws eks describe-nodegroup --cluster-name $CLUSTER_NAME --nodegroup-name $NODE_GROUP_NAME --query "nodegroup.nodeRole" --output text | cut -d'/' -f2-) aws iam attach-role-policy --role-name $NODEGROUP_ROLE_NAME --policy-arn arn:aws:iam::362708816803:policy/FileCacheCSIPolicy

kubectl annotate serviceaccount file-cache-csi-controller-sa -n kube-system meta.helm.sh/release-name=aws-file-cache-csi-driver --overwrite kubectl annotate serviceaccount file-cache-csi-controller-sa -n kube-system meta.helm.sh/release-namespace=kube-system --overwrite kubectl label serviceaccount file-cache-csi-controller-sa -n kube-system app.kubernetes.io/managed-by=Helm --overwrite

helm repo add aws-file-cache-csi-driver https://kubernetes-sigs.github.io/aws-file-cache-csi-driver/ helm repo update helm install aws-file-cache-csi-driver aws-file-cache-csi-driver/aws-file-cache-csi-driver \ -n kube-system \ --set clusterName=$CLUSTER_NAME \ --set serviceAccount.create=false \ --set serviceAccount.name=file-cache-csi-controller-sa

export CLUSTER_SG=$(aws eks describe-cluster --name $CLUSTER_NAME --query cluster.resourcesVpcConfig.clusterSecurityGroupId --output text) aws ec2 authorize-security-group-ingress --group-id $CLUSTER_SG --protocol tcp --port 988 --cidr 0.0.0.0/0 > /dev/null aws ec2 authorize-security-group-egress --group-id $CLUSTER_SG --protocol tcp --port 988 --cidr 0.0.0.0/0 > /dev/null aws ec2 authorize-security-group-ingress --group-id $CLUSTER_SG --protocol tcp --port 1018-1023 --cidr 0.0.0.0/0 > /dev/null aws ec2 authorize-security-group-egress --group-id $CLUSTER_SG --protocol tcp --port 1018-1023 --cidr 0.0.0.0/0 > /dev/null

kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: fc-sc provisioner: filecache.csi.aws.com parameters: subnetId: subnet-02799ffd9a3b1d8ee securityGroupIds: sg-0b0f4c71db4e2eff6 dataRepositoryAssociations: "FileCachePath=/ns1/,DataRepositoryPath=nfs://10.0.92.69/,NFS={Version=NFS3},DataRepositorySubdirectories=[subdir1,subdir2,subdir3]" fileCacheType: "LUSTRE" fileCacheTypeVersion: "2.12" weeklyMaintenanceStartTime: "7:00:00" LustreConfiguration: "DeploymentType=CACHE_1,PerUnitStorageThroughput=1000,MetadataConfiguration={StorageCapacity=2400}" copyTagsToDataRepositoryAssociations: "true" extraTags: "skills=app" mountOptions: - flock

kubectl apply -f storageclass.yaml kubectl describe sc

apiVersion: v1 kind: PersistentVolumeClaim metadata: name: fc-claim spec: accessModes: - ReadWriteMany storageClassName: fc-sc resources: requests: storage: 1200Gi

kubectl apply -f claim.yaml kubectl describe pvc

apiVersion: v1 kind: Pod metadata: name: fc-app spec: containers: - name: app image: centos command: ["/bin/sh"] args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"] volumeMounts: - name: persistent-storage mountPath: /data volumes: - name: persistent-storage persistentVolumeClaim: claimName: fc-claim

kubectl apply -f pod.yaml kubectl exec -ti fc-app -- df -h Filesystem Size Used Avail Use% Mounted on overlay 80G 4.0G 77G 5% / tmpfs 64M 0 64M 0% /dev tmpfs 3.8G 0 3.8G 0% /sys/fs/cgroup 192.168.100.210@tcp:/d4v2dbev 1.2T 11M 1.2T 1% /data /dev/nvme0n1p1 80G 4.0G 77G 5% /etc/hosts shm 64M 0 64M 0% /dev/shm tmpfs 7.0G 12K 7.0G 1% /run/secrets/kubernetes.io/serviceaccount tmpfs 3.8G 0 3.8G 0% /proc/acpi tmpfs 3.8G 0 3.8G 0% /sys/firmware kubectl exec -it fc-app -- ls /data > out.txt