We use Parameter Store and EFS at our company heavily. Here are some simple steps to setup our EKS cluster with access to both.
Parameter Store
You can use ConfigMap to generate a list of parameter store ARNs into a file; then use aws-cli to parse that and stored into a new file. OR what I ended up doing was use secrets-init.
Setup a service account with parameter store access in Terraform:
The module.default-eks-c01.oidc-provider-url was created from the previous doc: EKS Group Access using AWS IAM. The aws_iam_policy.param_store_policy_ro.arn was created from the previous doc: AWS Parameter Store.
data "aws_iam_policy_document" "eks-default-service-account-assume-role-policy" {
statement {
actions = ["sts:AssumeRoleWithWebIdentity"]
effect = "Allow"
condition {
test = "StringEquals"
variable = "${replace(module.default-eks-c01.oidc-provider-url, "https://", "")}:sub"
values = ["system:serviceaccount:default:default-service-account"]
}
principals {
identifiers = [replace(module.default-eks-c01.oidc-provider-arn, "https://", "")]
type = "Federated"
}
}
}
resource "aws_iam_role" "eks-access-role-default-service-account" {
name = "${var.env}-eks-default-service-account-oidc-iam-role"
assume_role_policy = data.aws_iam_policy_document.eks-default-service-account-assume-role-policy.json
description = "Managed by Terraform"
}
resource "aws_iam_role_policy_attachment" "policy-attachment-eks-default-service-account" {
role = aws_iam_role.eks-access-role-default-service-account.name
policy_arn = aws_iam_policy.param_store_policy_ro.arn
}
ServiceAccount
Create the default-service-account ServiceAccount account to be attached to the application’s Deployment yaml file.
---
apiVersion: v1
kind: ServiceAccount
metadata:
annotations:
eks.amazonaws.com/role-arn: arn:aws:iam::xxxyyyzzz:role/dev-eks-default-service-account-oidc-iam-role
name: default-service-account
Deployment
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: dev-frontend-app-c01
spec:
replicas: 2
selector:
matchLabels:
app: dev-frontend-app-c01
template:
metadata:
labels:
app: dev-frontend-app-c01
name: dev-frontend-app-c01
spec:
serviceAccountName: default-service-account
initContainers:
- name: secrets-init
image: doitintl/secrets-init:v0.2.1
command:
- sh
args:
- -c
- cp /usr/local/bin/secrets-init /secrets-init/bin/;
/secrets-init/bin/secrets-init env |
grep -E `env | awk -F'=' '$2 ~ /arn:aws:ssm:.+:parameter/ {print $1}' | paste -sd '|' -` >
/parameter-store/secrets
env:
- name: AWS_REGION
value: us-east-1
- name: LICENSE_KEY
value: arn:aws:ssm:us-east-1:xxxyyyzzz:parameter/dev/newrelic/LICENSE_KEY
volumeMounts:
- mountPath: /secrets-init/bin
name: secrets-init-volume
- mountPath: /parameter-store
name: parameter-store-volume
containers:
- name: frontend
image: nginx
env:
- name: AWS_REGION
value: us-east-1
- name: LICENSE_KEY
value: arn:aws:ssm:us-east-1:xxxyyyzzz:parameter/dev/newrelic/LICENSE_KEY
ports:
- containerPort: 80
volumeMounts:
- mountPath: /secrets-init/bin
name: secrets-init-volume
- mountPath: /parameter-store
name: parameter-store-volume
volumes:
- name: secrets-init-volume
emptyDir: {}
- name: parameter-store-volume
emptyDir: {}
In the initContainers: block, we load the secrets-init container and store the values in a file to be used in the main containers: block.
The parameter store values will be stored in /parameter-store/secrets. We can source the file or eval them into environment variables before we start our application.
root@dev-frontend-app-c01-6fbf55dfdf-fbshf:/# cat /parameter-store/secrets
LICENSE_KEY=1234567890
Instead of running the the secrets-init env command in the init container, we can remove the following args:
/secrets-init/bin/secrets-init env |
grep -E `env | awk -F'=' '$2 ~ /arn:aws:ssm:.+:parameter/ {print $1}' | paste -sd '|' -` >
/parameter-store/secrets
Since our main containers: block has the parameter environments, we can just run the secrets-init in the main container.
root@dev-frontend-app-c01-6fbf55dfdf-fbshf:/# echo $LICENSE_KEY
arn:aws:ssm:us-east-1:xxxyyyzzz:parameter/dev/newrelic/LICENSE_KEY
root@dev-frontend-app-c01-6fbf55dfdf-fbshf:/# apt-get update && apt-get install awscli -y
...
root@dev-frontend-app-c01-6fbf55dfdf-fbshf:/# /secrets-init/bin/secrets-init env | grep LICENSE_KEY
LICENSE_KEY=1234567890
root@dev-frontend-app-c01-6fbf55dfdf-fbshf:/# /secrets-init/bin/secrets-init sh -c 'echo $LICENSE_KEY'
1234567890
EFS
Once you have the efs-provisioner service running by following the instructions from AWS Knowledge Center, we can start creating PVC in our application deployments.
Create Persistent Volume Claim
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
volume.beta.kubernetes.io/storage-class: efs
name: dev-eks-efs-data-frontend-c01
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 5Mi
$ kubectl get pvc dev-eks-efs-data-frontend-c01
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
dev-eks-efs-data-frontend-c01 Bound pvc-2c97c0ca-810a-11ea-bd9e-1238d378ba75 5Mi RWX aws-efs 6m14s
Create Deployment with EFS volume
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: dev-frontend-app-c01
spec:
replicas: 2
selector:
matchLabels:
app: dev-frontend-app-c01
template:
metadata:
labels:
app: dev-frontend-app-c01
name: dev-frontend-app-c01
spec:
containers:
- name: frontend
image: nginx
ports:
- containerPort: 80
volumeMounts:
- mountPath: /efs
name: efs-pvc
volumes:
- name: efs-pvc
persistentVolumeClaim:
claimName: dev-eks-efs-data-initjs-c01
$ kubectl exec -it dev-frontend-app-c01-7f85fc6c69-4hgfl -- 'df'
Filesystem 1K-blocks Used Available Use% Mounted on
overlay 20959212 8425264 12533948 41% /
...
fs-xxxxxxxx.efs.us-east-1.amazonaws.com:/dev-eks-efs-data-initjs-c01-pvc-xxxx 9007199254739968 13434880 9007199241305088 1% /efs
...