VMware/TMConEKS

TMC on EKS Setup

Create Harbor on AWS

SSH to Harbor-AWS EC2 Instance

#On Plex
ssh -i ~/.ssh/aws-keypair.pem [email protected]

Install Harbor-AWS

#THIS IS JUST PULLING IN MY STAR CERT and KEY FOR DERSLLC
mkdir /data
cd /data
curl https://ders-gitlab.dersllc.com/ders/ders-proxy/-/raw/master/STAR_dersllc_com.crt > /data/ders-star-chain.crt
curl https://ders-gitlab.dersllc.com/ders/ders-proxy/-/raw/master/AddTrustExternalCARoot.crt >> /data/ders-star-chain.crt
cp /data/ders-star-chain.crt /data/ders-star-chain.pem
curl https://ders-gitlab.dersllc.com/ders/ders-proxy/-/raw/master/AddTrustExternalCARoot.crt > /data/ders-ca.crt
cat /data/ders-ca.crt >> /etc/pki/tls/certs/ca-bundle.crt
curl https://ders-gitlab.dersllc.com/ders/ders-proxy/-/raw/master/dersllc-new.key > /data/ders-star.key

yum -y install docker
service docker start
systemctl enable docker

wget https://github.com/docker/compose/releases/download/v2.18.1/docker-compose-linux-x86_64 -O /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
wget https://github.com/goharbor/harbor/releases/download/v2.8.1/harbor-offline-installer-v2.8.1.tgz
tar -zxvf harbor-offline-installer-v2.8.1.tgz
cd harbor
cp harbor.yml.tmpl harbor.yml
vi harbor.yml
# add Hostname and certs and location
#### harbor-aws.dersllc.com
#### /data/ders-star-chain.crt
#### /data/ders-star.key
#### /data/harbor-data

./install.sh --with-trivy --with-notary

Load Harbor with Images

# On a Jumpbox...
docker login harbor-aws.dersllc.com
#################################
#     Set Install Variables     #
#################################
export IMGPKG_REGISTRY_HOSTNAME_0="harbor-aws.dersllc.com"
export IMGPKG_REGISTRY_USERNAME_0="admin"
export IMGPKG_REGISTRY_PASSWORD_0=<PASSWORD>
export PRIVATE_IMAGE_REGISTRY_CA_PATH="/data/ders-ca.crt"
export PRIVATE_IMAGE_REGISTRY="harbor-aws.dersllc.com"
export TKG_IMAGE_REGISTRY="projects.registry.vmware.com/tkg"
export TKG_REPO_VERSION="v2.2.0_update.1"
export TMC_HARBOR_PROJECT="tmc-sm-1.0.0"
export TMC_BUNDLE="tmc-self-managed-1.0.0"
#########################
#     Prep TMC Bits     #
#########################
mkdir tmc
tar -xf $TMC_BUNDLE.tar -C tmc
chmod +x /usr/local/bin/tmc
tmc/tmc-sm push-images harbor --project $IMGPKG_REGISTRY_HOSTNAME_0/$TMC_HARBOR_PROJECT --username $IMGPKG_REGISTRY_USERNAME_0 --password $IMGPKG_REGISTRY_PASSWORD_0
############################################################
#     Upload Tanzu Standard Packages for TMC into Reop     #
############################################################
imgpkg copy --registry-ca-cert-path $PRIVATE_IMAGE_REGISTRY_CA_PATH \
-b $TKG_IMAGE_REGISTRY/packages/standard/repo:$TKG_REPO_VERSION \
--to-repo $PRIVATE_IMAGE_REGISTRY/$TMC_HARBOR_PROJECT/498533941640.dkr.ecr.us-west-2.amazonaws.com/packages/standard/repo

Setup EKS Cluster

#########################
#     AWS AUTH INFO     #
#########################
export AWS_ACCESS_KEY_ID=<AWS_ACCESS_KEY_ID>
export AWS_SECRET_ACCESS_KEY=<AWS_SECRET_ACCESS_KEY>
export AWS_SESSION_TOKEN=<AWS_SESSION_TOKEN>

##########################
#     INSTALL AWSCLI     #
##########################
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install
aws --version

##########################
#     Install EKSCLI     #
##########################
curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp
sudo mv /tmp/eksctl /usr/local/bin

export eks_region=us-east-2
export eks_cluster_name=ders-tmc-sm
export eks_nodegroup_size=m5.xlarge

eksctl create cluster --name $eks_cluster_name -r $eks_region -t $eks_nodegroup_size --nodes 3 --nodes-min 3 --nodes-max 11
#Wait for Status Complete....... 

#Get Kubeconfig for EKS if needed
#aws eks update-kubeconfig --region $eks_region --name $eks_cluster_name

#Add another Node group if needed
#export eks_nodegroup_name=ders-tmc-nodes
#eksctl create nodegroup -c $eks_cluster_name -r $eks_region -n $eks_nodegroup_name -t $eks_nodegroup_size --nodes 3 --nodes-min 3 --nodes-max 11

Prepare EKS for TMC Deploy

# Prep the EBS CSI
eksctl utils associate-iam-oidc-provider \
 --region=$eks_region \
 --cluster=$eks_cluster_name \
 --approve

eksctl create iamserviceaccount \
 --name ebs-csi-controller-sa \
 --namespace kube-system \
 --cluster $eks_cluster_name \
 --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \
 --approve \
 --role-only \
 --region=$eks_region \
 --role-name AmazonEKS_EBS_CSI_DriverRole

eksctl create addon --name aws-ebs-csi-driver \
 --cluster $eks_cluster_name \
 --service-account-role-arn arn:aws:iam::$(aws sts get-caller-identity --query Account --output text):role/AmazonEKS_EBS_CSI_DriverRole \
 --region=$eks_region \
 --force

Create CA Cert for Cert-Manager

################################################
#     Create Config for CA SSL Certificate     #
################################################
mkdir -p /tmp/ssl
cd /tmp/ssl
cat <<EOF > ca.cnf
[ req ]
default_bits = 2048
default_md = sha256
prompt = no
encrypt_key = no
distinguished_name = dn
[ dn ]
countryName = US
stateOrProvinceName = Ohio
localityName = Beavercreek
organizationName = DER's LLC
organizationalUnitName = IT
commonName = ca.dersllc.com
[ext]
keyUsage=critical,keyCertSign,cRLSign
basicConstraints=critical,CA:true,pathlen:1
subjectAltName=DNS:ca.dersllc.com 
EOF

###############################################################
#     Create a Self-Signed CA Cert Using the Config Above     #
###############################################################
openssl req -x509 -nodes -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout ders-ca.key -out ders-ca.cer -extensions ext -config ca.cnf

########################################################################################
#     Get Base64 encoded PEM and KEY for the K8S Issuer YAML Used in the Next Step     #
########################################################################################
cat ders-ca.cer | base64 -w0
cat ders-ca.key | base64 -w0

Install TMC

################################################
#     Setup TMC Namespace and Cert-Manager     #
################################################
kubectl create ns tmc-local
# Deploy Cert-Manager into EKS Cluster
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.12.0/cert-manager.yaml
sleep 60
# Add ders-ca issuer (this is referenced in the tmc-eks-values.yaml)
kubectl apply -f https://ders-gitlab.dersllc.com/ders/vmware-se/-/raw/main/HomeLab/TMC%20on%20TKGm/tmc-issuer.yaml 

################################################################
#     Setup Harbor Credentials For The tmc-local Namespace     #
################################################################
kubectl create secret generic regcred --from-file=.dockerconfigjson=/root/.docker/config.json --type=kubernetes.io/dockerconfigjson -n tmc-local
kubectl patch serviceaccount default -p "{\"imagePullSecrets\": [{\"name\": \"regcred\"}]}" -n tmc-local

##################################
#     Create TMC Values File     #
##################################
#tmc/tmc-sm generate-values-schema --output-file tmc-eks-values.yaml  #(Depending on the version or TMC you will use this or the next command.)
#tmc/tmc-sm show-values-schema --output-filet tmc-eks-values.yaml

#############################################
#     Pull TMC Values File and Validate     #
#############################################
curl https://ders-gitlab.dersllc.com/ders/vmware-se/-/raw/main/HomeLab/TMC/tmc-eks-values.yaml > tmc-eks-values.yaml
tmc/tmc-sm validate-values tmc-values.yaml

############################################
#     Finally Install TMC Self-Managed     #
############################################
tmc/tmc-sm deploy --image-prefix $PRIVATE_IMAGE_REGISTRY/$TMC_HARBOR_PROJECT --kubeconfig ~/.kube/config --values=tmc-eks-values.yaml

###############################################################
#     During Install get the LB IP/Name and add it to DNS     #
###############################################################
kubectl get service -n tmc-local contour-envoy -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'

Uninstall TMC

tmc/tmc-local uninstall --kubeconfig ~/.kube/config

Setup Inspection Images

Run the below command to create the download script.

cat > ./inspection-images.sh << "EOF"
   #!/bin/bash
   
   # https://github.com/vmware-tanzu/sonobuoy/releases
   VERSION=${1:-"v0.56.16"}
   LATEST_RELEASE=${2:-"sonobuoy_0.56.16_linux_amd64.tar.gz"}
   CUSTOM_REGISTRY=${3:-"harbor-aws.dersllc.com/tmc-1.0.0-beta.2-rc.3"}
   DOCKER_PROXY=${4:-"harbor.tanzu.io:8443/dockerhub-proxy-cache"} # optional argument
   CUSTOM_TMC_REPO="${CUSTOM_REGISTRY}/498533941640.dkr.ecr.us-west-2.amazonaws.com"
   
   # https://kubernetes.io/releases/patch-releases/
   k8s_versions=(v1.23.8 v1.24.10)
   
   wget "https://github.com/vmware-tanzu/sonobuoy/releases/download/${VERSION}/${LATEST_RELEASE}"
   tar -xvf ${LATEST_RELEASE}
   
   for i in "${k8s_versions[@]}"
   do
   echo "================CHECKING K8S: $i======================="
   ./sonobuoy images list --kubernetes-version $i > images_$i.txt
   
   while read image
   do
   echo "================CHECKING IMAGE: $image=================="
   base=$(basename "$image")
   output=${image#*/*}
   
   if $image == *"docker"* && -n $DOCKER_PROXY ;
   then
       docker pull $DOCKER_PROXY/$output
       docker tag $DOCKER_PROXY/$output ${CUSTOM_TMC_REPO}/extensions/inspection-images/$base
   else
       docker pull $image
       docker tag $image ${CUSTOM_TMC_REPO}/extensions/inspection-images/$base
   fi
   
   docker push ${CUSTOM_TMC_REPO}/extensions/inspection-images/$base
   echo "===================PUSHING: ${CUSTOM_TMC_REPO}/extensions/inspection-images/$base ==========="
   done < images_$i.txt
   done
   
   # not part of sonobuoy image list, install manually, update these as images are found
   docker pull k8s.gcr.io/e2e-test-images/agnhost:2.31
   docker pull k8s.gcr.io/pause:3.9
   docker tag k8s.gcr.io/e2e-test-images/agnhost:2.31 ${CUSTOM_TMC_REPO}/extensions/inspection-images/agnhost:2.31
   docker tag k8s.gcr.io/pause:3.9 ${CUSTOM_TMC_REPO}/extensions/inspection-images/pause:3.9
   docker push ${CUSTOM_TMC_REPO}/extensions/inspection-images/agnhost:2.31
   docker push ${CUSTOM_TMC_REPO}/extensions/inspection-images/pause:3.9
   
   # clean up text files and sonobuoy tar
   rm images_*
   rm sonobuoy_*
EOF

Edit the file and set the Variables at the top. (VERSION, LATEST_RELEASE, CUSTOM_REGISTRY, and k8s_versions)

vi inspection-images.sh

Save the file and change the permissions

chmod +x inspection-images.sh

Run the Script

./inspection-images.sh