VMware/TMC

Tanzu Mission Control (TMC)

TMC on EKS

Prerequisites

 ####################################################
 #     Make Sure KeyCloak SSO Is Up and Running     #
 ####################################################
 ssh [email protected]
 docker stop keycloak; docker rm keycloak 
 docker run -d --name keycloak -p 8080:8080 -e KEYCLOAK_ADMIN=admin -e KEYCLOAK_ADMIN_PASSWORD='DERS4me!'  quay.io/keycloak/keycloak:20.0.2 start --proxy edge --hostname-strict=false

Install TMC

 #################################
 #     Set Install Variables     #
 #################################
 export IMGPKG_REGISTRY_HOSTNAME_0="harbor.dersllc.com"
 export IMGPKG_REGISTRY_USERNAME_0="admin"
 export IMGPKG_REGISTRY_PASSWORD_0="<PASSWORD>"
 export PRIVATE_IMAGE_REGISTRY_CA_PATH="/data/ders-ca.crt"
 export PRIVATE_IMAGE_REGISTRY="harbor.dersllc.com"
 export TKG_IMAGE_REGISTRY="projects.registry.vmware.com/tkg"
 export TKG_REPO_VERSION="v2024.2.1_tmc.1"
 export TMC_PROJECT="tmc-1.2"
 export TMC_BUNDLE="tmc_self_managed_1.2.0"
 
 #########################
 #     Prep TMC Bits     #
 #########################
 mkdir ./tanzumc
 tar -xvf $TMC_BUNDLE.tar -C ./tanzumc
 #chmod +x /usr/local/bin/tmc
 tanzumc/tmc-sm push-images harbor --project $IMGPKG_REGISTRY_HOSTNAME_0/$TMC_PROJECT --username $IMGPKG_REGISTRY_USERNAME_0 --password $IMGPKG_REGISTRY_PASSWORD_0
 
 ##################################################
 #     Upload Tanzu Standard Packages for TMC     #
 ##################################################
 imgpkg copy -b $TKG_IMAGE_REGISTRY/packages/standard/repo:$TKG_REPO_VERSION --to-tar tanzu-std-$TKG_REPO_VERSION.tar
 
 imgpkg copy --registry-ca-cert-path $PRIVATE_IMAGE_REGISTRY_CA_PATH \
   --tar tanzu-std-$TKG_REPO_VERSION.tar \
   --to-repo $PRIVATE_IMAGE_REGISTRY/$TMC_PROJECT/498533941640.dkr.ecr.us-west-2.amazonaws.com/packages/standard/repo
 
 imgpkg copy --registry-ca-cert-path $PRIVATE_IMAGE_REGISTRY_CA_PATH \
 -b $TKG_IMAGE_REGISTRY/packages/standard/repo:$TKG_REPO_VERSION \
 --to-repo $PRIVATE_IMAGE_REGISTRY/$TMC_PROJECT/498533941640.dkr.ecr.us-west-2.amazonaws.com/packages/standard/repo
 
 #######################################
 #     Install Tanzu Standard Repo     #
 #######################################
 kubectl config use-context tmc-admin@tmc
 tanzu package repository add tanzu-standard \
   --url $PRIVATE_IMAGE_REGISTRY/$TMC_PROJECT/498533941640.dkr.ecr.us-west-2.amazonaws.com/packages/standard/repo:$TKG_REPO_VERSION \
   --namespace tkg-system
 tanzu package repository get tanzu-standard --namespace tkg-system
 tanzu package available list --namespace tkg-system
 
 #########################################
 #     Install Tanzu Mission Control     #
 #########################################
 kubectl config use-context tmc-admin@tmc
 kubectl create ns tmc-local
 tanzu package install cert-manager -p cert-manager.tanzu.vmware.com -v 1.12.2+vmware.1-tkg.1 -n tkg-system
 kubectl apply -f https://ders-gitlab.dersllc.com/ders/vmware-se/-/raw/main/HomeLab/TMC%20on%20TKGm/tmc-issuer.yaml
 kubectl create secret generic regcred --from-file=.dockerconfigjson=/root/.docker/config.json --type=kubernetes.io/dockerconfigjson -n tmc-local
 kubectl patch serviceaccount default -p "{\"imagePullSecrets\": [{\"name\": \"regcred\"}]}" -n tmc-local
 
 #tanzumc/tmc-sm generate-values-schema --output-file tmc-values.yaml
 #tanzumc/tmc-sm show-values-schema --output-filet tmc-values-defrault.json
 
 curl https://ders-gitlab.dersllc.com/ders/vmware-se/-/raw/main/HomeLab/TMC%20on%20TKGm/tmc-values.yaml > tmc-values.yaml
 #tanzumc/tmc-sm validate-values tmc-values.yaml
 #tanzumc/tmc-sm deploy --image-prefix $PRIVATE_IMAGE_REGISTRY/$TMC_PROJECT --kubeconfig ~/.kube/config --values=tmc-values.yaml

cat tanzumc/pushed-package-repository.json
tanzu package repository add tanzu-mission-control-packages --url "harbor.dersllc.com/tmc-1.1/package-repository:1.1.0" --namespace tmc-local
tanzu package repository list --namespace tmc-local

tanzu package install tanzu-mission-control -p "tmc.tanzu.vmware.com" --version "1.1.0" --values-file "tmc-values.yaml" --namespace tmc-local

Register TKGS Supervisor Cluster

ssh [email protected]
shell
/usr/lib/vmware-wcp/decryptK8Pwd.py

ssh [email protected]
ssh [email protected]
ssh [email protected]

#SSH to each host
curl --insecure https://ders-gitlab.dersllc.com/ders/ders-proxy/-/raw/master/AddTrustExternalCARoot.crt > /etc/ssl/certs/ders-star-chain.pem
chmod 644 /etc/ssl/certs/ders-star-chain.pem
cat /etc/ssl/certs/ders-star-chain.pem

#curl --insecure https://ders-gitlab.dersllc.com/ders/ders-proxy/-/raw/master/AddTrustExternalCARoot.crt >> /etc/kubernetes/pki/ca.crt
#cat /etc/kubernetes/pki/ca.crt
systemctl restart containerd.service

Transfer Tanzu Packages to TMC Repo

export IMGPKG_REGISTRY_HOSTNAME_0="harbor.dersllc.com"
export IMGPKG_REGISTRY_USERNAME_0="admin"
export IMGPKG_REGISTRY_PASSWORD_0="<PASSWORD>"
export TKG_IMAGE_REGISTRY="projects.registry.vmware.com/tkg"
export PRIVATE_IMAGE_REGISTRY="harbor.dersllc.com"
export TMC_PROJECT="tmc-1.0.0-beta.1-rc.2"
imgpkg copy \
  -b harbor.dersllc.com/tanzu_21/packages/standard/repo:v2.1.1 \
  --to-tar tanzu-std-2.1.1.tar

imgpkg copy --registry-ca-cert-path $REGISTRY_CA_PATH \
  --tar tanzu-std-2.1.1.tar \
  --to-repo harbor.dersllc.com/tmc-1.0.0-beta.1-rc.2/498533941640.dkr.ecr.us-west-2.amazonaws.com/packages/standard/repo 

imgpkg copy --registry-ca-cert-path=/data/cert/ca.pem \
  -b ${TKG_IMAGE_REGISTRY}/packages/standard/repo:v2.1.1 --to-repo harbor.dersllc.com/tmc-1.0.0-beta.1-rc.2/498533941640.dkr.ecr.us-west-2.amazonaws.com/packages/standard/repo

Setup Inspection Images

Run the below command to create the download script.

cat > ./inspection-images.sh << "EOF"
   #!/bin/bash
   
   # https://github.com/vmware-tanzu/sonobuoy/releases
   VERSION=${1:-"v0.56.16"}
   LATEST_RELEASE=${2:-"sonobuoy_0.56.16_linux_amd64.tar.gz"}
   CUSTOM_REGISTRY=${3:-"harbor.dersllc.com/tmc"}
   DOCKER_PROXY=${4:-"harbor.tanzu.io:8443/dockerhub-proxy-cache"} # optional argument
   CUSTOM_TMC_REPO="${CUSTOM_REGISTRY}/498533941640.dkr.ecr.us-west-2.amazonaws.com"
   
   # https://kubernetes.io/releases/patch-releases/
   k8s_versions=(v1.26.5 v1.24.10)
   
   wget "https://github.com/vmware-tanzu/sonobuoy/releases/download/${VERSION}/${LATEST_RELEASE}"
   tar -xvf ${LATEST_RELEASE}
   
   for i in "${k8s_versions[@]}"
   do
   echo "================CHECKING K8S: $i======================="
   ./sonobuoy images list --kubernetes-version $i > images_$i.txt
   
   while read image
   do
   echo "================CHECKING IMAGE: $image=================="
   base=$(basename "$image")
   output=${image#*/*}
   
   if $image == *"docker"* && -n $DOCKER_PROXY ;
   then
       docker pull $DOCKER_PROXY/$output
       docker tag $DOCKER_PROXY/$output ${CUSTOM_TMC_REPO}/extensions/inspection-images/$base
   else
       docker pull $image
       docker tag $image ${CUSTOM_TMC_REPO}/extensions/inspection-images/$base
   fi
   
   docker push ${CUSTOM_TMC_REPO}/extensions/inspection-images/$base
   echo "===================PUSHING: ${CUSTOM_TMC_REPO}/extensions/inspection-images/$base ==========="
   done < images_$i.txt
   done
   
   # not part of sonobuoy image list, install manually, update these as images are found
   docker pull k8s.gcr.io/e2e-test-images/agnhost:2.31
   docker pull k8s.gcr.io/pause:3.9
   docker tag k8s.gcr.io/e2e-test-images/agnhost:2.31 ${CUSTOM_TMC_REPO}/extensions/inspection-images/agnhost:2.31
   docker tag k8s.gcr.io/pause:3.9 ${CUSTOM_TMC_REPO}/extensions/inspection-images/pause:3.9
   docker push ${CUSTOM_TMC_REPO}/extensions/inspection-images/agnhost:2.31
   docker push ${CUSTOM_TMC_REPO}/extensions/inspection-images/pause:3.9
   
   # clean up text files and sonobuoy tar
   rm images_*
   rm sonobuoy_*
EOF

Edit the file and set the Variables at the top.

vi inspection-images.sh

Save the file and change the permissions

chmod +x inspection-images.sh

Run the Script

./inspection-images.sh

Uninstall TMC

tanzu package installed delete tanzu-mission-control --namespace tmc-local

Troubleshooting TMC

Force Install of TMC Agent on TKGS

ssh [email protected]
shell
/usr/lib/vmware-wcp/decryptK8Pwd.py

#SSH to each supervisor host
curl --insecure https://ders-gitlab.dersllc.com/ders/ders-proxy/-/raw/master/AddTrustExternalCARoot.crt > /etc/ssl/certs/ders-star-chain.pem
chmod 644 /etc/ssl/certs/ders-star-chain.pem
cat /etc/ssl/certs/ders-star-chain.pem

# On one of the hosts
export REG_URL="https://tmc.dersllc.com/installer?id=77c352bf3e6e43e10b01abe83cf3a2b46220826d0dc8fd6182a018df05a491b5&source=registration&type=tkgs"
curl --insecure "$REG_URL" > tmc-reg.yaml
sed -i 's/{{.Namespace}}/svc-tmc-c8/g' tmc-reg.yaml
kubectl apply -f tmc-reg.yaml

References

https://beyondelastic.com/2023/07/25/tmc-self-managed-e2e-implementation-guide/

KeyCloak

https://gist.github.com/gorkemozlu/3b09a27de9c7c1e3d27a4402bfb70aba#file-keycloak-md

Troubleshooting

ERROR: "Could not exchange authorization code"

Option 1

• Uninstall TMC
• Delete TMC-LOCAL Namespace.

kubectl delete ns tmc-local

• Re-Install TMC

Note: Something is still hanging onto an old version of the authorization info. must blow the entire namespace away to make sure all of the config is deleted.

Option 2

Run this command:

kubectl -n tmc-local delete oidcclient/client.oauth.pinniped.dev-auth-manager-pinniped-oidc-client secret/client.oauth.pinniped.dev-auth-manager-pinniped-oidc-client-client-secret-generated; 
kubectl -n tmc-local delete po -lapp=authenticator; 
kubectl delete lease authenticator-leader-elect;

and wait for a couple of minutes for the resources to be reconciled.. Then run this command to get a confirmation:

kubectl -n tmc-local get oidcclient/client.oauth.pinniped.dev-auth-manager-pinniped-oidc-client secret/client.oauth.pinniped.dev-auth-manager-pinniped-oidc-client-client-secret-generated

Once this is done, please try login

ERROR: Failed to validate token claims

#GET Grpc-Metadata-X-User-Id: from the Request Header
JWT_HEADER=eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJhZGRpdGlvbmFsQ2xhaW1zIjp7ImVtYWlsIjoiZGFuaWVsLnJvZXNzbmVyQGRlcnNsbGMuY29tIiwibmFtZSI6ImFkbWluIiwidGVuYW50X2lkIjoiMSJ9LCJhdF9oYXNoIjoiZmZvVEpyMGdQUzJuNHhLeHgzNkZxUSIsImF1ZCI6WyJjbGllbnQub2F1dGgucGlubmlwZWQuZGV2LWF1dGgtbWFuYWdlci1waW5uaXBlZC1vaWRjLWNsaWVudCJdLCJhdXRoX3RpbWUiOjE3MjIzNTMzNDcsImF6cCI6ImNsaWVudC5vYXV0aC5waW5uaXBlZC5kZXYtYXV0aC1tYW5hZ2VyLXBpbm5pcGVkLW9pZGMtY2xpZW50IiwiZXhwIjoxNzIyMzYzMTM5LCJncm91cHMiOlsiYWRtaW4iLCJjcmVhdGUtcmVhbG0iLCJkZWZhdWx0LXJvbGVzLW1hc3RlciIsIm9mZmxpbmVfYWNjZXNzIiwidG1jOmFkbWluIiwidW1hX2F1dGhvcml6YXRpb24iXSwiaWF0IjoxNzIyMzYzMDE5LCJpc3MiOiJodHRwczovL3Bpbm5pcGVkLXN1cGVydmlzb3IudG1jLmRlcnNsbGMuY29tL3Byb3ZpZGVyL3Bpbm5pcGVkIiwianRpIjoiY2I1Y2Y4ZGEtYWYzZS00ODgwLWFkMTktMzZkZTRkZWEzMmQ4IiwicmF0IjoxNzIyMzUzMzQ3LCJzdWIiOiJodHRwczovL3Nzby5kZXJzbGxjLmNvbS9yZWFsbXMvbWFzdGVyP2lkcE5hbWU9cGlubmlwZWQtdXBzdHJlYW1cdTAwMjZzdWI9MDc2MzkxZGEtNDdlMC00OTZlLTg0OGItM2NlZjQ2MmVjMGQwIiwidXNlcm5hbWUiOiJkYW5pZWwucm9lc3NuZXJAZGVyc2xsYy5jb20ifQ.9ITbF9nujGT812SM0eXmWhrKmGrAGimUMCbx5SCuFiJtt32aYl2yHMuGhIBNE_keJJ0QJ_n2Of91FWP9FWhRnA
echo $JWT_HEADER | sed 's/\./\n/g' | cut -d. -f1  | base64 --decode | jq

kubectl logs -n tmc-local --selector app=cluster-agent > cluster-agent-logs.txt
cat cluster-agent-logs.txt | grep error
cat cluster-agent-logs.txt | grep x509

kubectl get cm tls-ca-bundles -n tmc-local -o yaml

Make Sure that the Harbor Cert is in the tmc-values.yaml and make sure the syntax is correct!!!