xservices.ml.aws.caas.upbound.io
The following resources are composed to implement the referenced Composite Resource Definition (XRD).
XIRSA
Release
Object
Object
NodeGroup
Release
Release
Object
Object
Release
Release
Object
Object
Object
kind: Composition
apiVersion: apiextensions.crossplane.io/v1
metadata:
name: xservices.ml.aws.caas.upbound.io
creationTimestamp: null
spec:
compositeTypeRef:
apiVersion: ml.aws.caas.upbound.io/v1alpha1
kind: XService
patchSets:
- name: providerConfigRef
patches:
- type: FromCompositeFieldPath
fromFieldPath: spec.parameters.providerConfigName
toFieldPath: spec.providerConfigRef.name
- name: deletionPolicy
patches:
- type: FromCompositeFieldPath
fromFieldPath: spec.parameters.deletionPolicy
toFieldPath: spec.deletionPolicy
- name: region
patches:
- type: FromCompositeFieldPath
fromFieldPath: spec.parameters.region
toFieldPath: spec.forProvider.region
resources:
- name: loadbalancerControllerIrsa
base:
apiVersion: aws.caas.upbound.io/v1alpha1
kind: XIRSA
spec:
parameters:
condition: StringEquals
policyDocument: >
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"iam:CreateServiceLinkedRole"
],
"Resource": "*",
"Condition": {
"StringEquals": {
"iam:AWSServiceName": "elasticloadbalancing.amazonaws.com"
}
}
},
{
"Effect": "Allow",
"Action": [
"ec2:DescribeAccountAttributes",
"ec2:DescribeAddresses",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeInternetGateways",
"ec2:DescribeVpcs",
"ec2:DescribeVpcPeeringConnections",
"ec2:DescribeSubnets",
"ec2:DescribeSecurityGroups",
"ec2:DescribeInstances",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeTags",
"ec2:GetCoipPoolUsage",
"ec2:DescribeCoipPools",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeLoadBalancerAttributes",
"elasticloadbalancing:DescribeListeners",
"elasticloadbalancing:DescribeListenerCertificates",
"elasticloadbalancing:DescribeSSLPolicies",
"elasticloadbalancing:DescribeRules",
"elasticloadbalancing:DescribeTargetGroups",
"elasticloadbalancing:DescribeTargetGroupAttributes",
"elasticloadbalancing:DescribeTargetHealth",
"elasticloadbalancing:DescribeTags"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"cognito-idp:DescribeUserPoolClient",
"acm:ListCertificates",
"acm:DescribeCertificate",
"iam:ListServerCertificates",
"iam:GetServerCertificate",
"waf-regional:GetWebACL",
"waf-regional:GetWebACLForResource",
"waf-regional:AssociateWebACL",
"waf-regional:DisassociateWebACL",
"wafv2:GetWebACL",
"wafv2:GetWebACLForResource",
"wafv2:AssociateWebACL",
"wafv2:DisassociateWebACL",
"shield:GetSubscriptionState",
"shield:DescribeProtection",
"shield:CreateProtection",
"shield:DeleteProtection"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"ec2:AuthorizeSecurityGroupIngress",
"ec2:RevokeSecurityGroupIngress"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"ec2:CreateSecurityGroup"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"ec2:CreateTags"
],
"Resource": "arn:aws:ec2:*:*:security-group/*",
"Condition": {
"StringEquals": {
"ec2:CreateAction": "CreateSecurityGroup"
},
"Null": {
"aws:RequestTag/elbv2.k8s.aws/cluster": "false"
}
}
},
{
"Effect": "Allow",
"Action": [
"ec2:CreateTags",
"ec2:DeleteTags"
],
"Resource": "arn:aws:ec2:*:*:security-group/*",
"Condition": {
"Null": {
"aws:RequestTag/elbv2.k8s.aws/cluster": "true",
"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
}
}
},
{
"Effect": "Allow",
"Action": [
"ec2:AuthorizeSecurityGroupIngress",
"ec2:RevokeSecurityGroupIngress",
"ec2:DeleteSecurityGroup"
],
"Resource": "*",
"Condition": {
"Null": {
"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
}
}
},
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:CreateLoadBalancer",
"elasticloadbalancing:CreateTargetGroup"
],
"Resource": "*",
"Condition": {
"Null": {
"aws:RequestTag/elbv2.k8s.aws/cluster": "false"
}
}
},
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:CreateListener",
"elasticloadbalancing:DeleteListener",
"elasticloadbalancing:CreateRule",
"elasticloadbalancing:DeleteRule"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:AddTags",
"elasticloadbalancing:RemoveTags"
],
"Resource": [
"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
],
"Condition": {
"Null": {
"aws:RequestTag/elbv2.k8s.aws/cluster": "true",
"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
}
}
},
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:AddTags",
"elasticloadbalancing:RemoveTags"
],
"Resource": [
"arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*",
"arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*",
"arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*",
"arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*"
]
},
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:ModifyLoadBalancerAttributes",
"elasticloadbalancing:SetIpAddressType",
"elasticloadbalancing:SetSecurityGroups",
"elasticloadbalancing:SetSubnets",
"elasticloadbalancing:DeleteLoadBalancer",
"elasticloadbalancing:ModifyTargetGroup",
"elasticloadbalancing:ModifyTargetGroupAttributes",
"elasticloadbalancing:DeleteTargetGroup"
],
"Resource": "*",
"Condition": {
"Null": {
"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
}
}
},
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:AddTags"
],
"Resource": [
"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
],
"Condition": {
"StringEquals": {
"elasticloadbalancing:CreateAction": [
"CreateTargetGroup",
"CreateLoadBalancer"
]
},
"Null": {
"aws:RequestTag/elbv2.k8s.aws/cluster": "false"
}
}
},
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:RegisterTargets",
"elasticloadbalancing:DeregisterTargets"
],
"Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*"
},
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:SetWebAcl",
"elasticloadbalancing:ModifyListener",
"elasticloadbalancing:AddListenerCertificates",
"elasticloadbalancing:RemoveListenerCertificates",
"elasticloadbalancing:ModifyRule"
],
"Resource": "*"
}
]
}
serviceAccount:
name: aws-load-balancer-controller
namespace: kube-system
patches:
- fromFieldPath: spec.parameters.providerConfigName
toFieldPath: spec.parameters.providerConfigName
- fromFieldPath: spec.parameters.deletionPolicy
toFieldPath: spec.parameters.deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: metadata.name
transforms:
- type: string
string:
fmt: "%s-loadbalancer-controller"
- fromFieldPath: spec.parameters.id
toFieldPath: spec.parameters.id
- type: ToCompositeFieldPath
fromFieldPath: status.roleArn
toFieldPath: status.status.loadbalancerControllerRoleArn
policy:
fromFieldPath: Optional
- name: loadbalancerControllerChart
base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
spec:
forProvider:
chart:
name: aws-load-balancer-controller
repository: https://aws.github.io/eks-charts
version: 1.4.4
namespace: kube-system
values:
image:
repository: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon/aws-load-balancer-controller
serviceAccount:
name: aws-load-balancer-controller
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- fromFieldPath: status.status.clusterName
toFieldPath: spec.forProvider.values.clusterName
policy:
fromFieldPath: Required
- fromFieldPath: status.status.loadbalancerControllerRoleArn
toFieldPath: spec.forProvider.values.serviceAccount.annotations[eks.amazonaws.com/role-arn]
policy:
fromFieldPath: Required
- name: gp2NoDefault
base:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
forProvider:
manifest:
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
storageclass.kubernetes.io/is-default-class: "false"
name: gp2
parameters:
fsType: ext4
type: gp2
provisioner: kubernetes.io/aws-ebs
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- name: gp3AsDefault
base:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
forProvider:
manifest:
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
annotations:
storageclass.kubernetes.io/is-default-class: "true"
name: gp3
parameters:
csi.storage.k8s.io/fstype: ext4
encrypted: "true"
type: gp3
provisioner: ebs.csi.aws.com
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- name: eksNodeGroupGPU
base:
apiVersion: eks.aws.upbound.io/v1beta1
kind: NodeGroup
spec:
forProvider:
amiType: AL2_x86_64_GPU
diskSize: 100
instanceTypes:
- g5.12xlarge
labels:
NodeGroupType: gpu
WorkerType: ON_DEMAND
nodeRoleArnSelector:
matchLabels:
role: nodegroup
scalingConfig:
- desiredSize: 1
maxSize: 100
minSize: 1
subnetIdSelector:
matchLabels:
access: public
taint:
- effect: NO_SCHEDULE
key: nvidia.com/gpu
value: EXISTS
patches:
- type: PatchSet
patchSetName: providerConfigRef
- type: PatchSet
patchSetName: deletionPolicy
- type: PatchSet
patchSetName: region
- fromFieldPath: spec.parameters.nodes.count
toFieldPath: spec.forProvider.scalingConfig[0].desiredSize
- fromFieldPath: spec.parameters.nodes.instanceType
toFieldPath: spec.forProvider.instanceTypes[0]
- fromFieldPath: spec.parameters.nodes.diskSize
toFieldPath: spec.forProvider.diskSize
- fromFieldPath: spec.parameters.id
toFieldPath: spec.forProvider.subnetIdSelector.matchLabels[networks.aws.caas.upbound.io/network-id]
- fromFieldPath: spec.parameters.id
toFieldPath: spec.forProvider.subnetIdSelector.matchLabels[crossplane.io/claim-name]
- fromFieldPath: spec.parameters.id
toFieldPath: spec.forProvider.clusterNameSelector.matchLabels[crossplane.io/claim-name]
- fromFieldPath: spec.parameters.id
toFieldPath: spec.forProvider.nodeRoleArnSelector.matchLabels[crossplane.io/claim-name]
- type: ToCompositeFieldPath
fromFieldPath: status.atProvider.clusterName
toFieldPath: status.status.clusterName
policy:
fromFieldPath: Optional
- name: ingressChart
base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
spec:
forProvider:
chart:
name: ingress-nginx
repository: https://kubernetes.github.io/ingress-nginx
version: 4.1.3
namespace: ingress-nginx
values:
controller:
podAnnotations:
fluentbit.io/parser: k8s-nginx-ingress
service:
annotations:
service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
service.beta.kubernetes.io/aws-load-balancer-type: external
externalTrafficPolicy: Local
ports:
http: 80
https: 443
targetPorts:
http: http
https: http
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- name: nvidiaChart
base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
spec:
forProvider:
chart:
name: nvidia-device-plugin
repository: https://nvidia.github.io/k8s-device-plugin
version: 0.14.0
namespace: nvidia-device-plugin
values:
gfd:
enabled: true
nfd:
enabled: true
worker:
tolerations:
- effect: NoSchedule
key: nvidia.com/gpu
operator: Exists
- operator: Exists
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- name: secretNotebook
base:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
forProvider:
manifest:
apiVersion: v1
kind: Secret
metadata:
name: notebook
namespace: jupyterhub
references:
- patchesFrom:
apiVersion: v1
fieldPath: data[dogbooth.ipynb]
kind: Secret
toFieldPath: data[dogbooth.ipynb]
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- fromFieldPath: spec.parameters.jupyter[0].name
toFieldPath: spec.references[0].patchesFrom.name
- fromFieldPath: spec.parameters.jupyter[0].namespace
toFieldPath: spec.references[0].patchesFrom.namespace
- name: secretHfToken
base:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
forProvider:
manifest:
apiVersion: v1
kind: Secret
metadata:
name: hf-token
namespace: jupyterhub
references:
- patchesFrom:
apiVersion: v1
fieldPath: data.token
kind: Secret
toFieldPath: data.token
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- fromFieldPath: spec.parameters.jupyter[1].name
toFieldPath: spec.references[0].patchesFrom.name
- fromFieldPath: spec.parameters.jupyter[1].namespace
toFieldPath: spec.references[0].patchesFrom.namespace
- name: jupyterChart
base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
spec:
forProvider:
chart:
name: jupyterhub
repository: https://jupyterhub.github.io/helm-chart/
version: 3.0.3
namespace: jupyterhub
values:
hub:
config:
Authenticator:
admin_users:
- admin1
allowed_users:
- user1
DummyAuthenticator:
password: never-do-this
JupyterHub:
authenticator_class: dummy
proxy:
service:
annotations:
service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
service.beta.kubernetes.io/aws-load-balancer-ip-address-type: ipv4
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
service.beta.kubernetes.io/aws-load-balancer-type: external
scheduling:
userScheduler:
enabled: false
singleuser:
cmd: null
extraEnv:
HUGGING_FACE_HUB_TOKEN:
valueFrom:
secretKeyRef:
key: token
name: hf-token
extraResource:
limits:
nvidia.com/gpu: "1"
extraTolerations:
- effect: NoSchedule
key: nvidia.com/gpu
operator: Exists
image:
name: public.ecr.aws/h3o5n2r0/gpu-jupyter
pullPolicy: Always
tag: v1.5_cuda-11.6_ubuntu-20.04_python-only
memory:
guarantee: 24G
startTimeout: 600
storage:
capacity: 100Gi
extraVolumeMounts:
- mountPath: /dev/shm
name: shm-volume
- mountPath: /home/jovyan/dogbooth
name: notebook
extraVolumes:
- emptyDir:
medium: Memory
name: shm-volume
- name: notebook
secret:
items:
- key: dogbooth.ipynb
path: dogbooth.ipynb
secretName: notebook
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- name: kuberayChart
base:
apiVersion: helm.crossplane.io/v1beta1
kind: Release
spec:
forProvider:
chart:
name: kuberay-operator
repository: https://ray-project.github.io/kuberay-helm/
version: 0.6.0
namespace: kuberay-operator
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- name: namespaceDogbooth
base:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
forProvider:
manifest:
apiVersion: v1
kind: Namespace
metadata:
name: dogbooth
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- name: rayServiceDogbooth
base:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
forProvider:
manifest:
apiVersion: ray.io/v1alpha1
kind: RayService
metadata:
name: dogbooth
namespace: dogbooth
spec:
deploymentUnhealthySecondThreshold: 600
rayClusterConfig:
headGroupSpec:
rayStartParams:
dashboard-host: 0.0.0.0
template:
spec:
containers:
- image: public.ecr.aws/data-on-eks/dogbooth:0.0.1-gpu
name: ray-head
ports:
- containerPort: 6379
name: gcs-server
- containerPort: 8265
name: dashboard
- containerPort: 10001
name: client
- containerPort: 8000
name: serve
resources:
limits:
cpu: 2
memory: 16Gi
nvidia.com/gpu: 1
requests:
cpu: 2
memory: 16Gi
nvidia.com/gpu: 1
rayVersion: 2.6.0
workerGroupSpecs:
- groupName: small-group
maxReplicas: 5
minReplicas: 1
rayStartParams: {}
replicas: 1
template:
spec:
containers:
- image: public.ecr.aws/data-on-eks/dogbooth:0.0.1-gpu
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- ray stop
name: ray-worker
resources:
limits:
cpu: "2"
memory: 16Gi
nvidia.com/gpu: 1
requests:
cpu: "2"
memory: 16Gi
nvidia.com/gpu: 1
serveConfig:
importPath: dogbooth:entrypoint
runtimeEnv: |
env_vars: {"MODEL_ID": "askulkarni2/dogbooth"}
serviceUnhealthySecondThreshold: 600
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name
- name: ingressDogbooth
base:
apiVersion: kubernetes.crossplane.io/v1alpha1
kind: Object
spec:
forProvider:
manifest:
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /$1
name: dogbooth
namespace: dogbooth
spec:
ingressClassName: nginx
rules:
- http:
paths:
- backend:
service:
name: dogbooth-head-svc
port:
number: 8265
path: /dogbooth/(.*)
pathType: ImplementationSpecific
- backend:
service:
name: dogbooth-head-svc
port:
number: 8000
path: /dogbooth/serve/(.*)
pathType: ImplementationSpecific
patches:
- type: PatchSet
patchSetName: deletionPolicy
- fromFieldPath: spec.parameters.id
toFieldPath: spec.providerConfigRef.name