kind: Composition
apiVersion: apiextensions.crossplane.io/v1
metadata:
  name: xservices.ml.aws.caas.upbound.io
  creationTimestamp: null
spec:
  compositeTypeRef:
    apiVersion: ml.aws.caas.upbound.io/v1alpha1
    kind: XService
  patchSets:
    - name: providerConfigRef
      patches:
        - type: FromCompositeFieldPath
          fromFieldPath: spec.parameters.providerConfigName
          toFieldPath: spec.providerConfigRef.name
    - name: deletionPolicy
      patches:
        - type: FromCompositeFieldPath
          fromFieldPath: spec.parameters.deletionPolicy
          toFieldPath: spec.deletionPolicy
    - name: region
      patches:
        - type: FromCompositeFieldPath
          fromFieldPath: spec.parameters.region
          toFieldPath: spec.forProvider.region
  resources:
    - name: loadbalancerControllerIrsa
      base:
        apiVersion: aws.caas.upbound.io/v1alpha1
        kind: XIRSA
        spec:
          parameters:
            condition: StringEquals
            policyDocument: >
              {
                  "Version": "2012-10-17",
                  "Statement": [
                      {
                          "Effect": "Allow",
                          "Action": [
                              "iam:CreateServiceLinkedRole"
                          ],
                          "Resource": "*",
                          "Condition": {
                              "StringEquals": {
                                  "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com"
                              }
                          }
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "ec2:DescribeAccountAttributes",
                              "ec2:DescribeAddresses",
                              "ec2:DescribeAvailabilityZones",
                              "ec2:DescribeInternetGateways",
                              "ec2:DescribeVpcs",
                              "ec2:DescribeVpcPeeringConnections",
                              "ec2:DescribeSubnets",
                              "ec2:DescribeSecurityGroups",
                              "ec2:DescribeInstances",
                              "ec2:DescribeNetworkInterfaces",
                              "ec2:DescribeTags",
                              "ec2:GetCoipPoolUsage",
                              "ec2:DescribeCoipPools",
                              "elasticloadbalancing:DescribeLoadBalancers",
                              "elasticloadbalancing:DescribeLoadBalancerAttributes",
                              "elasticloadbalancing:DescribeListeners",
                              "elasticloadbalancing:DescribeListenerCertificates",
                              "elasticloadbalancing:DescribeSSLPolicies",
                              "elasticloadbalancing:DescribeRules",
                              "elasticloadbalancing:DescribeTargetGroups",
                              "elasticloadbalancing:DescribeTargetGroupAttributes",
                              "elasticloadbalancing:DescribeTargetHealth",
                              "elasticloadbalancing:DescribeTags"
                          ],
                          "Resource": "*"
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "cognito-idp:DescribeUserPoolClient",
                              "acm:ListCertificates",
                              "acm:DescribeCertificate",
                              "iam:ListServerCertificates",
                              "iam:GetServerCertificate",
                              "waf-regional:GetWebACL",
                              "waf-regional:GetWebACLForResource",
                              "waf-regional:AssociateWebACL",
                              "waf-regional:DisassociateWebACL",
                              "wafv2:GetWebACL",
                              "wafv2:GetWebACLForResource",
                              "wafv2:AssociateWebACL",
                              "wafv2:DisassociateWebACL",
                              "shield:GetSubscriptionState",
                              "shield:DescribeProtection",
                              "shield:CreateProtection",
                              "shield:DeleteProtection"
                          ],
                          "Resource": "*"
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "ec2:AuthorizeSecurityGroupIngress",
                              "ec2:RevokeSecurityGroupIngress"
                          ],
                          "Resource": "*"
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "ec2:CreateSecurityGroup"
                          ],
                          "Resource": "*"
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "ec2:CreateTags"
                          ],
                          "Resource": "arn:aws:ec2:*:*:security-group/*",
                          "Condition": {
                              "StringEquals": {
                                  "ec2:CreateAction": "CreateSecurityGroup"
                              },
                              "Null": {
                                  "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
                              }
                          }
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "ec2:CreateTags",
                              "ec2:DeleteTags"
                          ],
                          "Resource": "arn:aws:ec2:*:*:security-group/*",
                          "Condition": {
                              "Null": {
                                  "aws:RequestTag/elbv2.k8s.aws/cluster": "true",
                                  "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
                              }
                          }
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "ec2:AuthorizeSecurityGroupIngress",
                              "ec2:RevokeSecurityGroupIngress",
                              "ec2:DeleteSecurityGroup"
                          ],
                          "Resource": "*",
                          "Condition": {
                              "Null": {
                                  "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
                              }
                          }
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "elasticloadbalancing:CreateLoadBalancer",
                              "elasticloadbalancing:CreateTargetGroup"
                          ],
                          "Resource": "*",
                          "Condition": {
                              "Null": {
                                  "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
                              }
                          }
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "elasticloadbalancing:CreateListener",
                              "elasticloadbalancing:DeleteListener",
                              "elasticloadbalancing:CreateRule",
                              "elasticloadbalancing:DeleteRule"
                          ],
                          "Resource": "*"
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "elasticloadbalancing:AddTags",
                              "elasticloadbalancing:RemoveTags"
                          ],
                          "Resource": [
                              "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
                              "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
                              "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
                          ],
                          "Condition": {
                              "Null": {
                                  "aws:RequestTag/elbv2.k8s.aws/cluster": "true",
                                  "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
                              }
                          }
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "elasticloadbalancing:AddTags",
                              "elasticloadbalancing:RemoveTags"
                          ],
                          "Resource": [
                              "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*",
                              "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*",
                              "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*",
                              "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*"
                          ]
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "elasticloadbalancing:ModifyLoadBalancerAttributes",
                              "elasticloadbalancing:SetIpAddressType",
                              "elasticloadbalancing:SetSecurityGroups",
                              "elasticloadbalancing:SetSubnets",
                              "elasticloadbalancing:DeleteLoadBalancer",
                              "elasticloadbalancing:ModifyTargetGroup",
                              "elasticloadbalancing:ModifyTargetGroupAttributes",
                              "elasticloadbalancing:DeleteTargetGroup"
                          ],
                          "Resource": "*",
                          "Condition": {
                              "Null": {
                                  "aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
                              }
                          }
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "elasticloadbalancing:AddTags"
                          ],
                          "Resource": [
                              "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
                              "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
                              "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
                          ],
                          "Condition": {
                              "StringEquals": {
                                  "elasticloadbalancing:CreateAction": [
                                      "CreateTargetGroup",
                                      "CreateLoadBalancer"
                                  ]
                              },
                              "Null": {
                                  "aws:RequestTag/elbv2.k8s.aws/cluster": "false"
                              }
                          }
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "elasticloadbalancing:RegisterTargets",
                              "elasticloadbalancing:DeregisterTargets"
                          ],
                          "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*"
                      },
                      {
                          "Effect": "Allow",
                          "Action": [
                              "elasticloadbalancing:SetWebAcl",
                              "elasticloadbalancing:ModifyListener",
                              "elasticloadbalancing:AddListenerCertificates",
                              "elasticloadbalancing:RemoveListenerCertificates",
                              "elasticloadbalancing:ModifyRule"
                          ],
                          "Resource": "*"
                      }
                  ]
              }
            serviceAccount:
              name: aws-load-balancer-controller
              namespace: kube-system
      patches:
        - fromFieldPath: spec.parameters.providerConfigName
          toFieldPath: spec.parameters.providerConfigName
        - fromFieldPath: spec.parameters.deletionPolicy
          toFieldPath: spec.parameters.deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: metadata.name
          transforms:
            - type: string
              string:
                fmt: "%s-loadbalancer-controller"
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.parameters.id
        - type: ToCompositeFieldPath
          fromFieldPath: status.roleArn
          toFieldPath: status.status.loadbalancerControllerRoleArn
          policy:
            fromFieldPath: Optional
    - name: loadbalancerControllerChart
      base:
        apiVersion: helm.crossplane.io/v1beta1
        kind: Release
        spec:
          forProvider:
            chart:
              name: aws-load-balancer-controller
              repository: https://aws.github.io/eks-charts
              version: 1.4.4
            namespace: kube-system
            values:
              image:
                repository: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon/aws-load-balancer-controller
              serviceAccount:
                name: aws-load-balancer-controller
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
        - fromFieldPath: status.status.clusterName
          toFieldPath: spec.forProvider.values.clusterName
          policy:
            fromFieldPath: Required
        - fromFieldPath: status.status.loadbalancerControllerRoleArn
          toFieldPath: spec.forProvider.values.serviceAccount.annotations[eks.amazonaws.com/role-arn]
          policy:
            fromFieldPath: Required
    - name: gp2NoDefault
      base:
        apiVersion: kubernetes.crossplane.io/v1alpha1
        kind: Object
        spec:
          forProvider:
            manifest:
              apiVersion: storage.k8s.io/v1
              kind: StorageClass
              metadata:
                annotations:
                  storageclass.kubernetes.io/is-default-class: "false"
                name: gp2
              parameters:
                fsType: ext4
                type: gp2
              provisioner: kubernetes.io/aws-ebs
              reclaimPolicy: Delete
              volumeBindingMode: WaitForFirstConsumer
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
    - name: gp3AsDefault
      base:
        apiVersion: kubernetes.crossplane.io/v1alpha1
        kind: Object
        spec:
          forProvider:
            manifest:
              apiVersion: storage.k8s.io/v1
              kind: StorageClass
              metadata:
                annotations:
                  storageclass.kubernetes.io/is-default-class: "true"
                name: gp3
              parameters:
                csi.storage.k8s.io/fstype: ext4
                encrypted: "true"
                type: gp3
              provisioner: ebs.csi.aws.com
              reclaimPolicy: Delete
              volumeBindingMode: WaitForFirstConsumer
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
    - name: eksNodeGroupGPU
      base:
        apiVersion: eks.aws.upbound.io/v1beta1
        kind: NodeGroup
        spec:
          forProvider:
            amiType: AL2_x86_64_GPU
            diskSize: 100
            instanceTypes:
              - g5.12xlarge
            labels:
              NodeGroupType: gpu
              WorkerType: ON_DEMAND
            nodeRoleArnSelector:
              matchLabels:
                role: nodegroup
            scalingConfig:
              - desiredSize: 1
                maxSize: 100
                minSize: 1
            subnetIdSelector:
              matchLabels:
                access: public
            taint:
              - effect: NO_SCHEDULE
                key: nvidia.com/gpu
                value: EXISTS
      patches:
        - type: PatchSet
          patchSetName: providerConfigRef
        - type: PatchSet
          patchSetName: deletionPolicy
        - type: PatchSet
          patchSetName: region
        - fromFieldPath: spec.parameters.nodes.count
          toFieldPath: spec.forProvider.scalingConfig[0].desiredSize
        - fromFieldPath: spec.parameters.nodes.instanceType
          toFieldPath: spec.forProvider.instanceTypes[0]
        - fromFieldPath: spec.parameters.nodes.diskSize
          toFieldPath: spec.forProvider.diskSize
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.forProvider.subnetIdSelector.matchLabels[networks.aws.caas.upbound.io/network-id]
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.forProvider.subnetIdSelector.matchLabels[crossplane.io/claim-name]
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.forProvider.clusterNameSelector.matchLabels[crossplane.io/claim-name]
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.forProvider.nodeRoleArnSelector.matchLabels[crossplane.io/claim-name]
        - type: ToCompositeFieldPath
          fromFieldPath: status.atProvider.clusterName
          toFieldPath: status.status.clusterName
          policy:
            fromFieldPath: Optional
    - name: ingressChart
      base:
        apiVersion: helm.crossplane.io/v1beta1
        kind: Release
        spec:
          forProvider:
            chart:
              name: ingress-nginx
              repository: https://kubernetes.github.io/ingress-nginx
              version: 4.1.3
            namespace: ingress-nginx
            values:
              controller:
                podAnnotations:
                  fluentbit.io/parser: k8s-nginx-ingress
                service:
                  annotations:
                    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
                    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
                    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
                    service.beta.kubernetes.io/aws-load-balancer-type: external
                  externalTrafficPolicy: Local
                  ports:
                    http: 80
                    https: 443
                  targetPorts:
                    http: http
                    https: http
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
    - name: nvidiaChart
      base:
        apiVersion: helm.crossplane.io/v1beta1
        kind: Release
        spec:
          forProvider:
            chart:
              name: nvidia-device-plugin
              repository: https://nvidia.github.io/k8s-device-plugin
              version: 0.14.0
            namespace: nvidia-device-plugin
            values:
              gfd:
                enabled: true
              nfd:
                enabled: true
                worker:
                  tolerations:
                    - effect: NoSchedule
                      key: nvidia.com/gpu
                      operator: Exists
                    - operator: Exists
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
    - name: secretNotebook
      base:
        apiVersion: kubernetes.crossplane.io/v1alpha1
        kind: Object
        spec:
          forProvider:
            manifest:
              apiVersion: v1
              kind: Secret
              metadata:
                name: notebook
                namespace: jupyterhub
          references:
            - patchesFrom:
                apiVersion: v1
                fieldPath: data[dogbooth.ipynb]
                kind: Secret
              toFieldPath: data[dogbooth.ipynb]
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
        - fromFieldPath: spec.parameters.jupyter[0].name
          toFieldPath: spec.references[0].patchesFrom.name
        - fromFieldPath: spec.parameters.jupyter[0].namespace
          toFieldPath: spec.references[0].patchesFrom.namespace
    - name: secretHfToken
      base:
        apiVersion: kubernetes.crossplane.io/v1alpha1
        kind: Object
        spec:
          forProvider:
            manifest:
              apiVersion: v1
              kind: Secret
              metadata:
                name: hf-token
                namespace: jupyterhub
          references:
            - patchesFrom:
                apiVersion: v1
                fieldPath: data.token
                kind: Secret
              toFieldPath: data.token
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
        - fromFieldPath: spec.parameters.jupyter[1].name
          toFieldPath: spec.references[0].patchesFrom.name
        - fromFieldPath: spec.parameters.jupyter[1].namespace
          toFieldPath: spec.references[0].patchesFrom.namespace
    - name: jupyterChart
      base:
        apiVersion: helm.crossplane.io/v1beta1
        kind: Release
        spec:
          forProvider:
            chart:
              name: jupyterhub
              repository: https://jupyterhub.github.io/helm-chart/
              version: 3.0.3
            namespace: jupyterhub
            values:
              hub:
                config:
                  Authenticator:
                    admin_users:
                      - admin1
                    allowed_users:
                      - user1
                  DummyAuthenticator:
                    password: never-do-this
                  JupyterHub:
                    authenticator_class: dummy
              proxy:
                service:
                  annotations:
                    service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
                    service.beta.kubernetes.io/aws-load-balancer-ip-address-type: ipv4
                    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
                    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
                    service.beta.kubernetes.io/aws-load-balancer-type: external
              scheduling:
                userScheduler:
                  enabled: false
              singleuser:
                cmd: null
                extraEnv:
                  HUGGING_FACE_HUB_TOKEN:
                    valueFrom:
                      secretKeyRef:
                        key: token
                        name: hf-token
                extraResource:
                  limits:
                    nvidia.com/gpu: "1"
                extraTolerations:
                  - effect: NoSchedule
                    key: nvidia.com/gpu
                    operator: Exists
                image:
                  name: public.ecr.aws/h3o5n2r0/gpu-jupyter
                  pullPolicy: Always
                  tag: v1.5_cuda-11.6_ubuntu-20.04_python-only
                memory:
                  guarantee: 24G
                startTimeout: 600
                storage:
                  capacity: 100Gi
                  extraVolumeMounts:
                    - mountPath: /dev/shm
                      name: shm-volume
                    - mountPath: /home/jovyan/dogbooth
                      name: notebook
                  extraVolumes:
                    - emptyDir:
                        medium: Memory
                      name: shm-volume
                    - name: notebook
                      secret:
                        items:
                          - key: dogbooth.ipynb
                            path: dogbooth.ipynb
                        secretName: notebook
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
    - name: kuberayChart
      base:
        apiVersion: helm.crossplane.io/v1beta1
        kind: Release
        spec:
          forProvider:
            chart:
              name: kuberay-operator
              repository: https://ray-project.github.io/kuberay-helm/
              version: 0.6.0
            namespace: kuberay-operator
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
    - name: namespaceDogbooth
      base:
        apiVersion: kubernetes.crossplane.io/v1alpha1
        kind: Object
        spec:
          forProvider:
            manifest:
              apiVersion: v1
              kind: Namespace
              metadata:
                name: dogbooth
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
    - name: rayServiceDogbooth
      base:
        apiVersion: kubernetes.crossplane.io/v1alpha1
        kind: Object
        spec:
          forProvider:
            manifest:
              apiVersion: ray.io/v1alpha1
              kind: RayService
              metadata:
                name: dogbooth
                namespace: dogbooth
              spec:
                deploymentUnhealthySecondThreshold: 600
                rayClusterConfig:
                  headGroupSpec:
                    rayStartParams:
                      dashboard-host: 0.0.0.0
                    template:
                      spec:
                        containers:
                          - image: public.ecr.aws/data-on-eks/dogbooth:0.0.1-gpu
                            name: ray-head
                            ports:
                              - containerPort: 6379
                                name: gcs-server
                              - containerPort: 8265
                                name: dashboard
                              - containerPort: 10001
                                name: client
                              - containerPort: 8000
                                name: serve
                            resources:
                              limits:
                                cpu: 2
                                memory: 16Gi
                                nvidia.com/gpu: 1
                              requests:
                                cpu: 2
                                memory: 16Gi
                                nvidia.com/gpu: 1
                  rayVersion: 2.6.0
                  workerGroupSpecs:
                    - groupName: small-group
                      maxReplicas: 5
                      minReplicas: 1
                      rayStartParams: {}
                      replicas: 1
                      template:
                        spec:
                          containers:
                            - image: public.ecr.aws/data-on-eks/dogbooth:0.0.1-gpu
                              lifecycle:
                                preStop:
                                  exec:
                                    command:
                                      - /bin/sh
                                      - -c
                                      - ray stop
                              name: ray-worker
                              resources:
                                limits:
                                  cpu: "2"
                                  memory: 16Gi
                                  nvidia.com/gpu: 1
                                requests:
                                  cpu: "2"
                                  memory: 16Gi
                                  nvidia.com/gpu: 1
                serveConfig:
                  importPath: dogbooth:entrypoint
                  runtimeEnv: |
                    env_vars: {"MODEL_ID": "askulkarni2/dogbooth"}
                serviceUnhealthySecondThreshold: 600
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name
    - name: ingressDogbooth
      base:
        apiVersion: kubernetes.crossplane.io/v1alpha1
        kind: Object
        spec:
          forProvider:
            manifest:
              apiVersion: networking.k8s.io/v1
              kind: Ingress
              metadata:
                annotations:
                  nginx.ingress.kubernetes.io/rewrite-target: /$1
                name: dogbooth
                namespace: dogbooth
              spec:
                ingressClassName: nginx
                rules:
                  - http:
                      paths:
                        - backend:
                            service:
                              name: dogbooth-head-svc
                              port:
                                number: 8265
                          path: /dogbooth/(.*)
                          pathType: ImplementationSpecific
                        - backend:
                            service:
                              name: dogbooth-head-svc
                              port:
                                number: 8000
                          path: /dogbooth/serve/(.*)
                          pathType: ImplementationSpecific
      patches:
        - type: PatchSet
          patchSetName: deletionPolicy
        - fromFieldPath: spec.parameters.id
          toFieldPath: spec.providerConfigRef.name