mkdir software \ && cd software \ && ARCH=amd64 PLATFORM=$(uname -s)_$ARCH && \ curl -sLO "https://github.com/eksctl-io/eksctl/releases/latest/download/eksctl_$PLATFORM.tar.gz" && \ tar -xzf eksctl_$PLATFORM.tar.gz -C /tmp && \ sudo mv /tmp/eksctl /usr/local/bin && \ rm eksctl_$PLATFORM.tar.gz && \ curl -o kubectl https://s3.us-west-2.amazonaws.com/amazon-eks/1.27.4/2023-08-16/bin/linux/amd64/kubectl && \ chmod +x kubectl && \ sudo mv kubectl /usr/local/bin && \ wget -q https://get.helm.sh/helm-v3.12.3-linux-amd64.tar.gz && \ tar -zxvf helm-v3.12.3-linux-amd64.tar.gz && \ sudo mv linux-amd64/helm /usr/local/bin/helm && \ rm -rf helm-v3.12.3-linux-amd64.tar.gz linux-amd64 && \ curl -L https://github.com/a8m/envsubst/releases/download/v1.2.0/envsubst-$(uname -s)-$(uname -m) -o envsubst && \ chmod +x envsubst && \ sudo mv envsubst /usr/local/bin && \ sudo yum install -y yum-utils && \ sudo yum-config-manager --add-repo https://rpm.releases.hashicorp.com/AmazonLinux/hashicorp.repo && \ sudo yum -y install terraform && \ sudo yum update -y && \ sudo amazon-linux-extras install -y docker \ && echo $(nohup sudo dockerd > /tmp/dockerd.log 2>&1 &);
export ECR_REPOSITORY=<ecrrepourl> export PRODUCT_VERSION=<version>
cluster.yaml
--- apiVersion: eksctl.io/v1alpha5 kind: ClusterConfig metadata: name: ${PRODUCT_NAME}-cluster region: ${AWS_REGION} version: "1.30" availabilityZones: ["${AWS_REGION}a", "${AWS_REGION}b"] fargateProfiles: - name: fp-default selectors: # All workloads in the "default" Kubernetes namespace will be # scheduled onto Fargate: - namespace: default # All workloads in the "kube-system" Kubernetes namespace will be # scheduled onto Fargate: - namespace: kube-system - namespace: mcp iam: withOIDC: true serviceAccounts: - metadata: name: ${PRODUCT_NAME}-sa namespace: mcp attachPolicy: Version: '2012-10-17' Statement: - Action: - "aws-marketplace:MeterUsage" - "aws-marketplace:RegisterUsage" - "license-manager:CheckoutLicense" - "license-manager:CheckInLicense" - "license-manager:ExtendLicenseConsumption" - "license-manager:GetLicense" Effect: Allow Resource: "*" - metadata: name: ${PRODUCT_NAME}-icsa namespace: kube-system attachPolicy: Version: '2012-10-17' Statement: - Effect: Allow Action: - acm:DescribeCertificate - acm:ListCertificates - acm:GetCertificate Resource: "*" - Effect: Allow Action: - ec2:AuthorizeSecurityGroupIngress - ec2:CreateSecurityGroup - ec2:CreateTags - ec2:DeleteTags - ec2:DeleteSecurityGroup - ec2:DescribeAccountAttributes - ec2:DescribeAddresses - ec2:DescribeInstances - ec2:DescribeInstanceStatus - ec2:DescribeInternetGateways - ec2:DescribeNetworkInterfaces - ec2:DescribeSecurityGroups - ec2:DescribeSubnets - ec2:DescribeTags - ec2:DescribeVpcs - ec2:ModifyInstanceAttribute - ec2:ModifyNetworkInterfaceAttribute - ec2:RevokeSecurityGroupIngress Resource: "*" - Effect: Allow Action: - elasticloadbalancing:AddListenerCertificates - elasticloadbalancing:AddTags - elasticloadbalancing:CreateListener - elasticloadbalancing:CreateLoadBalancer - elasticloadbalancing:CreateRule - elasticloadbalancing:CreateTargetGroup - elasticloadbalancing:DeleteListener - elasticloadbalancing:DeleteLoadBalancer - elasticloadbalancing:DeleteRule - elasticloadbalancing:DeleteTargetGroup - elasticloadbalancing:DeregisterTargets - elasticloadbalancing:DescribeListenerCertificates - elasticloadbalancing:DescribeListeners - elasticloadbalancing:DescribeLoadBalancers - elasticloadbalancing:DescribeLoadBalancerAttributes - elasticloadbalancing:DescribeRules - elasticloadbalancing:DescribeSSLPolicies - elasticloadbalancing:DescribeTags - elasticloadbalancing:DescribeTargetGroups - elasticloadbalancing:DescribeTargetGroupAttributes - elasticloadbalancing:DescribeTargetHealth - elasticloadbalancing:ModifyListener - elasticloadbalancing:ModifyLoadBalancerAttributes - elasticloadbalancing:ModifyRule - elasticloadbalancing:ModifyTargetGroup - elasticloadbalancing:ModifyTargetGroupAttributes - elasticloadbalancing:RegisterTargets - elasticloadbalancing:RemoveListenerCertificates - elasticloadbalancing:RemoveTags - elasticloadbalancing:SetIpAddressType - elasticloadbalancing:SetSecurityGroups - elasticloadbalancing:SetSubnets - elasticloadbalancing:SetWebACL Resource: "*" - Effect: Allow Action: - iam:CreateServiceLinkedRole - iam:GetServerCertificate - iam:ListServerCertificates Resource: "*" - Effect: Allow Action: - cognito-idp:DescribeUserPoolClient Resource: "*" - Effect: Allow Action: - waf-regional:GetWebACLForResource - waf-regional:GetWebACL - waf-regional:AssociateWebACL - waf-regional:DisassociateWebACL Resource: "*" - Effect: Allow Action: - tag:GetResources - tag:TagResources Resource: "*" - Effect: Allow Action: - waf:GetWebACL Resource: "*" - Effect: Allow Action: - shield:DescribeProtection - shield:GetSubscriptionState - shield:DeleteProtection - shield:CreateProtection - shield:DescribeSubscription - shield:ListProtections Resource: "*" cloudWatch: clusterLogging: enableTypes: ["*"] logRetentionInDays: 1
export AWS_REGION=<REGION> export PRODUCT_NAME=<PRODUCT_NAME> envsubst < cluster.yaml > $PRODUCT_NAME-cluster.yaml eksctl create cluster -f $PRODUCT_NAME-cluster.yaml
deployment.yaml
--- apiVersion: apps/v1 kind: Deployment metadata: name: llmstudio-proxy namespace: mcp labels: app: llmstudio-proxy spec: replicas: 1 selector: matchLabels: app: llmstudio-proxy template: metadata: labels: app: llmstudio-proxy spec: serviceAccountName: ${PRODUCT_NAME}-sa volumes: - name: shared-data emptyDir: {} containers: - name: llmstudio-proxy image: $ECR_REPOSITORY:$PRODUCT_VERSION ports: - containerPort: 50001 env: - name: OPENAI_API_KEY value: $OPENAI_API_KEY # Add here the keys for other providers as needed --- apiVersion: v1 kind: Service metadata: name: llmstudio-proxy namespace: mcp spec: type: NodePort selector: app: llmstudio-proxy ports: - protocol: TCP port: 50001 targetPort: 50001
export OPENAI_API_KEY="...."
envsubst < deployment.yaml > $PRODUCT_NAME-deployment.yaml
kubectl apply -f $PRODUCT_NAME-deployment.yaml
kubectl get pods -n mcp
Running
NAME READY STATUS RESTARTS AGE llmstudio-proxy 1/1 Running 0 79s
kubectl logs "$(kubectl get pods -n mcp --output name)" -c contract-app -n mcp
Running LLMstudio Proxy on http://0.0.0.0:50001