### Quick Start: CFKE Cluster and OVH Node Provisioning with Terraform

Source: https://cloudfleet.ai/docs/terraform/introduction

A comprehensive Terraform example that provisions a Cloudfleet CFKE cluster and configures self-managed nodes on OVH. It includes provider configurations for both Cloudfleet and OVH, defines necessary variables, creates an SSH key, and sets up cluster and node join information.

```Terraform
terraform {
    required_providers {
        cloudfleet = {
            source = "terraform.cloudfleet.ai/cloudfleet/cloudfleet"
        }
        ovh = {
            source = "ovh/ovh"
        }
    }
}

variable "ovh_application_key" {
    description = "OVH application key for API access"
    type        = string
}
variable "ovh_application_secret" {
    description = "OVH application secret for API access"
    type        = string
    sensitive   = true
}
variable "ovh_consumer_key" {
    description = "OVH consumer key for API access"
    type        = string
}

# Create an SSH key pair for OVH instances
variable "ovh_service_name" {
    description = "OVH service name for the cloud project"
    type        = string
}

provider "cloudfleet" {}

# Create a CFKE cluster
resource "cloudfleet_cfke_cluster" "example" {
    name   = "my-cluster"
    region = "europe-central-1a"
    tier   = "basic"
}

# Generate cloud-init user-data to be used in self-managed nodes
resource "cloudfleet_cfke_node_join_information" "ovh" {
    cluster_id = cloudfleet_cfke_cluster.example.id
    region     = "DE1"
    zone       = "DE1"

    node_labels = {
        "cfke.io/provider" = "ovh"
    }
}

provider "ovh" {
    endpoint           = "ovh-eu"
    application_key    = var.ovh_application_key
    application_secret = var.ovh_application_secret
    consumer_key       = var.ovh_consumer_key
}

data "ovh_cloud_project_flavors" "flavor" {
    service_name = var.ovh_service_name
    name_filter  = "b2-7"
}

data "ovh_cloud_project_images" "images" {
    service_name = var.ovh_service_name
    region       = "DE1"
    os_type      = "linux"
}

locals {
    flavor_id = [for flavor in data.ovh_cloud_project_flavors.flavor.flavors : flavor.id if flavor.region == "DE1"][0]
    image_id  = [for image in data.ovh_cloud_project_images.images.images : image.id if image.name == "Ubuntu 24.04"][0]
}

resource "tls_private_key" "ssh_key" {
    algorithm = "ED25519"
}

resource "ovh_cloud_project_ssh_key" "cfke" {
    service_name = var.ovh_service_name
    name         = "cfke-test"
    public_key   = tls_private_key.ssh_key.public_key_openssh
}
```

--------------------------------

### Deploy GPU Workload with Resource Requests and Limits

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

This example shows how to deploy a Kubernetes Job that requires NVIDIA GPUs. CFKE handles the provisioning of GPU-enabled nodes and driver installation. The configuration specifies node selectors for the GPU type and cloud provider, and requests a specific number of GPUs in the container resources.

```yaml
# gpu-training-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: ml-training
spec:
  template:
    spec:
      nodeSelector:
        cfke.io/accelerator-name: L4
        cfke.io/provider: aws
        topology.kubernetes.io/region: us-west-2
      containers:
      - name: pytorch-training
        image: pytorch/pytorch:2.0.1-cuda11.8-runtime
        command: ["python", "train.py"]
        resources:
          requests:
            memory: "16Gi"
            cpu: "8"
          limits:
            nvidia.com/gpu: "1"
      restartPolicy: Never
```

```bash
kubectl apply -f gpu-training-job.yaml

# Monitor job status
kubectl get jobs
# NAME          COMPLETIONS   DURATION   AGE
# ml-training   0/1           45s        45s

# Check node with GPU
kubectl get nodes -o custom-columns=NAME:.metadata.name,GPU:.status.capacity.'nvidia\.com/gpu'
# NAME                          GPU
# cfke-aws-us-west-2a-gpu1      1

# View job logs
kubectl logs job/ml-training
# Loading model...
# Training started with GPU: Tesla L4
# Epoch 1/10: loss=0.532
```

--------------------------------

### Install and Verify Kubernetes Nodes and GPU Servers

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

Commands to install Kubernetes components, configure WireGuard VPN, verify node status, add GPU servers with NVIDIA driver installation, and check GPU capacity. It uses `kubectl` for verification and `cloudfleet` CLI for node management.

```bash
# Installing Kubernetes components...
# Configuring WireGuard VPN...
# Node successfully joined cluster

# Verify node
kubectl get nodes
# NAME                        STATUS   ROLES    AGE   VERSION
# self-managed-datacenter-1   Ready    <none>   30s   v1.28.3

# Add GPU server with NVIDIA driver installation
cloudfleet clusters add-self-managed-node cls_abc123def456 \
  --host 10.0.5.20 \
  --ssh-username ubuntu \
  --region gpu-lab \
  --zone server-rack-1 \
  --install-nvidia-drivers

# Verify GPU capacity
kubectl get node self-managed-gpu-lab -o jsonpath='{.status.capacity.nvidia\.com/gpu}'
# 4
```

--------------------------------

### Verify NVIDIA GPU Node Labeling

Source: https://cloudfleet.ai/docs/hybrid-and-on-premises/self-managed-nodes

This command retrieves node information and displays their names and capacities. It is used to verify that nodes with NVIDIA GPUs are correctly labeled with `cfke.io/accelerator-manufacturer: NVIDIA` and that the `nvidia.com/gpu` capacity field is updated.

```bash
kubectl get node -o custom-columns=NAME:.metadata.name,CAPACITY:.status.capacity

```

--------------------------------

### Create Kubernetes Deployment for Nginx

Source: https://cloudfleet.ai/docs/introduction/getting-started

This command creates a Kubernetes Deployment named 'nginx-deployment' using the latest Nginx image and specifies that two replicas of the application should be run. This ensures basic availability and demonstrates workload distribution.

```bash
kubectl create deployment nginx-deployment --image=nginx:latest --replicas=2

```

--------------------------------

### Install Cloudfleet CLI using Winget (Windows)

Source: https://cloudfleet.ai/docs/introduction/install-cloudfleet-cli

Installs the Cloudfleet CLI on Windows using the Winget package manager. This command supports both AMD64 and ARM architectures. Ensure Winget is available on your system.

```bash
winget install Cloudfleet.CLI
```

--------------------------------

### Verify Nodes in Kubernetes Cluster

Source: https://cloudfleet.ai/docs/hybrid-and-on-premises/self-managed-nodes

Command to list all nodes currently registered in a Kubernetes cluster. This is used to verify that a self-managed node has been successfully added and is recognized by the cluster.

```bash
kubectl get nodes
```

--------------------------------

### Node Selector Example for AWS Zone

Source: https://cloudfleet.ai/docs/workload-management/node-provisioner

This example demonstrates how to use a `nodeSelector` in Kubernetes to schedule a pod to a node in a specific AWS zone. It utilizes the `cfke.io/provider` and `topology.kubernetes.io/zone` labels to specify AWS as the provider and 'us-west-2' as the desired zone.

```yaml
nodeSelector:
  cfke.io/provider: aws
  topology.kubernetes.io/zone: us-west-2

```

--------------------------------

### Install Cloudfleet CLI using RPM (Red Hat/Fedora)

Source: https://cloudfleet.ai/docs/introduction/install-cloudfleet-cli

Installs the Cloudfleet CLI on Red Hat-based Linux distributions using an RPM package. Download the appropriate RPM file (x64 or ARM) and then use this command to install it. Requires sudo privileges.

```bash
sudo rpm -i cloudfleet.rpm
```

--------------------------------

### Install Cloudfleet CLI using APT (Debian/Ubuntu)

Source: https://cloudfleet.ai/docs/introduction/install-cloudfleet-cli

Installs the Cloudfleet CLI on Debian-based Linux distributions using the APT package manager. This involves adding the Cloudfleet APT repository and then installing the package. Requires sudo privileges.

```bash
curl -fsSL https://downloads.cloudfleet.ai/apt/pubkey.gpg | sudo tee /usr/share/keyrings/cloudfleet-archive-keyring.gpg >/dev/null
echo "deb [signed-by=/usr/share/keyrings/cloudfleet-archive-keyring.gpg] https://downloads.cloudfleet.ai/apt stable main" | sudo tee /etc/apt/sources.list.d/cloudfleet.list
sudo apt-get update
sudo apt-get install cloudfleet
```

--------------------------------

### Deploy OVH Infrastructure with Terraform CLI

Source: https://cloudfleet.ai/docs/hybrid-and-on-premises/ovh

These commands initialize the Terraform configuration and apply the defined infrastructure to provision OVH instances and connect them to the CFKE cluster. Ensure Terraform CLI is installed and configured with necessary OVH credentials.

```bash
terraform init
terraform apply

```

--------------------------------

### Requesting GPU Resources for a Pod in Kubernetes

Source: https://cloudfleet.ai/docs/workload-management/node-provisioner

This example shows how to specify a request for accelerator resources, specifically an NVIDIA GPU, within a Kubernetes Pod manifest. This ensures that the pod is scheduled onto nodes equipped with the necessary hardware. CFKE will automatically install NVIDIA drivers if a GPU-enabled instance is provisioned.

```yaml
spec:
  template:
    spec:
      containers:
      - resources:
          limits:
            nvidia.com/gpu: "1"
```

--------------------------------

### Install Cloudfleet CLI using Homebrew (macOS)

Source: https://cloudfleet.ai/docs/introduction/install-cloudfleet-cli

Installs the Cloudfleet CLI on macOS using the Homebrew package manager. Ensure Homebrew is installed before running this command. This is the recommended method for macOS users.

```bash
brew install cloudfleetai/tap/cloudfleet-cli
```

--------------------------------

### List GCP VPC Networks using gcloud CLI

Source: https://cloudfleet.ai/docs/cloud-infrastructure/fleet-configuration

Checks the existing VPC networks in your GCP project. Ensure your project has a 'default' VPC network with automatic subnet creation mode for CFKE node provisioning. This command is useful for initial validation of network configuration.

```bash
gcloud compute networks list --project=YOUR_PROJECT_ID
```

--------------------------------

### Add Self-Managed Node to Cloudfleet Cluster

Source: https://cloudfleet.ai/docs/hybrid-and-on-premises/self-managed-nodes

Command to add a self-managed node to a Cloudfleet cluster. Requires cluster ID, host IP, SSH credentials, and region/zone information. Optional flags for username, key, and port can be omitted if defaults are used. The CLI handles package installation and node registration.

```bash
cloudfleet clusters add-self-managed-node CLUSTER_ID --host HOST_IP --ssh-username SSH_USERNAME --ssh-key SSH_KEY_LOCATION --ssh-port SSH_PORT --region DATACENTER_REGION --zone DATACENTER_ZONE
```

--------------------------------

### Terraform Configuration for Cloudfleet AI Cluster and Fleet

Source: https://cloudfleet.ai/docs/cloud-infrastructure/fleet-configuration

This Terraform configuration defines the necessary providers, variables, and resources to create a Cloudfleet AI CFKE cluster and a multi-cloud Fleet. It includes specific configurations for AWS, GCP, and Hetzner Cloud, setting up IAM roles, project bindings, and cluster details. Ensure you have the Cloudfleet Terraform provider and other required providers (AWS, Google) configured.

```terraform
terraform {
    required_providers {
        cloudfleet = {
            source = "terraform.cloudfleet.ai/cloudfleet/cloudfleet"
        }
    }
}

variable "cfke_control_plane_region" {
    description = "CFKE control plane region where the cluster is deployed"
    type        = string
    default     = "europe-central-1a"
}

variable "gcp_project" {
    type        = string
    description = "GCP project ID where CFKE nodes will be provisioned"
}

variable "hetzner_api_key" {
    description = "API key for Hetzner Cloud"
    type        = string
    sensitive   = true
}

variable "aws_region" {
    description = "AWS region where CFKE nodes will be provisioned"
    type        = string
    default     = "eu-central-1"
}

variable "aws_profile" {
    description = "AWS profile to use authenticate with AWS"
    type        = string
    default     = "default"
}

variable "hetzner_api_key" {
    description = "API key for Hetzner Cloud"
    type        = string
    sensitive   = true
}

provider "aws" {
    region  = var.aws_region
    profile = var.aws_profile
}

provider "cloudfleet" {
    profile = "default"
}

resource "cloudfleet_cfke_cluster" "cfke_test" {
    name   = "cfke-test"
    region = var.cfke_control_plane_region
    tier   = "basic"
}

resource "google_project_iam_custom_role" "cfke_node_autoprovisioner" {
    project = var.gcp_project
    permissions = [
        "compute.instances.create",
        "compute.instances.delete",
        "compute.instances.get",
        "compute.instances.list",
        "compute.disks.create",
        "compute.subnetworks.use",
        "compute.subnetworks.useExternalIp",
        "compute.instances.setMetadata",
        "compute.instances.setTags",
        "compute.instances.setLabels"
    ]
    role_id = "cfke.nodeAutoprovisioner"
    title   = "CFKE Node-autoprovisioner"
}

resource "google_project_iam_binding" "gcp_project_binding" {
    project = var.gcp_project
    role    = google_project_iam_custom_role.cfke_node_autoprovisioner.id
    members = [
        "principal://iam.googleapis.com/projects/207152264238/locations/global/workloadIdentityPools/cfke/subject/${cloudfleet_cfke_cluster.hetzner_test.id}"
    ]
}

module "cfke_connected_fleet" {
    source               = "registry.terraform.io/cloudfleetai/cfke-connected-fleet/aws"
    version              = "~> 0.1.0"
    control_plane_region = cloudfleet_cfke_cluster.cfke_test.region
    cluster_id           = cloudfleet_cfke_cluster.cfke_test.id
}

resource "cloudfleet_cfke_fleet" "fleet" {

    depends_on = [
        google_project_iam_binding.gcp_project_binding
    ]

    cluster_id = cloudfleet_cfke_cluster.cfke_test.id
    name       = "cfke-multi-cloud-fleet"

    limits {
        cpu = 24
    }

    hetzner {
        api_key = var.hetzner_api_key
    }

    aws {
        role_arn = module.cfke_connected_fleet.fleet_arn
    }

    gcp {
        project_id = var.gcp_project
    }
}

```

--------------------------------

### Schedule Workload to Specific Cloud Provider using Node Selectors

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

This example demonstrates how to deploy a Kubernetes Deployment that is scheduled to a specific cloud provider and region using node selectors. It requires a Deployment YAML file and the kubectl apply command.

```yaml
# locked-to-gcp-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: homepage
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      nodeSelector:
        cfke.io/provider: gcp
        topology.kubernetes.io/region: europe-west4
      containers:
      - name: nginx
        image: nginx:1.24
        resources:
          requests:
            cpu: 50m
            memory: 56Mi
        ports:
        - containerPort: 8080
```

```bash
kubectl apply -f locked-to-gcp-deployment.yaml

# Verify pods are scheduled on GCP nodes
kubectl get pods -o wide
# NAME                        READY   STATUS    NODE                              REGION
# homepage-abc123-xyz789      1/1     Running   cfke-gcp-europe-west4-a-node1    europe-west4
# homepage-abc123-pqr456      1/1     Running   cfke-gcp-europe-west4-b-node1    europe-west4
# homepage-abc123-stu789      1/1     Running   cfke-gcp-europe-west4-c-node1    europe-west4
```

--------------------------------

### Authenticate Cloudfleet CLI with User Profile

Source: https://cloudfleet.ai/docs/introduction/install-cloudfleet-cli

Adds a user profile for CLI authentication. This method requires browser interaction for authentication and is suitable for interactive use. Replace the placeholder UUID with your actual value. Requires sudo privileges if modifying system-wide configurations.

```bash
cloudfleet auth add-profile user default dc78c04e-6651-4e5d-9c04-079f6532989b
```

--------------------------------

### Check Kubernetes Deployment Status

Source: https://cloudfleet.ai/docs/introduction/getting-started

This command retrieves the status of all Kubernetes Deployments in the cluster, including the 'nginx-deployment'. It shows the desired number of replicas versus the number that are currently ready and available.

```bash
kubectl get deployments

```

--------------------------------

### List GCP Organization Policies using gcloud CLI

Source: https://cloudfleet.ai/docs/cloud-infrastructure/fleet-configuration

Lists the organization policies for your GCP project. This is important for checking restrictions on public IP addresses, specifically the 'constraints/compute.vmExternalIpAccess' policy, which must not be set to 'DENY' for CFKE nodes.

```bash
gcloud org-policies list --project=YOUR_PROJECT_ID
```

--------------------------------

### Authenticate Cloudfleet CLI with Token Profile

Source: https://cloudfleet.ai/docs/introduction/install-cloudfleet-cli

Adds a token-based profile for CLI authentication, suitable for programmatic access. Replace `ORGANIZATION_ID`, `TOKEN_ID`, and `TOKEN_SECRET` with your actual credentials. Requires sudo privileges if modifying system-wide configurations.

```bash
cloudfleet auth add-profile token default ORGANIZATION_ID TOKEN_ID TOKEN_SECRET
```

--------------------------------

### Create GCP Default VPC Network using gcloud CLI

Source: https://cloudfleet.ai/docs/cloud-infrastructure/fleet-configuration

Creates a new 'default' VPC network with automatic subnet mode if one does not exist. This network configuration is a prerequisite for provisioning CFKE nodes in your GCP project. It ensures subnets are automatically created in all regions.

```bash
gcloud compute networks create default \
--subnet-mode=auto \
--project=YOUR_PROJECT_ID
```

--------------------------------

### Configure kubectl to connect to a CFKE cluster

Source: https://cloudfleet.ai/docs/introduction/getting-started

Updates the local kubeconfig file to enable communication with a specific Cloudfleet Kubernetes Engine (CFKE) cluster. Replace YOUR_CLUSTER_ID with the actual ID of your cluster.

```bash
cloudfleet clusters kubeconfig YOUR_CLUSTER_ID
```

--------------------------------

### Kubernetes Topology Spread Constraints Example

Source: https://cloudfleet.ai/docs/workload-management/node-provisioner

Defines how pods should be spread across topology domains like zones and hostnames. This configuration helps minimize the impact of node failures by distributing pods. `maxSkew` controls the maximum difference in pod count between eligible domains, and `whenUnsatisfiable: ScheduleAnyway` ensures pods are scheduled even if constraints cannot be perfectly met. `labelSelector` filters which pods are considered in the topology calculations.

```yaml
spec:
  topologySpreadConstraints:
    - maxSkew: 1
      topologyKey: "topology.kubernetes.io/zone"
      whenUnsatisfiable: ScheduleAnyway
      labelSelector:
        matchLabels:
          environment: dev
    - maxSkew: 1
      topologyKey: "kubernetes.io/hostname"
      whenUnsatisfiable: ScheduleAnyway
      labelSelector:
        matchLabels:
          environment: dev

```

--------------------------------

### Check Kubernetes Pod Status

Source: https://cloudfleet.ai/docs/introduction/getting-started

This command lists all the Pods within the Kubernetes cluster and displays their current status. Initially, Pods may go through states like 'Pending' or 'ContainerCreating' before reaching the 'Running' state, especially if new nodes need to be provisioned.

```bash
kubectl get pods

```

--------------------------------

### Configure Cloudfleet CLI authentication profile

Source: https://cloudfleet.ai/docs/introduction/getting-started

Adds a new authentication profile named 'default' for the Cloudfleet CLI. This command requires your organization ID, which can be found in the Cloudfleet console under Billing -> Payment.

```bash
cloudfleet auth add-profile user default YOUR_ORGANIZATION_ID
```

--------------------------------

### Configure CiliumBGPPeerConfig for BGP Peers

Source: https://cloudfleet.ai/docs/networking/on-premises-load-balancing-with-bgp

This manifest defines BGP peer configurations, including graceful restart settings and address family/safi details. It uses `matchLabels` to associate specific BGP advertisements with this peer configuration. The `restartTimeSeconds` should be tuned based on network stability requirements.

```yaml
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPPeerConfig
metadata:
    name: tor-rack-1
spec:
    gracefulRestart:
        enabled: true
        restartTimeSeconds: 15
    families:
        - afi: ipv4
          safi: unicast
          advertisements:
              matchLabels:
                  advertise: "bgp"

```

--------------------------------

### Terraform Module for CFKE AWS Integration

Source: https://cloudfleet.ai/docs/cloud-infrastructure/fleet-configuration

This Terraform module automates the creation of necessary IAM roles, policies, VPCs, and subnets for connecting a CFKE cluster to an AWS account. It ensures secure access by managing IAM roles internally and restricts Cloudfleet's permissions to specific tagged resources.

```terraform
module "cfke_connected_fleet" {
  source               = "registry.terraform.io/cloudfleetai/cfke-connected-fleet/aws"
  # version              = "~> 1.0.0" # (Optional) Specify a version if you want to pin
  control_plane_region = "CONTROL_PLANE_REGION"
  cluster_id           = "CLUSTER_ID"
}

```

--------------------------------

### Check BGP Advertisements with Cilium CLI

Source: https://cloudfleet.ai/docs/networking/on-premises-load-balancing-with-bgp

This command-line interface command is used to view the BGP routes currently being advertised by the Cilium cluster. It's essential for confirming that services and Pod CIDRs are correctly advertised to the BGP network.

```bash
$ cilium bgp routes

```

--------------------------------

### Configure CiliumBGPAdvertisement for Services and Pod Network

Source: https://cloudfleet.ai/docs/networking/on-premises-load-balancing-with-bgp

This manifest configures BGP advertisements for Cilium services and optionally for the Pod network. It allows specifying which services to advertise using selectors and can include the Pod CIDR. Ensure the `advertisementType` and `service` configurations match your desired routing policy.

```yaml
apiVersion: "cilium.io/v2alpha1"
kind: CiliumBGPAdvertisement
metadata:
  name: services
  labels:
    advertise: bgp # This label is used in the next step
spec:
  advertisements:
    - advertisementType: PodCIDR # This is optional and can be omitted if you don't want to advertise the Pod network

    - advertisementType: Service
      service:
        addresses:
          - LoadBalancerIP
      selector:
        matchExpressions:
          # To enable BGP advertisement for all LoadBalancer services, you can use the following expression
          # See https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/#multipool-ipam to learn why
          - { key: somekey, operator: NotIn, values: [ 'never-used-value' ] }

```

--------------------------------

### Deploy Istio CNI Node Agent with CFKE's Cilium

Source: https://cloudfleet.ai/docs/release-notes/release-notes

This example illustrates how to configure CFKE's Cilium deployment to allow other CNI plugins, such as Istio's CNI node agent, to run alongside it. This provides flexibility in managing network plugins within the cluster. Consult the Istio deployment tutorial for detailed steps.

```yaml
# This is a conceptual example. Actual configuration depends on the Cilium installation method.
# Refer to the specific Istio deployment tutorial for precise instructions.

# Example Cilium ConfigMap snippet (if applicable for customization)
apiVersion: v1
kind: ConfigMap
metadata:
  name: cilium-config
  namespace: kube-system
data:
  # Configuration to allow other CNIs. This might vary based on Cilium version.
  enable-other-cni: "true"

# Corresponding Istio CNI agent deployment (example)
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: istio-cni-node-agent
  namespace: kube-system
spec:
  template:
    spec:
      containers:
      - name: istio-cni
        image: istio/cni:latest # Use the appropriate Istio CNI image
        # ... other container configurations
```

--------------------------------

### Configure CiliumBGPClusterConfig for BGP Instances and Peers

Source: https://cloudfleet.ai/docs/networking/on-premises-load-balancing-with-bgp

This manifest sets up BGP cluster configurations, specifying node selectors for cluster membership and defining BGP instances with local ASN and peer details. It references `CiliumBGPPeerConfig` for peer settings. This is crucial for multi-datacenter or cloud environments.

```yaml
apiVersion: cilium.io/v2alpha1
kind: CiliumBGPClusterConfig
metadata:
  name: tor-rack-1
spec:
  nodeSelector:
    matchLabels:
      topology.kubernetes.io/region: berlin
      topology.kubernetes.io/zone: rack-1
  bgpInstances:
    - name: "cloudfleet"
      localASN: 65001 # The ASN number you want to use for your cluster
      peers:
        - name: unifi
          peerASN: 65000 # Router's ASN number
          peerAddress: "172.16.10.1" # Router's IP address
          peerConfigRef:
            name: tor-rack-1 # Reference to the peer configuration. We created this in the previous step

```

--------------------------------

### Generate Node Join Information for CFKE - Terraform

Source: https://cloudfleet.ai/docs/terraform/resources

Generates cloud-init userdata for joining self-managed nodes to CFKE clusters, supporting any cloud provider or on-premises infrastructure. Requires cluster ID, zone, and region, with optional settings for base64 encoding, gzip compression, NVIDIA driver installation, and node labels.

```terraform
# Basic node join information
resource "cloudfleet_cfke_node_join_information" "basic" {
  cluster_id = cloudfleet_cfke_cluster.example.id
  zone       = "us-west-2a"
  region     = "us-west-2"
}

# GPU node with NVIDIA drivers and custom labels
resource "cloudfleet_cfke_node_join_information" "gpu_node" {
  cluster_id             = cloudfleet_cfke_cluster.example.id
  zone                   = "us-west-2a"
  region                 = "us-west-2"
  install_nvidia_drivers = true

  node_labels = {
    "node-type"     = "gpu-worker"
    "environment"   = "production"
    "workload-type" = "machine-learning"
  }
}
```

--------------------------------

### Describe HPA Resource Details using kubectl (Bash)

Source: https://cloudfleet.ai/docs/workload-management/pod-autoscaling

This command provides detailed information about a specific Horizontal Pod Autoscaler (HPA) resource. It includes configuration details, events, and current metrics, offering a deeper insight into the autoscaler's state and behavior than the 'get hpa' command. This is helpful for troubleshooting scaling issues.

```bash
kubectl describe hpa my-app-hpa

```

--------------------------------

### Remove Self-Managed Node from Cluster

Source: https://cloudfleet.ai/docs/hybrid-and-on-premises/self-managed-nodes

These commands remove a self-managed node from a Kubernetes cluster by uninstalling the kubelet, deleting Kubernetes configuration files, and then removing the node object from the cluster using kubectl.

```bash
sudo apt remove -y kubelet
sudo rm -rf /etc/kubernetes/
sudo rm -rf /var/lib/kubelet/
kubectl delete node NODE_NAME

```

--------------------------------

### CiliumEgressGatewayPolicy Manifest Example

Source: https://cloudfleet.ai/docs/networking/egress-gateways

This manifest defines a CiliumEgressGatewayPolicy, specifying which pods can send egress traffic, to which destinations, and through which egress gateway node and IP. It requires a running Cilium installation with the Egress Gateway feature enabled.

```yaml
apiVersion: cilium.io/v2
kind: CiliumEgressGatewayPolicy
metadata:
  name: my-app-egress-policy
spec:
  selectors:
    - podSelector:
        matchLabels:
          app: my-app
  destinationCIDRs:
    - "192.0.2.0/24"
    - "203.0.113.10/32"
  egressGateway:
    nodeSelector:
      matchLabels:
        kubernetes.io/hostname: self-managed-egress-node
    egressIP: "10.0.1.100"
```

--------------------------------

### Create a Fleet for Auto-Provisioning

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

Configure a Fleet to authorize CFKE to provision nodes in your cloud provider account. Fleets support AWS, GCP, and Hetzner Cloud.

```APIDOC
## POST /v1/clusters/{cluster_id}/fleets

### Description
Creates a new Fleet for a given cluster, enabling auto-provisioning of nodes in a specified cloud provider.

### Method
POST

### Endpoint
https://api.cloudfleet.ai/v1/clusters/{cluster_id}/fleets

### Parameters
#### Path Parameters
- **cluster_id** (string) - Required - The ID of the cluster to associate the Fleet with.

#### Request Body
- **name** (string) - Required - The name of the Fleet.
- **provider** (string) - Required - The cloud provider (e.g., 'aws', 'gcp', 'hetzner').
- **max_vcpu** (integer) - Required - The maximum number of vCPUs the Fleet can provision.
- **hetzner_api_token** (string) - Optional - API token for Hetzner Cloud (required if provider is 'hetzner').
- **aws_role_arn** (string) - Optional - IAM role ARN for AWS (required if provider is 'aws').
- **gcp_project_id** (string) - Optional - Project ID for GCP (required if provider is 'gcp').

### Request Example (GCP)
```json
{
  "name": "production-gcp",
  "provider": "gcp",
  "gcp_project_id": "my-gcp-project",
  "max_vcpu": 128
}
```

### Response
#### Success Response (201 Created)
- **name** (string) - The name of the created Fleet.
- **provider** (string) - The cloud provider for the Fleet.
- **gcp_project_id** (string) - The GCP project ID (if applicable).
- **max_vcpu** (integer) - The maximum vCPU capacity for the Fleet.
- **status** (string) - The status of the Fleet (e.g., 'active').
- **current_vcpu_usage** (integer) - The current vCPU utilization of the Fleet.

#### Response Example
```json
{
  "name": "production-gcp",
  "provider": "gcp",
  "gcp_project_id": "my-gcp-project",
  "max_vcpu": 128,
  "status": "active",
  "current_vcpu_usage": 0
}
```
```

--------------------------------

### Add Custom NVIDIA GPU Model Label

Source: https://cloudfleet.ai/docs/hybrid-and-on-premises/self-managed-nodes

This command adds a custom label to a specific node to identify its GPU model, such as 'V100'. This is useful for differentiating between various GPU models within a cluster for scheduling purposes.

```bash
kubectl label node NODE_NAME cfke.io/accelerator-model=V100

```

--------------------------------

### Terraform and kubectl Commands for Provisioning and Validation

Source: https://cloudfleet.ai/docs/hybrid-and-on-premises/scaleway

These commands are used to initialize the Terraform project, apply the configuration to provision Scaleway infrastructure and join instances to the CFKE cluster, and finally, to verify that the new nodes have successfully joined the cluster using kubectl.

```bash
terraform init
terraform apply

```

```bash
kubectl get nodes

```

--------------------------------

### Create CFKE Kubernetes Cluster using CLI and API

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

Creates a new Cloudfleet Kubernetes Engine (CFKE) cluster with a managed control plane. Clusters provision in about 2 minutes and offer a standard Kubernetes API endpoint. Supports CLI and REST API.

```bash
# Using Cloudfleet CLI
cloudfleet clusters create \
  --name my-cluster \
  --region europe-central-1a \
  --tier basic
```

```bash
# Using REST API with curl
curl -X POST https://api.cloudfleet.ai/v1/clusters \
  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "my-cluster",
    "region": "europe-central-1a",
    "tier": "basic"
  }'
```

```json
{
  "id": "cls_abc123def456",
  "name": "my-cluster",
  "region": "europe-central-1a",
  "tier": "basic",
  "status": "provisioning",
  "endpoint": "https://cls-abc123def456.api.cloudfleet.ai",
  "certificate_authority": "LS0tLS1CRUdJTi...",
  "kubernetes_version": "1.28.3"
}
```

--------------------------------

### Scaleway Node Join and Server Provisioning (Terraform)

Source: https://cloudfleet.ai/docs/terraform/resources

This configuration sets up a Scaleway instance to join a CFKE cluster. It specifies `base64_encode = false` and `gzip = false` for Scaleway's user data requirements. The `scaleway_instance_server` resource creates the VM, passing the rendered join information as cloud-init user data.

```terraform
resource "cloudfleet_cfke_node_join_information" "scaleway" {
  cluster_id = cloudfleet_cfke_cluster.example.id
  region     = "fr-par"
  zone       = "fr-par-1"

  node_labels = {
    "cfke.io/provider" = "scaleway"
  }

  # Scaleway requires uncompressed userdata
  base64_encode = false
  gzip          = false
}

resource "scaleway_instance_server" "worker" {
  name  = "cfke-worker-scaleway"
  type  = "DEV1-M"
  image = "ubuntu_jammy"

  user_data = {
    cloud-init = cloudfleet_cfke_node_join_information.scaleway.rendered
  }
}
```

--------------------------------

### Create CFKE Fleet for Auto-provisioning using CLI and API

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

Authorizes CFKE to provision nodes in your cloud provider account by creating a Fleet. Supports Hetzner, AWS, and GCP providers with specific configuration parameters for each.

```bash
# Create Hetzner Cloud Fleet
cloudfleet clusters fleets create cls_abc123def456 \
  --name production-hetzner \
  --provider hetzner \
  --hetzner-api-token ${HETZNER_TOKEN} \
  --max-vcpu 96
```

```bash
# Create AWS Fleet using Terraform-generated IAM role
cloudfleet clusters fleets create cls_abc123def456 \
  --name production-aws \
  --provider aws \
  --aws-role-arn arn:aws:iam::123456789012:role/cfke-fleet-role \
  --max-vcpu 256
```

```bash
# Via REST API - GCP Fleet
curl -X POST https://api.cloudfleet.ai/v1/clusters/cls_abc123def456/fleets \
  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "production-gcp",
    "provider": "gcp",
    "gcp_project_id": "my-gcp-project",
    "max_vcpu": 128
  }'
```

```json
{
  "name": "production-gcp",
  "provider": "gcp",
  "gcp_project_id": "my-gcp-project",
  "max_vcpu": 128,
  "status": "active",
  "current_vcpu_usage": 0
}
```

--------------------------------

### Create Kubernetes Service Account Definition

Source: https://cloudfleet.ai/docs/workload-management/accessing-cloud-apis

This YAML defines a Kubernetes Service Account named 'example-application' in the 'default' namespace. This Service Account can then be assigned to application pods to grant them specific permissions and identities within the cluster, enabling secure access to external resources.

```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  namespace: default
  name: example-application
```

--------------------------------

### Get Specific Cluster Details

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

Retrieves detailed information about a specific Kubernetes cluster by its ID.

```APIDOC
## Get Specific Cluster Details

### Description
Retrieves detailed information about a specific Kubernetes cluster by its ID.

### Method
GET

### Endpoint
/v1/clusters/{cluster_id}

### Path Parameters
- **cluster_id** (string) - Required - The unique identifier of the cluster.

### Query Parameters
None

### Request Body
None

### Request Example
```bash
curl -X GET https://api.cloudfleet.ai/v1/clusters/cls_abc123def456 \
  -H "Authorization: Bearer ${ACCESS_TOKEN}"
```

### Response
#### Success Response (200)
- **id** (string) - Unique identifier for the cluster.
- **name** (string) - Name of the cluster.
- **region** (string) - The cloud provider region where the cluster is deployed.
- **tier** (string) - The pricing tier of the cluster.
- **status** (string) - The current status of the cluster.
- **endpoint** (string) - The API endpoint for accessing the cluster's Kubernetes API.
- **certificate_authority** (string) - The base64 encoded CA certificate for the cluster.
- **kubernetes_version** (string) - The version of Kubernetes running on the cluster.
- **fleets** (array) - A list of fleets associated with the cluster.
  - **name** (string) - Name of the fleet.
  - **provider** (string) - Cloud provider of the fleet (e.g., 'aws', 'gcp').
  - **max_vcpu** (integer) - Maximum vCPU capacity available in the fleet.
  - **current_vcpu_usage** (integer) - Current vCPU usage across the fleet.

#### Response Example
```json
{
  "id": "cls_abc123def456",
  "name": "production-cluster",
  "region": "europe-central-1a",
  "tier": "pro",
  "status": "running",
  "endpoint": "https://cls-abc123def456.api.cloudfleet.ai",
  "certificate_authority": "LS0tLS1CRUdJTi...",
  "kubernetes_version": "1.28.3",
  "fleets": [
    {
      "name": "production-aws",
      "provider": "aws",
      "max_vcpu": 256,
      "current_vcpu_usage": 48
    },
    {
      "name": "production-hetzner",
      "provider": "hetzner",
      "max_vcpu": 96,
      "current_vcpu_usage": 24
    }
  ]
}
```
```

--------------------------------

### Verify kubectl cluster connectivity

Source: https://cloudfleet.ai/docs/introduction/getting-started

Checks the connectivity to the Kubernetes cluster by displaying information about the Kubernetes control plane. This command is used after configuring kubectl with the cluster's kubeconfig.

```bash
kubectl cluster-info
```

--------------------------------

### Create Cloudfleet API Tokens

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

Instructions for generating API tokens for programmatic access to Cloudfleet API and CFKE clusters using the `cloudfleet` CLI and `curl` for REST API calls. It also includes steps for configuring the CLI with a token and using it in GitHub Actions.

```bash
# Create API token via CLI
cloudfleet tokens create \
  --name "ci-pipeline-token" \
  --role Administrator

# Output:
# Token created successfully
# ID: nFYyVdtg8K1aDujwk3YFh1
# Secret: sct_abc123def456ghi789jkl012mno345pqr678stu901vwx234yz
# Save the secret securely - it will not be shown again

# Via REST API
curl -X POST https://api.cloudfleet.ai/v1/tokens \
  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
  -H "Content-Type: application/json" \
  -d '{
    "name": "github-actions-token",
    "role": "User"
  }'

# Response
# {
#   "id": "2iKo4Asy51EUx2gpJFW76t",
#   "name": "github-actions-token",
#   "role": "User",
#   "secret": "sct_xyz987wvu654tsr321ponm987lkj654ihg321",
#   "created_at": "2025-10-14T10:30:00Z"
# }

# Configure CLI with token for automation
cloudfleet auth add-profile token ci-profile \
  dc78c04e-6651-4e5d-9c04-079f6532989b \
  nFYyVdtg8K1aDujwk3YFh1 \
  sct_abc123def456ghi789jkl012mno345pqr678stu901vwx234yz

# Use token in GitHub Actions
# .github/workflows/deploy.yml
# env:
#   CLOUDFLEET_ORGANIZATION_ID: dc78c04e-6651-4e5d-9c04-079f6532989b
#   CLOUDFLEET_ACCESS_TOKEN_ID: nFYyVdtg8K1aDujwk3YFh1
#   CLOUDFLEET_ACCESS_TOKEN_SECRET: ${{ secrets.CLOUDFLEET_TOKEN_SECRET }}
```

--------------------------------

### Terraform Configuration for Vultr CFKE Node Integration

Source: https://cloudfleet.ai/docs/hybrid-and-on-premises/vultr

This Terraform configuration defines resources for creating Vultr instances and configuring them to join a CFKE cluster. It uses the cloudfleet and vultr providers, generates cloud-init data for automatic node registration, and sets up a VPC for node communication. Dependencies include the Terraform CLI, Cloudfleet CLI, and a Vultr account with a Personal Access Token.

```hcl
variable "cfke_cluster_id" {
  type    = string
  default = "CFKE Cluster ID"
}

variable "vultr_api_key" {
  type = string
  description = "Vultr Personal Access Token"
}

variable "region" {
  type    = string
  default = "fra"
}

terraform {
  required_providers {
    cloudfleet = {
      source = "terraform.cloudfleet.ai/cloudfleet/cloudfleet"
    }

    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.0"
    }

    vultr = {
      source  = "vultr/vultr"
      version = "~> 2.0"
    }
  }
}

provider "vultr" {
  api_key = var.vultr_api_key
}

data "cloudfleet_cfke_cluster" "cluster" {
  id = var.cfke_cluster_id
}

// Generate the cloud-init configuration to join the Vultr instance to the CFKE cluster
resource "cloudfleet_cfke_node_join_information" "vultr" {
  cluster_id = data.cloudfleet_cfke_cluster.cluster.id
  region     = var.region
  zone       = var.region
  node_labels = {
    "cfke.io/provider" = "vultr"
  }
}

// Vultr resources

// Create a VPC for the CFKE nodes. This is optional since Vultr instances can communicate over the public internet, but when nodes are part of the same VPC, they establish encrypted tunnels over the private network
resource "vultr_vpc" "cfke_vpc" {
  region      = var.region
  description = "CFKE VPC"
}

data "vultr_os" "ubuntu" {
  filter {
    name = "name"
    values = ["Ubuntu 24.04 LTS x64"]
  }
}

resource "vultr_instance" "cfke_node" {
  count       = 1
  plan        = "vc2-2c-4gb"
  region      = var.region
  os_id       = data.vultr_os.ubuntu.id
  enable_ipv6 = true
  label       = "cfke-node-${count.index + 1}"
  user_data   = cloudfleet_cfke_node_join_information.vultr.rendered // Use the generated cloud-init configuration
  hostname    = "cfke-node-${count.index + 1}"
  vpc_ids = [vultr_vpc.cfke_vpc.id]
}

```

--------------------------------

### Get Specific Cluster Details via Cloudfleet REST API

Source: https://context7.com/context7/cloudfleet_ai/llms.txt

Retrieves detailed information for a specific cluster by its ID using the Cloudfleet AI REST API. Requires an authorization token and the cluster ID in the URL.

```bash
curl -X GET https://api.cloudfleet.ai/v1/clusters/cls_abc123def456 \
  -H "Authorization: Bearer ${ACCESS_TOKEN}"
```