### Kernel Module Build and Load Instructions (Bash)

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

These bash commands guide the user through compiling and loading the picoevb-rdma kernel module on target systems. It covers installing necessary dependencies, executing build scripts for different architectures (Jetson/PC), loading the module, and verifying its successful installation.

```bash
# Install dependencies
sudo apt update
sudo apt install build-essential bc

# Build for Jetson/Drive AGX Xavier (native compilation)
cd kernel-module/
./build-for-jetson-drive-igpu-native.sh

# Or build for PC with CUDA support
./build-for-pc-native.sh

# Load the kernel module
sudo insmod ./picoevb-rdma.ko

# Verify module loaded successfully
lspci -v | grep -A 5 "Memory controller: NVIDIA"
# Should show: Kernel driver in use: picoevb-rdma

# Check device file created
ls -l /dev/picoevb
# Expected: crw------- 1 root root

# Optional: Set permissions for non-root access
sudo chmod 666 /dev/picoevb

```

--------------------------------

### Build and Run Client Applications (Jetson/Drive AGX Xavier)

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Instructions for compiling and running client applications on Jetson or Drive AGX Xavier platforms. This involves installing dependencies, building the applications, and executing various RDMA and CUDA transfer tests. Success for some tests is indicated by the absence of output.

```bash
# Install dependencies
sudo apt update
sudo apt install build-essential bc

# Build on Jetson/Drive AGX Xavier
cd client-applications/
./build-for-jetson-drive-igpu-native.sh

# Run tests (requires kernel module loaded)
sudo ./rdma-malloc          # Test malloc-based transfer
sudo ./rdma-cuda            # Test CUDA memory transfer
sudo ./rdma-malloc-h2c-perf # Host-to-card performance test
sudo ./rdma-malloc-c2h-perf # Card-to-host performance test
sudo ./rdma-cuda-h2c-perf   # CUDA host-to-card performance
sudo ./rdma-cuda-c2h-perf   # CUDA card-to-host performance
sudo ./set-leds 2           # Control FPGA LEDs (0-7)

# Expected output from successful test:
# (no output means success for rdma-malloc/rdma-cuda)
# Performance tests show: Bytes:XXXXX usecs:XXXX MB/s:XX.XX
```

--------------------------------

### CUDA Memory Transfer and DMA Example (C)

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

This C code demonstrates a full CUDA memory transfer workflow. It includes allocating pinned host memory, enabling synchronous operations, pinning memory for RDMA, filling memory with a CUDA kernel, performing a host-to-host-like DMA transfer via the FPGA, and validating the transferred data. Error handling and cleanup are also included.

```c
#include <cuda.h>
#include <cuda_runtime_api.h>
#include <stdio.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include "picoevb-rdma-ioctl.h"

#define SURFACE_SIZE (1024 * 1024)
#define DATA(x, y) (((y & 0xffff) << 16) | ((x) & 0xffff))

__global__ void fill_surface(uint32_t *output, uint32_t xor_val) {
    unsigned int pos_x = (blockIdx.x * blockDim.x) + threadIdx.x;
    unsigned int pos_y = (blockIdx.y * blockDim.y) + threadIdx.y;
    unsigned int offset = (pos_y * 1024) + pos_x;
    output[offset] = DATA(pos_x, pos_y) ^ xor_val;
}

int main() {
    int fd = open("/dev/picoevb", O_RDWR);
    if (fd < 0) {
        perror("open() failed");
        return 1;
    }

    // Allocate CUDA memory
    uint32_t *src_d, *dst_d;
    cudaError_t ce = cudaHostAlloc(&src_d, SURFACE_SIZE * sizeof(uint32_t),
                                    cudaHostAllocDefault);
    if (ce != cudaSuccess) {
        fprintf(stderr, "cudaHostAlloc(src) failed: %d\n", ce);
        return 1;
    }

    ce = cudaHostAlloc(&dst_d, SURFACE_SIZE * sizeof(uint32_t),
                       cudaHostAllocDefault);
    if (ce != cudaSuccess) {
        fprintf(stderr, "cudaHostAlloc(dst) failed: %d\n", ce);
        return 1;
    }

    // Enable synchronous memory operations
    unsigned int flag = 1;
    CUresult cr = cuPointerSetAttribute(&flag, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
                                         (CUdeviceptr)src_d);
    if (cr != CUDA_SUCCESS) {
        fprintf(stderr, "cuPointerSetAttribute(src) failed: %d\n", cr);
        return 1;
    }

    cr = cuPointerSetAttribute(&flag, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
                               (CUdeviceptr)dst_d);
    if (cr != CUDA_SUCCESS) {
        fprintf(stderr, "cuPointerSetAttribute(dst) failed: %d\n", cr);
        return 1;
    }

    // Pin memory for RDMA
    struct picoevb_rdma_pin_cuda pin_src, pin_dst;
    pin_src.va = (__u64)src_d;
    pin_src.size = SURFACE_SIZE * sizeof(uint32_t);
    if (ioctl(fd, PICOEVB_IOC_PIN_CUDA, &pin_src) != 0) {
        perror("ioctl(PIN_CUDA src) failed");
        return 1;
    }

    pin_dst.va = (__u64)dst_d;
    pin_dst.size = SURFACE_SIZE * sizeof(uint32_t);
    if (ioctl(fd, PICOEVB_IOC_PIN_CUDA, &pin_dst) != 0) {
        perror("ioctl(PIN_CUDA dst) failed");
        return 1;
    }

    // Fill surfaces using CUDA kernel
    dim3 dimGrid(64, 64);
    dim3 dimBlock(16, 16);
    fill_surface<<<dimGrid, dimBlock>>>(src_d, 0);
    fill_surface<<<dimGrid, dimBlock>>>(dst_d, 0xffffffffU);
    cudaDeviceSynchronize();

    // Perform DMA transfer via FPGA
    struct picoevb_rdma_h2c2h_dma dma_params;
    dma_params.src = pin_src.handle;
    dma_params.dst = pin_dst.handle;
    dma_params.len = SURFACE_SIZE * sizeof(uint32_t);
    dma_params.flags = PICOEVB_H2C2H_DMA_FLAG_SRC_IS_CUDA |
                       PICOEVB_H2C2H_DMA_FLAG_DST_IS_CUDA;

    if (ioctl(fd, PICOEVB_IOC_H2C2H_DMA, &dma_params) != 0) {
        perror("ioctl(DMA) failed");
        return 1;
    }

    printf("DMA transfer completed in %lu ns\n", dma_params.dma_time_ns);
    printf("Throughput: %.2f MB/s\n",
           (double)(SURFACE_SIZE * 4) / (dma_params.dma_time_ns / 1000.0));

    // Validate transfer
    int errors = 0;
    for (int y = 0; y < 1024 && errors < 10; y++) {
        for (int x = 0; x < 1024 && errors < 10; x++) {
            uint32_t expected = DATA(x, y);
            uint32_t actual = dst_d[y * 1024 + x];
            if (actual != expected) {
                fprintf(stderr, "Mismatch at [%d,%d]: expected 0x%x, got 0x%x\n",
                        x, y, expected, actual);
                errors++;
            }
        }
    }

    if (errors == 0) {
        printf("Validation successful - all data transferred correctly\n");
    }

    // Cleanup
    struct picoevb_rdma_unpin_cuda unpin;
    unpin.handle = pin_dst.handle;
    ioctl(fd, PICOEVB_IOC_UNPIN_CUDA, &unpin);
    unpin.handle = pin_src.handle;
    ioctl(fd, PICOEVB_IOC_UNPIN_CUDA, &unpin);

    cudaFreeHost(dst_d);
    cudaFreeHost(src_d);
    close(fd);
    return 0;
}

```

--------------------------------

### Build and Run Client Applications (PC)

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Instructions for compiling and running client applications on a PC. This involves building the applications using a provided script and executing various RDMA and CUDA transfer tests. Successful execution of some tests is indicated by no output.

```bash
# Build on PC
./build-for-pc-native.sh

# Run tests (requires kernel module loaded)
sudo ./rdma-malloc          # Test malloc-based transfer
sudo ./rdma-cuda            # Test CUDA memory transfer
sudo ./rdma-malloc-h2c-perf # Host-to-card performance test
sudo ./rdma-malloc-c2h-perf # Card-to-host performance test
sudo ./rdma-cuda-h2c-perf   # CUDA host-to-card performance
sudo ./rdma-cuda-c2h-perf   # CUDA card-to-host performance
sudo ./set-leds 2           # Control FPGA LEDs (0-7)

# Expected output from successful test:
# (no output means success for rdma-malloc/rdma-cuda)
# Performance tests show: Bytes:XXXXX usecs:XXXX MB/s:XX.XX
```

--------------------------------

### Build Kernel Module for Jetson Drive IGPU on PC

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Builds the picoevb-rdma kernel module on an x86 Linux PC for use on a Jetson Drive AGX Xavier. This process generates the 'picoevb-rdma.ko' file. It requires kernel headers to be set correctly.

```shell
# kernel headers
KDIR=/path/to/linux-headers-4.9.140-tegra-linux_x86_64/kernel-4.9/ ./build-for-jetson-drive-igpu-on-pc.sh
```

--------------------------------

### Load Kernel Module

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Loads the compiled 'picoevb-rdma.ko' kernel module into the system. This command requires root privileges.

```shell
sudo insmod ./picoevb-rdma.ko
```

--------------------------------

### Build Kernel Module for PC Native

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Builds the picoevb-rdma kernel module for native execution on an x86 Linux PC. This process generates the 'picoevb-rdma.ko' file. It requires essential build tools like 'build-essential' and 'bc'.

```shell
sudo apt update
sudo apt install build-essential bc
cd /path/to/this/project/kernel-module/
./build-for-pc-native.sh
```

--------------------------------

### Control FPGA Board LEDs using PICOEVB_IOC_LED

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Controls the LEDs on the PicoEVB board by sending a control value via ioctl. The program expects a single command-line argument representing the desired LED state. Note that the hardware inverts the logic, so 0 typically means all LEDs are on, and 7 means all are off. It requires 'picoevb-rdma-ioctl.h' and opens '/dev/picoevb'.

```c
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include "picoevb-rdma-ioctl.h"

int main(int argc, char **argv) {
    if (argc != 2) {
        fprintf(stderr, "usage: set-leds <value>\n");
        return 1;
    }

    int fd = open("/dev/picoevb", O_RDWR);
    if (fd < 0) {
        perror("open() failed");
        return 1;
    }

    uint32_t led_value = strtoul(argv[1], NULL, 10);
    int ret = ioctl(fd, PICOEVB_IOC_LED, led_value);
    if (ret != 0) {
        perror("ioctl() failed");
        close(fd);
        return 1;
    }

    printf("LEDs set to value: %u\n", led_value);
    close(fd);
    return 0;
}

```

--------------------------------

### Build User-space Applications for PC Native

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Builds the client applications natively on an x86 Linux PC. This process requires CUDA development tools and essential build utilities.

```shell
sudo apt update
sudo apt install build-essential bc
cd /path/to/this/project/client-applications/
./build-for-pc-native.sh
```

--------------------------------

### Build User-space Applications for Jetson Drive AGX Xavier

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Builds the client applications natively on a Jetson/Drive AGX Xavier. This process requires CUDA development tools and essential build utilities. It generates the executable client applications.

```shell
sudo apt update
sudo apt install build-essential bc
cd /path/to/this/project/client-applications/
./build-for-jetson-drive-igpu-native.sh
```

--------------------------------

### Control PicoEVB LEDs

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Sets the state of the three LEDs on the PicoEVB board. Accepts a single binary command-line parameter. Note that the hardware inverts the input value (0 turns on all LEDs, 7 turns off all LEDs).

```shell
./set-leds 2
./set-leds 5
```

--------------------------------

### Query FPGA Card Information using PICOEVB_IOC_CARD_INFO

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Retrieves hardware information about the FPGA card, specifically the size of its internal RAM. This function requires the 'picoevb-rdma-ioctl.h' header and opens '/dev/picoevb'. It populates a 'picoevb_rdma_card_info' structure, which contains the 'fpga_ram_size'. Error handling for device opening and ioctl calls is included.

```c
#include "picoevb-rdma-ioctl.h"

int fd = open("/dev/picoevb", O_RDWR);
if (fd < 0) {
    perror("Cannot open device");
    return -1;
}

struct picoevb_rdma_card_info card_info;
int ret = ioctl(fd, PICOEVB_IOC_CARD_INFO, &card_info);
if (ret != 0) {
    perror("Failed to get card info");
    close(fd);
    return -1;
}

printf("FPGA RAM size: %lu bytes (%.2f MB)\n",
       card_info.fpga_ram_size,
       (double)card_info.fpga_ram_size / (1024.0 * 1024.0));

close(fd);

```

--------------------------------

### PICOEVB_IOC_LED - Control FPGA Board LEDs

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

This endpoint provides control over the LEDs on the PicoEVB board. It allows setting specific states for diagnostic or status indication purposes.

```APIDOC
## PICOEVB_IOC_LED - Control FPGA Board LEDs

### Description
Sets LED values on PicoEVB board (0=all on, 7=all off due to hardware inversion).

### Method
IOCTL

### Endpoint
`/dev/picoevb`

### Parameters
#### IOCTL Command
- **PICOEVB_IOC_LED**

#### Request Body
- **led_value** (uint32_t) - The value to set for the LEDs. Note that the hardware may invert the logic, so 0 might turn all LEDs on and 7 might turn them all off.

### Request Example
```c
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#include <sys/ioctl.h>
#include "picoevb-rdma-ioctl.h"

// Example usage from command line: ./set-leds 3
int main(int argc, char **argv) {
    if (argc != 2) {
        fprintf(stderr, "usage: set-leds <value>\n");
        return 1;
    }

    int fd = open("/dev/picoevb", O_RDWR);
    if (fd < 0) {
        perror("open() failed");
        return 1;
    }

    uint32_t led_value = strtoul(argv[1], NULL, 10);
    int ret = ioctl(fd, PICOEVB_IOC_LED, led_value);
    if (ret != 0) {
        perror("ioctl() failed");
        close(fd);
        return 1;
    }

    printf("LEDs set to value: %u\n", led_value);
    close(fd);
    return 0;
}
```

### Response
#### Success Response (0 on success)
Indicates the IOCTL call was processed without error. The `ret` value from `ioctl` indicates success (0) or failure (non-zero).

#### Response Example
```c
// No specific response body, success is indicated by ret == 0
printf("LEDs set to value: %u\n", led_value);
```
```

--------------------------------

### PICOEVB_IOC_CARD_INFO - Query FPGA Card Information

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

This endpoint allows querying hardware information about the FPGA card, most notably its internal RAM size, which is crucial for planning data transfers.

```APIDOC
## PICOEVB_IOC_CARD_INFO - Query FPGA Card Information

### Description
Retrieves hardware information including FPGA internal RAM size.

### Method
IOCTL

### Endpoint
`/dev/picoevb`

### Parameters
#### IOCTL Command
- **PICOEVB_IOC_CARD_INFO**

#### Request Body Structure
```c
struct picoevb_rdma_card_info {
    __u64 fpga_ram_size; // Output: Size of the FPGA's internal RAM in bytes
};
```

### Request Example
```c
#include "picoevb-rdma-ioctl.h"

int fd = open("/dev/picoevb", O_RDWR);

struct picoevb_rdma_card_info card_info;
ioctl(fd, PICOEVB_IOC_CARD_INFO, &card_info);

close(fd);
```

### Response
#### Success Response (0 on success)
- **fpga_ram_size** (uint64_t) - The total size of the FPGA's internal RAM in bytes.

#### Response Example
```c
printf("FPGA RAM size: %lu bytes (%.2f MB)\n",
       card_info.fpga_ram_size,
       (double)card_info.fpga_ram_size / (1024.0 * 1024.0));
```
```

--------------------------------

### Run Data Access Tests

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Executes data access tests 'rdma-malloc' and 'rdma-cuda' to verify PCIe data transfer capabilities using different memory allocation methods. These tests require the kernel module to be loaded.

```shell
sudo ./rdma-malloc
sudo ./rdma-cuda
```

--------------------------------

### Card to Host DMA Transfer using PICOEVB_IOC_C2H_DMA

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Performs a unidirectional DMA transfer from the FPGA's internal RAM buffer to host memory. This is useful for retrieving data processed by the FPGA for performance testing. It requires the 'picoevb-rdma-ioctl.h' header and opens '/dev/picoevb'. The function returns the time taken for the DMA operation, enabling bandwidth calculation, and prints the first 16 bytes of the received data for verification.

```c
#include "picoevb-rdma-ioctl.h"

#define TRANSFER_SIZE (64 * 1024 * 1024)

int fd = open("/dev/picoevb", O_RDWR);

// Allocate destination buffer
void *dst = calloc(TRANSFER_SIZE, 1);

// Transfer from FPGA RAM offset 0
struct picoevb_rdma_c2h_dma dma_params;
dma_params.src = 0;  // FPGA RAM offset
dma_params.dst = (__u64)dst;
dma_params.len = TRANSFER_SIZE;
dma_params.flags = 0;

int ret = ioctl(fd, PICOEVB_IOC_C2H_DMA, &dma_params);
if (ret == 0) {
    uint64_t usecs = dma_params.dma_time_ns / 1000;
    printf("Bytes: %d usecs: %lu MB/s: %.2f\n",
           TRANSFER_SIZE, usecs,
           (double)TRANSFER_SIZE / (double)usecs);
}

// Verify received data
printf("First 16 bytes: ");
for (int i = 0; i < 16; i++) {
    printf("%02x ", ((uint8_t*)dst)[i]);
}
printf("\n");

free(dst);
close(fd);

```

--------------------------------

### Cross-Compile User-space Applications for Jetson Drive AGX Xavier on PC

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Cross-compiles client applications on an x86 Linux PC to run on a Jetson/Drive AGX Xavier. Note that CUDA test applications may not fully support cross-compilation. Requires build tools and potentially adjusting the CROSS_COMPILE variable.

```shell
sudo apt update
sudo apt install build-essential bc
cd /path/to/this/project/client-applications/
./build-for-jetson-drive-igpu-on-pc.sh
```

--------------------------------

### Host to Card DMA Transfer using PICOEVB_IOC_H2C_DMA

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Performs a unidirectional DMA transfer from host memory to the FPGA's internal RAM buffer. This is useful for performance testing and loading data onto the FPGA. It requires the 'picoevb-rdma-ioctl.h' header and opens '/dev/picoevb'. The function returns the time taken for the DMA operation in nanoseconds, allowing for bandwidth calculation.

```c
#include "picoevb-rdma-ioctl.h"

#define TRANSFER_SIZE (64 * 1024 * 1024)

int fd = open("/dev/picoevb", O_RDWR);

// Get FPGA RAM size
struct picoevb_rdma_card_info card_info;
ioctl(fd, PICOEVB_IOC_CARD_INFO, &card_info);
printf("FPGA RAM size: %lu bytes\n", card_info.fpga_ram_size);

// Allocate and fill source buffer
void *src = calloc(TRANSFER_SIZE, 1);
memset(src, 0xFF, TRANSFER_SIZE);

// Transfer to FPGA RAM offset 0
struct picoevb_rdma_h2c_dma dma_params;
dma_params.src = (__u64)src;
dma_params.dst = 0;  // FPGA RAM offset
dma_params.len = TRANSFER_SIZE;
dma_params.flags = 0;

int ret = ioctl(fd, PICOEVB_IOC_H2C_DMA, &dma_params);
if (ret == 0) {
    uint64_t usecs = dma_params.dma_time_ns / 1000;
    printf("Bytes: %d usecs: %lu MB/s: %.2f\n",
           TRANSFER_SIZE, usecs,
           (double)TRANSFER_SIZE / (double)usecs);
}

free(src);
close(fd);

```

--------------------------------

### Run Performance Tests

Source: https://github.com/nvidia/jetson-rdma-picoevb/blob/master/README.md

Executes performance tests for uni-directional copy features ('h2c' for host-to-card, 'c2h' for card-to-host) using both malloc() and CUDA memory allocations. These tests help measure transfer speeds.

```shell
sudo ./rdma-malloc-h2c-perf
sudo ./rdma-malloc-c2h-perf
sudo ./rdma-cuda-h2c-perf
sudo ./rdma-cuda-c2h-perf
```

--------------------------------

### Kernel Module IOCTL Interface

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

This section details the IOCTL commands available through the Linux kernel module for managing CUDA memory and initiating DMA transfers between PCIe devices and GPU memory.

```APIDOC
## PICOEVB_IOC_PIN_CUDA - Pin CUDA memory for DMA access

### Description
Pins CUDA-allocated memory pages and returns a handle for subsequent DMA operations. This prepares GPU memory for direct PCIe access by the FPGA.

### Method
IOCTL

### Endpoint
/dev/picoevb

### Parameters
#### Path Parameters
None

#### Query Parameters
None

#### Request Body
- **va** (__u64) - Required - Virtual address of the CUDA memory to pin.
- **size** (__u64) - Required - Size of the memory region in bytes.

### Request Example
```c
#include <fcntl.h>
#include <sys/ioctl.h>
#include <cuda_runtime_api.h>
#include "picoevb-rdma-ioctl.h"

int fd = open("/dev/picoevb", O_RDWR);
// ... allocation and CUDA setup ...

struct picoevb_rdma_pin_cuda pin_params;
pin_params.va = (__u64)cuda_mem;
pin_params.size = 4 * 1024 * 1024;

int ret = ioctl(fd, PICOEVB_IOC_PIN_CUDA, &pin_params);
```

### Response
#### Success Response (0)
- **handle** (unsigned int) - A handle representing the pinned memory, to be used in subsequent operations.

#### Response Example
```json
{
  "handle": 12345
}
```

## PICOEVB_IOC_UNPIN_CUDA - Unpin CUDA memory

### Description
Releases pinned CUDA memory using the handle obtained from `PICOEVB_IOC_PIN_CUDA`, allowing the memory to be freed.

### Method
IOCTL

### Endpoint
/dev/picoevb

### Parameters
#### Path Parameters
None

#### Query Parameters
None

#### Request Body
- **handle** (unsigned int) - Required - The handle of the CUDA memory to unpin.

### Request Example
```c
#include "picoevb-rdma-ioctl.h"

struct picoevb_rdma_unpin_cuda unpin_params;
unpin_params.handle = pin_params.handle; // From previous PIN_CUDA operation

int ret = ioctl(fd, PICOEVB_IOC_UNPIN_CUDA, &unpin_params);
```

### Response
#### Success Response (0)
Indicates successful unpinning of the CUDA memory.

#### Response Example
```json
{
  "status": "success"
}
```

## PICOEVB_IOC_H2C2H_DMA - Bidirectional DMA transfer via FPGA RAM

### Description
Performs a bidirectional DMA transfer of data through the FPGA's internal memory. This operation supports both standard malloc'd memory and CUDA-allocated memory as source and destination.

### Method
IOCTL

### Endpoint
/dev/picoevb

### Parameters
#### Path Parameters
None

#### Query Parameters
None

#### Request Body
- **src** (__u64) - Required - Source memory address (virtual address for malloc, handle for CUDA pinned memory).
- **dst** (__u64) - Required - Destination memory address (virtual address for malloc, handle for CUDA pinned memory).
- **len** (__u64) - Required - The number of bytes to transfer.
- **flags** (unsigned int) - Optional - Flags to indicate memory types (e.g., `PICOEVB_H2C2H_DMA_FLAG_SRC_IS_CUDA`, `PICOEVB_H2C2H_DMA_FLAG_DST_IS_CUDA`).

### Request Example
```c
#include <stdlib.h>
#include <string.h>
#include "picoevb-rdma-ioctl.h"

#define BUFFER_SIZE (1024 * 1024)

int fd = open("/dev/picoevb", O_RDWR);

// Example 1: malloc to malloc transfer
uint32_t *src = malloc(BUFFER_SIZE);
uint32_t *dst = malloc(BUFFER_SIZE);

struct picoevb_rdma_h2c2h_dma dma_params;
dma_params.src = (__u64)src;
dma_params.dst = (__u64)dst;
dma_params.len = BUFFER_SIZE;
dma_params.flags = 0;

int ret = ioctl(fd, PICOEVB_IOC_H2C2H_DMA, &dma_params);
```

### Response
#### Success Response (0)
- **dma_time_ns** (__u64) - The time taken for the DMA transfer in nanoseconds.

#### Response Example
```json
{
  "dma_time_ns": 500000
}
```
```

--------------------------------

### Pin CUDA Memory for DMA Access (C)

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Pins CUDA-allocated memory pages using the PICOEVB_IOC_PIN_CUDA ioctl, preparing it for direct PCIe DMA access. It requires the 'picoevb-rdma-ioctl.h' header and returns a handle for subsequent operations. Input is the virtual address and size of CUDA memory; output is a handle.

```c
#include <fcntl.h>
#include <sys/ioctl.h>
#include <cuda_runtime_api.h>
#include "picoevb-rdma-ioctl.h"

int fd = open("/dev/picoevb", O_RDWR);
if (fd < 0) {
    perror("Failed to open device");
    return -1;
}

// Allocate CUDA memory
uint32_t *cuda_mem;
cudaError_t ce = cudaHostAlloc(&cuda_mem, 4 * 1024 * 1024, cudaHostAllocDefault);
if (ce != cudaSuccess) {
    fprintf(stderr, "CUDA allocation failed\n");
    return -1;
}

// Enable synchronous memory operations
unsigned int flag = 1;
CUresult cr = cuPointerSetAttribute(&flag, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
                                     (CUdeviceptr)cuda_mem);

// Pin the memory for RDMA
struct picoevb_rdma_pin_cuda pin_params;
pin_params.va = (__u64)cuda_mem;
pin_params.size = 4 * 1024 * 1024;

int ret = ioctl(fd, PICOEVB_IOC_PIN_CUDA, &pin_params);
if (ret != 0) {
    perror("Failed to pin CUDA memory");
    return -1;
}

printf("Memory pinned successfully, handle: %u\n", pin_params.handle);

```

--------------------------------

### Unload Kernel Module

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

This command unloads the 'picoevb-rdma' kernel module. It is typically used after testing or when no longer needing the RDMA functionality.

```bash
sudo rmmod picoevb-rdma
```

--------------------------------

### PICOEVB_IOC_C2H_DMA - Card to Host Unidirectional DMA

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

This endpoint enables unidirectional DMA transfers from the FPGA's internal RAM buffer to host memory. It is used for retrieving data from the FPGA for performance analysis or processing.

```APIDOC
## PICOEVB_IOC_C2H_DMA - Card to Host Unidirectional DMA

### Description
Transfers data from FPGA internal RAM buffer to host memory for performance testing.

### Method
IOCTL

### Endpoint
`/dev/picoevb`

### Parameters
#### IOCTL Command
- **PICOEVB_IOC_C2H_DMA**

#### Request Body Structure
```c
struct picoevb_rdma_c2h_dma {
    __u64 src;       // FPGA RAM offset (source)
    __u64 dst;       // Host memory destination address
    __u32 len;       // Transfer length in bytes
    __u32 flags;     // Transfer flags (currently unused, set to 0)
    __u64 dma_time_ns; // Output: DMA transfer time in nanoseconds
};
```

### Request Example
```c
#include "picoevb-rdma-ioctl.h"

#define TRANSFER_SIZE (64 * 1024 * 1024)

int fd = open("/dev/picoevb", O_RDWR);

// Allocate destination buffer
void *dst = calloc(TRANSFER_SIZE, 1);

// Transfer from FPGA RAM offset 0
struct picoevb_rdma_c2h_dma dma_params;
dma_params.src = 0;  // FPGA RAM offset
dma_params.dst = (__u64)dst;
dma_params.len = TRANSFER_SIZE;
dma_params.flags = 0;

int ret = ioctl(fd, PICOEVB_IOC_C2H_DMA, &dma_params);

// Verify received data
printf("First 16 bytes: ");
for (int i = 0; i < 16; i++) {
    printf("%02x ", ((uint8_t*)dst)[i]);
}
printf("\n");

free(dst);
close(fd);
```

### Response
#### Success Response (0 on success)
- **dma_time_ns** (uint64_t) - The duration of the DMA transfer in nanoseconds. The `ret` value from `ioctl` indicates success (0) or failure (non-zero).

#### Response Example
Upon successful transfer, `dma_params.dma_time_ns` will be populated. The example below shows how to calculate MB/s.
```c
if (ret == 0) {
    uint64_t usecs = dma_params.dma_time_ns / 1000;
    printf("Bytes: %d usecs: %lu MB/s: %.2f\n",
           TRANSFER_SIZE, usecs,
           (double)TRANSFER_SIZE / (double)usecs);
}
```
```

--------------------------------

### Bidirectional DMA Transfer via FPGA RAM (C)

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Performs a bidirectional DMA transfer using the PICOEVB_IOC_H2C2H_DMA ioctl, with data passing through FPGA internal memory. It supports both standard malloc'd memory and CUDA-allocated memory as source and destination. The ioctl call measures and returns the DMA transfer time in nanoseconds.

```c
#include <stdlib.h>
#include <string.h>
#include "picoevb-rdma-ioctl.h"

#define BUFFER_SIZE (1024 * 1024)

int fd = open("/dev/picoevb", O_RDWR);

// Example 1: malloc to malloc transfer
uint32_t *src = malloc(BUFFER_SIZE);
uint32_t *dst = malloc(BUFFER_SIZE);
memset(src, 0xAA, BUFFER_SIZE);
memset(dst, 0x00, BUFFER_SIZE);

struct picoevb_rdma_h2c2h_dma dma_params;
dma_params.src = (__u64)src;
dma_params.dst = (__u64)dst;
dma_params.len = BUFFER_SIZE;
dma_params.flags = 0;

int ret = ioctl(fd, PICOEVB_IOC_H2C2H_DMA, &dma_params);
if (ret != 0) {
    perror("DMA transfer failed");
    return -1;
}

printf("Transfer completed in %lu ns\n", dma_params.dma_time_ns);
printf("Throughput: %.2f MB/s\n",
       (double)BUFFER_SIZE / (dma_params.dma_time_ns / 1000.0));

// Example 2: CUDA to CUDA transfer (assuming pin_params_src and pin_params_dst are defined and populated)
// dma_params.src = pin_params_src.handle;
// dma_params.dst = pin_params_dst.handle;
// dma_params.len = BUFFER_SIZE;
// dma_params.flags = PICOEVB_H2C2H_DMA_FLAG_SRC_IS_CUDA |
//                    PICOEVB_H2C2H_DMA_FLAG_DST_IS_CUDA;

// ret = ioctl(fd, PICOEVB_IOC_H2C2H_DMA, &dma_params);
// if (ret == 0) {
//     printf("CUDA transfer time: %lu ns\n", dma_params.dma_time_ns);
// }

free(src);
free(dst);

```

--------------------------------

### PICOEVB_IOC_H2C_DMA - Host to Card Unidirectional DMA

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

This endpoint facilitates unidirectional DMA transfers from host memory to the FPGA's internal RAM buffer. It is primarily used for performance testing and data streaming to the FPGA.

```APIDOC
## PICOEVB_IOC_H2C_DMA - Host to Card Unidirectional DMA

### Description
Transfers data from host memory to FPGA internal RAM buffer for performance testing.

### Method
IOCTL

### Endpoint
`/dev/picoevb`

### Parameters
#### IOCTL Command
- **PICOEVB_IOC_H2C_DMA**

#### Request Body Structure
```c
struct picoevb_rdma_h2c_dma {
    __u64 src;       // Host memory source address
    __u64 dst;       // FPGA RAM offset (destination)
    __u32 len;       // Transfer length in bytes
    __u32 flags;     // Transfer flags (currently unused, set to 0)
    __u64 dma_time_ns; // Output: DMA transfer time in nanoseconds
};
```

### Request Example
```c
#include "picoevb-rdma-ioctl.h"

#define TRANSFER_SIZE (64 * 1024 * 1024)

int fd = open("/dev/picoevb", O_RDWR);

// Allocate and fill source buffer
void *src = calloc(TRANSFER_SIZE, 1);
memset(src, 0xFF, TRANSFER_SIZE);

// Transfer to FPGA RAM offset 0
struct picoevb_rdma_h2c_dma dma_params;
dma_params.src = (__u64)src;
dma_params.dst = 0;  // FPGA RAM offset
dma_params.len = TRANSFER_SIZE;
dma_params.flags = 0;

int ret = ioctl(fd, PICOEVB_IOC_H2C_DMA, &dma_params);

free(src);
close(fd);
```

### Response
#### Success Response (0 on success)
- **dma_time_ns** (uint64_t) - The duration of the DMA transfer in nanoseconds. The `ret` value from `ioctl` indicates success (0) or failure (non-zero).

#### Response Example
Upon successful transfer, `dma_params.dma_time_ns` will be populated. The example below shows how to calculate MB/s.
```c
if (ret == 0) {
    uint64_t usecs = dma_params.dma_time_ns / 1000;
    printf("Bytes: %d usecs: %lu MB/s: %.2f\n",
           TRANSFER_SIZE, usecs,
           (double)TRANSFER_SIZE / (double)usecs);
}
```
```

--------------------------------

### Unpin CUDA Memory (C)

Source: https://context7.com/nvidia/jetson-rdma-picoevb/llms.txt

Releases pinned CUDA memory using the PICOEVB_IOC_UNPIN_CUDA ioctl and the handle obtained from pinning. This operation allows the previously pinned CUDA memory to be safely freed. It requires the 'picoevb-rdma-ioctl.h' header.

```c
#include "picoevb-rdma-ioctl.h"

struct picoevb_rdma_unpin_cuda unpin_params;
unpin_params.handle = pin_params.handle; // Assuming pin_params is accessible

int ret = ioctl(fd, PICOEVB_IOC_UNPIN_CUDA, &unpin_params);
if (ret != 0) {
    perror("Failed to unpin CUDA memory");
    return -1;
}

// Now safe to free the CUDA memory
cudaFreeHost(cuda_mem);
close(fd);

```

=== COMPLETE CONTENT === This response contains all available snippets from this library. No additional content exists. Do not make further requests.