Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ compiles a Rust kernel to PTX, launches it on the GPU, and prints
| Example | Description |
|----------------------|--------------------------------------------------------------------------|
| `vecadd` | Vector addition -- canonical first example |
| `cudarc_slice` | Memory owned by cudarc passed straight to a generated launcher |
| `host_closure` | Generic kernels with closures passed from host |
| `generic` | Generic kernels with monomorphization (`scale<T>`) |
| `ord_cmp` | Device-side `Ord::cmp` lowering for signed and unsigned integers |
Expand Down
15 changes: 8 additions & 7 deletions crates/cuda-host/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,9 @@ Kernel parameters are mapped into host launch parameters:

| Kernel parameter | Host method parameter |
|------------------|-----------------------|
| `&[T]` | `&DeviceBuffer<T>` |
| `&mut [T]` | `&mut DeviceBuffer<T>` |
| `DisjointSlice<T>` | `&mut DeviceBuffer<T>` |
| `&[T]` | `&impl KernelSliceArg<Elem = T>` |
| `&mut [T]` | `&mut impl KernelSliceArgMut<Elem = T>` |
| `DisjointSlice<T>` | `&mut impl KernelSliceArgMut<Elem = T>` |
| `Uniform<T>` | `T` |
| `Copy` scalar, struct, closure, or raw pointer | unchanged |

Expand All @@ -82,8 +82,8 @@ what makes the value uniform: one marshalled value reaches every thread of the
launch. The device side receives the witness, which is what device APIs needing
a launch-uniform scalar require in place of an `unsafe` assertion.

A slice whose index space carries a runtime row width takes `RowWidth<T>`, which
binds the width to that slice for the launch. The same reasoning applies and for
A slice whose index space carries a runtime row width takes `RowWidth`, which
binds the width to any writable slice view for the launch. The same reasoning applies and for
the same reason, one step earlier: the row width reaches the device as one word the
host wrote, so `DisjointSlice::tile_2d32_rt` needs neither a stride argument nor
an `unsafe` assertion. The owned async launches take `RowWidthOwned<B>`.
Expand Down Expand Up @@ -214,8 +214,9 @@ let launch = unsafe {
launch.sync()?;
```

For async launches, device-slice parameters accept either `DeviceBuffer<T>` or
`cuda_async::simt::device_box::DeviceBox<[T]>`. The mutable
Device-slice parameters accept any `KernelSliceArg` / `KernelSliceArgMut`
implementor: `DeviceBuffer<T>`, `cuda_async::simt::device_box::DeviceBox<[T]>`,
or a caller-defined view. The mutable
`AsyncKernelLaunchBuilder` collects arguments and options. Finalizing it with a
raw configuration is unsafe and produces an immutable `AsyncKernelLaunch<'_>`;
geometry cannot be changed after that point. Rust keeps referenced buffers and
Expand Down
52 changes: 22 additions & 30 deletions crates/cuda-host/src/launch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
use std::ffi::c_void;
#[cfg(feature = "async")]
use std::future::IntoFuture;
#[cfg(feature = "async")]
use std::sync::Arc;

// =============================================================================
Expand Down Expand Up @@ -150,8 +149,8 @@ pub fn push_kernel_scalar<T: KernelScalar>(args: &mut Vec<*mut c_void>, value: &
/// parameters such as `&[T]`.
#[inline]
#[doc(hidden)]
pub fn read_only_device_buffer_arg<T>(
buffer: &cuda_core::DeviceBuffer<T>,
pub fn read_only_device_buffer_arg<B: KernelSliceArg + ?Sized>(
buffer: &B,
) -> (cuda_core::sys::CUdeviceptr, u64) {
(buffer.cu_deviceptr(), buffer.len() as u64)
}
Expand All @@ -160,10 +159,10 @@ pub fn read_only_device_buffer_arg<T>(
/// parameters such as `&mut [T]` and `DisjointSlice<T>`.
#[inline]
#[doc(hidden)]
pub fn writable_device_buffer_arg<T>(
buffer: &mut cuda_core::DeviceBuffer<T>,
pub fn writable_device_buffer_arg<B: KernelSliceArgMut + ?Sized>(
buffer: &mut B,
) -> (cuda_core::sys::CUdeviceptr, u64) {
(buffer.cu_deviceptr(), buffer.len() as u64)
read_only_device_buffer_arg(buffer)
}

/// Pushes a device slice argument pair into a CUDA driver argument list.
Expand All @@ -183,7 +182,8 @@ pub fn push_kernel_device_slice(
args.push(len as *mut u64 as *mut c_void);
}

/// A device buffer together with the row width the kernel will index it by.
/// A writable slice view (a `DeviceBuffer` unless another [`KernelSliceArgMut`]
/// is given) together with the row width the kernel will index it by.
///
/// Kernels whose index space fixes the row width in the type need nothing
/// here. A kernel taking `DisjointSlice<T, RuntimeRowMajorTiles<R, C>>` or
Expand All @@ -200,17 +200,17 @@ pub fn push_kernel_device_slice(
/// ```rust,ignore
/// kernels::sgemm(&stream, cfg, m, k, alpha, &a, &b, RowWidth::new(&mut c, n))?;
/// ```
pub struct RowWidth<'a, T> {
buffer: &'a mut cuda_core::DeviceBuffer<T>,
pub struct RowWidth<'a, B: KernelSliceArgMut + ?Sized> {
buffer: &'a mut B,
width: u32,
}

impl<'a, T> RowWidth<'a, T> {
impl<'a, B: KernelSliceArgMut + ?Sized> RowWidth<'a, B> {
/// Bind `width` as the row width for this launch's view of `buffer`.
///
/// `width` is a count of elements, not bytes.
#[inline]
pub fn new(buffer: &'a mut cuda_core::DeviceBuffer<T>, width: u32) -> Self {
pub fn new(buffer: &'a mut B, width: u32) -> Self {
RowWidth { buffer, width }
}

Expand Down Expand Up @@ -281,14 +281,11 @@ impl<B> RowWidthOwned<B> {
/// writable slice parameters whose index space carries a runtime row width.
#[inline]
#[doc(hidden)]
pub fn row_width_device_buffer_arg<T>(
bound: RowWidth<'_, T>,
pub fn row_width_device_buffer_arg<B: KernelSliceArgMut + ?Sized>(
bound: RowWidth<'_, B>,
) -> (cuda_core::sys::CUdeviceptr, u64, u32) {
(
bound.buffer.cu_deviceptr(),
bound.buffer.len() as u64,
bound.width,
)
let (ptr, len) = read_only_device_buffer_arg(bound.buffer);
(ptr, len, bound.width)
}

/// Pushes a row-width device slice argument triple into a driver argument
Expand All @@ -312,10 +309,10 @@ pub fn push_kernel_row_width_device_slice(
}

// =============================================================================
// Typed Async Kernel Arguments
// Typed Kernel Slice Arguments
// =============================================================================

/// A typed device allocation that can be passed to an async kernel launch as a
/// A typed device allocation that can be passed to a kernel launch as a
/// read-only device slice.
///
/// # Safety
Expand Down Expand Up @@ -343,7 +340,6 @@ pub fn push_kernel_row_width_device_slice(
/// fn len(&self) -> usize { 1_000_000 }
/// }
/// ```
#[cfg(feature = "async")]
pub unsafe trait KernelSliceArg {
/// Element type stored in the allocation.
type Elem;
Expand All @@ -360,7 +356,7 @@ pub unsafe trait KernelSliceArg {
}
}

/// A typed device allocation that can be passed to an async kernel launch as a
/// A typed device allocation that can be passed to a kernel launch as a
/// writable device slice.
///
/// # Safety
Expand All @@ -370,10 +366,8 @@ pub unsafe trait KernelSliceArg {
/// this rule: the implementor must own exclusive device-write authority for
/// the entire reported element range for the lifetime of the mutable borrow or
/// owned operation.
#[cfg(feature = "async")]
pub unsafe trait KernelSliceArgMut: KernelSliceArg {}

#[cfg(feature = "async")]
// SAFETY: DeviceBuffer owns the reported allocation and keeps its CUDA context
// alive; its pointer and length accessors describe that allocation exactly.
unsafe impl<T> KernelSliceArg for cuda_core::DeviceBuffer<T> {
Expand All @@ -388,14 +382,13 @@ unsafe impl<T> KernelSliceArg for cuda_core::DeviceBuffer<T> {
}
}

#[cfg(feature = "async")]
// SAFETY: &mut DeviceBuffer provides exclusive host authority to launch device
// writes through this adapter for the duration of the operation.
unsafe impl<T> KernelSliceArgMut for cuda_core::DeviceBuffer<T> {}

#[cfg(feature = "async")]
// SAFETY: DeviceBox owns the reported allocation and its raw constructor
// requires the pointer, element count, and device ordinal to be truthful.
#[cfg(feature = "async")]
unsafe impl<T: Send> KernelSliceArg for cuda_async::simt::device_box::DeviceBox<[T]> {
type Elem = T;

Expand All @@ -408,12 +401,11 @@ unsafe impl<T: Send> KernelSliceArg for cuda_async::simt::device_box::DeviceBox<
}
}

#[cfg(feature = "async")]
// SAFETY: &mut DeviceBox provides exclusive host authority to launch device
// writes through this adapter for the duration of the operation.
#[cfg(feature = "async")]
unsafe impl<T: Send> KernelSliceArgMut for cuda_async::simt::device_box::DeviceBox<[T]> {}

#[cfg(feature = "async")]
// SAFETY: Arc only extends the lifetime of B and delegates both truthful
// accessors unchanged to B's unsafe implementation.
unsafe impl<B> KernelSliceArg for Arc<B>
Expand Down Expand Up @@ -679,9 +671,9 @@ pub fn push_async_writable_device_slice<B>(
/// whose index space carries a runtime row width.
#[doc(hidden)]
#[cfg(feature = "async")]
pub fn push_async_row_width_device_slice<T>(
pub fn push_async_row_width_device_slice<B: KernelSliceArgMut + ?Sized>(
launch: &mut cuda_async::simt::launch::AsyncKernelLaunchBuilder<'_>,
bound: RowWidth<'_, T>,
bound: RowWidth<'_, B>,
) {
let (ptr, len, width) = row_width_device_buffer_arg(bound);
launch.push_scalar_arg(ptr);
Expand Down
21 changes: 10 additions & 11 deletions crates/cuda-host/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,24 +101,23 @@ pub use kernel_family::{
NoKernelSelectionCache, SelectedVariant, SelectionMode, SelectionSource,
};
pub use launch::{
CudaKernel, GenericCudaKernel, HasLength, KernelScalar, ReadOnly, RowWidth, RowWidthOwned,
Scalar, WriteOnly, push_kernel_device_slice, push_kernel_row_width_device_slice,
push_kernel_scalar, read_only_device_buffer_arg, row_width_device_buffer_arg,
writable_device_buffer_arg,
CudaKernel, GenericCudaKernel, HasLength, KernelScalar, KernelSliceArg, KernelSliceArgMut,
ReadOnly, RowWidth, RowWidthOwned, Scalar, WriteOnly, push_kernel_device_slice,
push_kernel_row_width_device_slice, push_kernel_scalar, read_only_device_buffer_arg,
row_width_device_buffer_arg, writable_device_buffer_arg,
};
#[doc(hidden)]
pub use type_id::__intern_generic_kernel_name;
pub use type_id::{type_id_u128, type_id_u128_of_val};

#[cfg(feature = "async")]
pub use launch::{
KernelSliceArg, KernelSliceArgMut, PreparedAsyncKernelLaunch, PreparedOwnedAsyncKernelLaunch,
load_cuda_module_from_async_context, load_kernel_module_async, new_async_kernel_launch_builder,
new_owned_async_kernel_launch, new_prepared_async_kernel_launch,
new_prepared_owned_async_kernel_launch, push_async_kernel_scalar,
push_async_owned_row_width_device_slice, push_async_read_only_device_slice,
push_async_row_width_device_slice, push_async_writable_device_slice,
set_async_kernel_cluster_dim, set_async_kernel_cooperative,
PreparedAsyncKernelLaunch, PreparedOwnedAsyncKernelLaunch, load_cuda_module_from_async_context,
load_kernel_module_async, new_async_kernel_launch_builder, new_owned_async_kernel_launch,
new_prepared_async_kernel_launch, new_prepared_owned_async_kernel_launch,
push_async_kernel_scalar, push_async_owned_row_width_device_slice,
push_async_read_only_device_slice, push_async_row_width_device_slice,
push_async_writable_device_slice, set_async_kernel_cluster_dim, set_async_kernel_cooperative,
};

/// The shared async crate, re-exported whole. Its root is cutile's Tile API;
Expand Down
58 changes: 58 additions & 0 deletions crates/cuda-host/tests/cuda_module_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,14 @@ mod kernels {
pub fn contracted_sizes(n: u32, input: &[u32]) {
let _ = (n, input);
}

/// A runtime row width binds through `RowWidth` on the host.
#[kernel]
pub fn row_width_bound(
output: cuda_device::DisjointSlice<u32, cuda_device::thread::Runtime2DIndex>,
) {
let _ = output;
}
}

#[cfg(feature = "async")]
Expand Down Expand Up @@ -434,9 +442,59 @@ fn generated_prepared_owned_async_methods_are_immutable_operations(
Ok(())
}

/// A caller-defined view that upholds the slice contracts. Sync launchers
/// now take slice parameters as `impl KernelSliceArg(Mut)`, as the async ones
/// already did, so such a type is accepted wherever a `DeviceBuffer` is.
struct CallerView<'a, T> {
buffer: &'a mut DeviceBuffer<T>,
}

// SAFETY: the view forwards the pointer and element count of a live
// `DeviceBuffer` that it borrows exclusively for its whole lifetime.
unsafe impl<T> cuda_host::KernelSliceArg for CallerView<'_, T> {
type Elem = T;

fn cu_deviceptr(&self) -> cuda_core::sys::CUdeviceptr {
self.buffer.cu_deviceptr()
}

fn len(&self) -> usize {
self.buffer.len()
}
}

// SAFETY: the exclusive borrow of the buffer is the device-write authority.
unsafe impl<T> cuda_host::KernelSliceArgMut for CallerView<'_, T> {}

/// Every sync slice shape: read-only, writable, row-width bound, and a
/// checked launcher whose `requires` clause reads `.len()` through the trait.
unsafe fn generated_sync_methods_accept_caller_defined_views(
module: &kernels::LoadedModule,
stream: &CudaStream,
config: LaunchConfig,
input: &CallerView<'_, f32>,
output: &mut CallerView<'_, f32>,
input_u32: &CallerView<'_, u32>,
output_u32: &mut CallerView<'_, u32>,
) -> Result<(), cuda_core::LaunchContractError> {
let params = AffineParams {
scale: 2.0,
bias: 1.0,
};
let raw = core::ptr::null::<f32>();
unsafe {
module.scalar_args(stream, config, 2.0, params, raw, input, output)?;
module.row_width_bound(stream, config, cuda_host::RowWidth::new(output_u32, 8))?;
}
let sizes = module.prepare_contracted_sizes(LaunchConfig1D::new(1, 64, 0))?;
module.contracted_sizes(stream, &sizes, 4, input_u32)?;
Ok(())
}

#[test]
fn generated_cuda_module_api_typechecks() {
let _ = generated_methods_accept_kernel_scalar_types;
let _ = generated_sync_methods_accept_caller_defined_views;
let _ = generated_prepared_methods_bind_the_exact_kernel_and_specialization;
#[cfg(feature = "async")]
let _ = generated_async_methods_accept_borrowed_buffers;
Expand Down
5 changes: 3 additions & 2 deletions crates/cuda-macros/src/cuda_module/contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -627,8 +627,9 @@ fn validate_requires_operand(expr: &Expr, params: &[CudaModuleParam]) -> syn::Re
/// different host type.
#[derive(Clone, Copy)]
pub(super) enum RequiresLenAccess {
/// Sync prepared launcher: slice parameters are `&DeviceBuffer<T>` or
/// `&mut DeviceBuffer<T>`, so `.len()` resolves to the inherent method.
/// Sync prepared launcher: slice parameters are `&impl KernelSliceArg` or
/// `&mut impl KernelSliceArgMut` (or `RowWidth`, which has an inherent
/// `.len()`), so `.len()` resolves without a `use`.
SyncBuffer,
/// Async prepared launcher: slice parameters are `&impl KernelSliceArg`
/// or `&mut impl KernelSliceArgMut`.
Expand Down
45 changes: 20 additions & 25 deletions crates/cuda-macros/src/cuda_module/model.rs
Original file line number Diff line number Diff line change
Expand Up @@ -164,45 +164,40 @@ fn cuda_module_host_type(
) -> syn::Result<(TokenStream2, TokenStream2, CudaModuleParamMarshal)> {
let async_lifetime = cuda_module_async_lifetime();
if let Some((elem_ty, mutable)) = cuda_module_slice_elem(ty) {
let sync_host_ty = if mutable {
quote! { &mut ::cuda_core::DeviceBuffer<#elem_ty> }
} else {
quote! { &::cuda_core::DeviceBuffer<#elem_ty> }
};
let (async_host_ty, marshal) = if mutable {
let elem_ty = quote! { #elem_ty };
let (view, marshal) = if mutable {
(
quote! { &#async_lifetime mut impl ::cuda_host::KernelSliceArgMut<Elem = #elem_ty> },
CudaModuleParamMarshal::WritableDeviceBuffer {
elem_ty: quote! { #elem_ty },
},
quote! { impl ::cuda_host::KernelSliceArgMut<Elem = #elem_ty> },
CudaModuleParamMarshal::WritableDeviceBuffer { elem_ty },
)
} else {
(
quote! { &#async_lifetime impl ::cuda_host::KernelSliceArg<Elem = #elem_ty> },
CudaModuleParamMarshal::ReadOnlyDeviceBuffer {
elem_ty: quote! { #elem_ty },
},
quote! { impl ::cuda_host::KernelSliceArg<Elem = #elem_ty> },
CudaModuleParamMarshal::ReadOnlyDeviceBuffer { elem_ty },
)
};
return Ok((sync_host_ty, async_host_ty, marshal));
let mutability = mutable.then(|| quote! { mut });
return Ok((
quote! { &#mutability #view },
quote! { &#async_lifetime #mutability #view },
marshal,
));
}

if let Some(elem_ty) = cuda_module_disjoint_slice_elem(ty) {
let elem_ty = quote! { #elem_ty };
let view = quote! { impl ::cuda_host::KernelSliceArgMut<Elem = #elem_ty> };
if cuda_module_disjoint_slice_has_row_width(ty) {
return Ok((
quote! { ::cuda_host::RowWidth<'_, #elem_ty> },
quote! { ::cuda_host::RowWidth<#async_lifetime, #elem_ty> },
CudaModuleParamMarshal::RowWidthDeviceBuffer {
elem_ty: quote! { #elem_ty },
},
quote! { ::cuda_host::RowWidth<'_, #view> },
quote! { ::cuda_host::RowWidth<#async_lifetime, #view + Send> },
CudaModuleParamMarshal::RowWidthDeviceBuffer { elem_ty },
));
}
return Ok((
quote! { &mut ::cuda_core::DeviceBuffer<#elem_ty> },
quote! { &#async_lifetime mut impl ::cuda_host::KernelSliceArgMut<Elem = #elem_ty> },
CudaModuleParamMarshal::WritableDeviceBuffer {
elem_ty: quote! { #elem_ty },
},
quote! { &mut #view },
quote! { &#async_lifetime mut #view },
CudaModuleParamMarshal::WritableDeviceBuffer { elem_ty },
));
}

Expand Down
Loading