Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

hmem: Define ofi_hmem_put_dmabuf_fd #10716

Merged
merged 7 commits into from
Jan 23, 2025
Merged

hmem: Define ofi_hmem_put_dmabuf_fd #10716

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e540e80
src/hmem: Define ofi_hmem_put_dmabuf_fd
iziemba Jan 20, 2025
7c74473
hmem/rocr: Support ofi_hmem_put_dmabuf_fd()
iziemba Jan 20, 2025
c93c257
hmem/cuda: Support ofi_hmem_put_dmabuf_fd()
iziemba Jan 22, 2025
50d7a22
prov/cxi: Integrate with ofi_hmem_put_dmabuf_fd
iziemba Jan 21, 2025
d1e73d0
prov/cxi: Test ROCR with DMA buf offset
iziemba Jan 21, 2025
fb19b96
prov/cxi: Test ROCR with DMA buf FD recycling
iziemba Jan 21, 2025
d28282f
prov/cxi: Test CUDA with DMA buf FD recycling
iziemba Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions include/ofi_hmem.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ struct ofi_hmem_ops {
const void *src, size_t size);
int (*get_dmabuf_fd)(const void *addr, uint64_t size, int *fd,
uint64_t *offset);
int (*put_dmabuf_fd)(int fd);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not just name it as close_dmabuf_fd ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I chose put_dmabuf_fd to semantically align with get_dmabuf_fd.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is there any hmem type where a get_dmabuf_fd does not implicitly increase the open count on the file? if not, I'd recommend going the other way and renaming get to open so that it's clear that it's not simply resolving an already-open file descriptor, but is actually opening a resource which must be eventually closed.

Copy link
Contributor

@j-xiong j-xiong Jan 23, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, the ZE hmem type always return the same fd.

};

extern struct ofi_hmem_ops hmem_ops[];
Expand Down Expand Up @@ -357,6 +358,11 @@ static inline int ofi_hmem_no_get_dmabuf_fd(const void *addr, uint64_t size,
return -FI_ENOSYS;
}

static inline int ofi_hmem_no_put_dmabuf_fd(int fd)
{
return -FI_ENOSYS;
}

static inline bool ofi_hmem_p2p_disabled(void)
{
return ofi_hmem_disable_p2p;
Expand Down Expand Up @@ -450,5 +456,6 @@ int ofi_hmem_dev_reg_copy_from_hmem(enum fi_hmem_iface iface, uint64_t handle,
void *dest, const void *src, size_t size);
int ofi_hmem_get_dmabuf_fd(enum fi_hmem_iface, const void *addr, uint64_t size,
int *fd, uint64_t *offset);
int ofi_hmem_put_dmabuf_fd(enum fi_hmem_iface iface, int fd);

#endif /* _OFI_HMEM_H_ */
11 changes: 11 additions & 0 deletions src/hmem.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ struct ofi_hmem_ops hmem_ops[] = {
.dev_reg_copy_to_hmem = ofi_hmem_system_dev_reg_copy,
.dev_reg_copy_from_hmem = ofi_hmem_system_dev_reg_copy,
.get_dmabuf_fd = ofi_hmem_no_get_dmabuf_fd,
.put_dmabuf_fd = ofi_hmem_no_put_dmabuf_fd,
},
[FI_HMEM_CUDA] = {
.initialized = false,
Expand All @@ -167,6 +168,7 @@ struct ofi_hmem_ops hmem_ops[] = {
.dev_reg_copy_to_hmem = cuda_dev_reg_copy_to_hmem,
.dev_reg_copy_from_hmem = cuda_dev_reg_copy_from_hmem,
.get_dmabuf_fd = cuda_get_dmabuf_fd,
.put_dmabuf_fd = ofi_hmem_no_put_dmabuf_fd,
},
[FI_HMEM_ROCR] = {
.initialized = false,
Expand All @@ -193,6 +195,7 @@ struct ofi_hmem_ops hmem_ops[] = {
.dev_reg_copy_to_hmem = rocr_dev_reg_copy_to_hmem,
.dev_reg_copy_from_hmem = rocr_dev_reg_copy_from_hmem,
.get_dmabuf_fd = rocr_hmem_get_dmabuf_fd,
.put_dmabuf_fd = ofi_hmem_no_put_dmabuf_fd,
},
[FI_HMEM_ZE] = {
.initialized = false,
Expand All @@ -219,6 +222,7 @@ struct ofi_hmem_ops hmem_ops[] = {
.dev_reg_copy_to_hmem = ze_dev_reg_copy_to_hmem,
.dev_reg_copy_from_hmem = ze_dev_reg_copy_from_hmem,
.get_dmabuf_fd = ze_hmem_get_dmabuf_fd,
.put_dmabuf_fd = ofi_hmem_no_put_dmabuf_fd,
},
[FI_HMEM_NEURON] = {
.initialized = false,
Expand All @@ -244,6 +248,7 @@ struct ofi_hmem_ops hmem_ops[] = {
.dev_reg_copy_to_hmem = ofi_hmem_no_dev_reg_copy_to_hmem,
.dev_reg_copy_from_hmem = ofi_hmem_no_dev_reg_copy_from_hmem,
.get_dmabuf_fd = neuron_get_dmabuf_fd,
.put_dmabuf_fd = ofi_hmem_no_put_dmabuf_fd,
},
[FI_HMEM_SYNAPSEAI] = {
.initialized = false,
Expand All @@ -269,6 +274,7 @@ struct ofi_hmem_ops hmem_ops[] = {
.dev_reg_copy_to_hmem = ofi_hmem_no_dev_reg_copy_to_hmem,
.dev_reg_copy_from_hmem = ofi_hmem_no_dev_reg_copy_from_hmem,
.get_dmabuf_fd = synapseai_get_dmabuf_fd,
.put_dmabuf_fd = ofi_hmem_no_put_dmabuf_fd,
},
};

Expand Down Expand Up @@ -820,3 +826,8 @@ int ofi_hmem_get_dmabuf_fd(enum fi_hmem_iface iface, const void *addr,
{
return hmem_ops[iface].get_dmabuf_fd(addr, size, fd, offset);
}

int ofi_hmem_put_dmabuf_fd(enum fi_hmem_iface iface, int fd)
{
return hmem_ops[iface].put_dmabuf_fd(fd);
}