libocxl
mmio.c
Go to the documentation of this file.
1 /*
2  * Copyright 2017 International Business Machines
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "libocxl_internal.h"
18 #include "sys/mman.h"
19 #include "errno.h"
20 #include <sys/types.h>
21 #include <sys/stat.h>
22 #include <fcntl.h>
23 #include <unistd.h>
24 #include <string.h>
25 #include <stdlib.h>
26 
52 static ocxl_err register_mmio(ocxl_afu *afu, void *addr, size_t size, ocxl_mmio_type type, ocxl_mmio_h *handle)
53 {
54  int available_mmio = -1;
55 
56  // Look for an available MMIO region that has been unmapped
57  for (uint16_t mmio = 0; mmio < afu->mmio_count; mmio++) {
58  if (!afu->mmios[mmio].start) {
59  available_mmio = mmio;
60  break;
61  }
62  }
63 
64  if (available_mmio == -1) {
65  if (afu->mmio_count == afu->mmio_max_count) {
66  ocxl_err rc = grow_buffer(afu, (void **)&afu->mmios, &afu->mmio_max_count, sizeof(ocxl_mmio_area), INITIAL_MMIO_COUNT);
67  if (rc != OCXL_OK) {
68  errmsg(afu, rc, "Could not grow MMIO buffer");
69  return rc;
70  }
71  }
72 
73  available_mmio = afu->mmio_count++;
74  }
75 
76  afu->mmios[available_mmio].start = addr;
77  afu->mmios[available_mmio].length = size;
78  afu->mmios[available_mmio].type = type;
79  afu->mmios[available_mmio].afu = afu;
80 
81  *handle = &afu->mmios[available_mmio];
82 
83  TRACE(afu, "Mapped %ld bytes of %s MMIO at %p",
84  size, type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID", addr);
85 
86  return OCXL_OK;
87 }
88 
96 ocxl_err global_mmio_open(ocxl_afu *afu)
97 {
98  char path[PATH_MAX + 1];
99  int length = snprintf(path, sizeof(path), "%s/global_mmio_area", afu->sysfs_path);
100  if (length >= (int)sizeof(path)) {
101  ocxl_err rc = OCXL_NO_DEV;
102  errmsg(afu, rc, "global MMIO path truncated");
103  return rc;
104  }
105 
106  int fd = open(path, O_RDWR | O_CLOEXEC);
107  if (fd < 0) {
108  ocxl_err rc = OCXL_NO_DEV;
109  errmsg(afu, rc, "Could not open global MMIO '%s': Error %d: %s", path, errno, strerror(errno));
110  return rc;
111  }
112 
113  afu->global_mmio_fd = fd;
114 
115  return OCXL_OK;
116 }
117 
141 static ocxl_err global_mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset,
142  ocxl_mmio_h *region) // static function extraction hack
143 {
144  if (afu->global_mmio.length == 0) {
145  ocxl_err rc = OCXL_NO_MEM;
146  errmsg(afu, rc, "Cannot map Global MMIO as there is 0 bytes allocated by the AFU");
147  return rc;
148  }
149 
150  if (flags) {
152  errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
153  return rc;
154  }
155 
156  void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->global_mmio_fd, offset);
157  if (addr == MAP_FAILED) {
158  ocxl_err rc = OCXL_NO_MEM;
159  errmsg(afu, rc, "Could not map global MMIO, %d: %s", errno, strerror(errno));
160  return rc;
161  }
162 
163  ocxl_mmio_h mmio_region;
164  ocxl_err rc = register_mmio(afu, addr, size, OCXL_GLOBAL_MMIO, &mmio_region);
165  if (rc != OCXL_OK) {
166  errmsg(afu, rc, "Could not register global MMIO region");
167  munmap(addr, size);
168  return rc;
169  }
170 
171  *region = mmio_region;
172 
173  return OCXL_OK;
174 }
175 
199 static ocxl_err mmio_map(ocxl_afu *afu, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
200 {
201  if (flags) {
203  errmsg(afu, rc, "MMIO flags of 0x%llx is not supported by this version of libocxl", flags);
204  return rc;
205  }
206 
207  if (afu->fd < 0) {
209  errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been opened");
210  return rc;
211  }
212 
213  if (!afu->attached) {
215  errmsg(afu, rc, "Could not map per-PASID MMIO as the AFU has not been attached");
216  return rc;
217  }
218 
219  void *addr = mmap(NULL, size, prot, MAP_SHARED, afu->fd, offset);
220  if (addr == MAP_FAILED) {
221  ocxl_err rc = OCXL_NO_MEM;
222  errmsg(afu, rc, "Could not map per-PASID MMIO: %d: %s", errno, strerror(errno));
223  return rc;
224  }
225 
226  ocxl_mmio_h mmio_region;
227  ocxl_err rc = register_mmio(afu, addr, size, OCXL_PER_PASID_MMIO, &mmio_region);
228  if (rc != OCXL_OK) {
229  errmsg(afu, rc, "Could not register global MMIO region", afu->identifier.afu_name);
230  munmap(addr, size);
231  return rc;
232  }
233 
234  *region = mmio_region;
235 
236  return OCXL_OK;
237 }
238 
260 ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags,
261  off_t offset, ocxl_mmio_h *region)
262 {
264 
265  if (size == 0) {
266  switch (type) {
267  case OCXL_PER_PASID_MMIO:
268  size = afu->per_pasid_mmio.length;
269  break;
270  case OCXL_GLOBAL_MMIO:
271  size = afu->global_mmio.length;
272  break;
273  }
274 
275  size -= offset;
276  }
277 
278  switch (type) {
279  case OCXL_GLOBAL_MMIO:
280  if (offset + size > afu->global_mmio.length) {
281  rc = OCXL_NO_MEM;
282  errmsg(afu, rc, "Offset(%#x) + size(%#x) of global MMIO map request exceeds available size of %#x",
283  offset, size, afu->global_mmio.length);
284  return rc;
285  }
286  return global_mmio_map(afu, size, prot, flags, offset, region);
287 
288  case OCXL_PER_PASID_MMIO:
289  if (offset + size > afu->per_pasid_mmio.length) {
290  rc = OCXL_NO_MEM;
291  errmsg(afu, rc, "Offset(%#x) + size(%#x) of per-pasid MMIO map request exceeds available size of %#x",
292  offset, size, afu->global_mmio.length);
293  return rc;
294  }
295  return mmio_map(afu, size, prot, flags, offset, region);
296 
297  default:
298  errmsg(afu, rc, "Unknown MMIO type %d", type);
299  return rc;
300  }
301 }
302 
322 {
323  return ocxl_mmio_map_advanced(afu, type, 0, PROT_READ | PROT_WRITE, 0, 0, region);
324 }
325 
334 {
335  if (!region->start) {
336  return;
337  }
338 
339  munmap(region->start, region->length);
340  region->start = NULL;
341 }
342 
358 {
359  switch (type) {
360  case OCXL_GLOBAL_MMIO:
361  return afu->global_mmio_fd;
362 
363  case OCXL_PER_PASID_MMIO:
364  return afu->fd;
365 
366  default:
367  errmsg(afu, OCXL_INVALID_ARGS, "Unknown MMIO type %d", type);
368  return -1;
369  }
370 }
371 
381 {
382  switch(type) {
383  case OCXL_GLOBAL_MMIO:
384  return afu->global_mmio.length;
385 
386  case OCXL_PER_PASID_MMIO:
387  return afu->per_pasid_mmio.length;
388 
389  default:
390  errmsg(afu, OCXL_INVALID_ARGS, "Invalid MMIO area requested '%d'", type);
391  return 0;
392  }
393 }
394 
395 
408 ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
409 {
410  if (!region->start) {
412  errmsg(region->afu, rc, "MMIO region has already been unmapped");
413  return rc;
414  }
415 
416  *address = region->start;
417  *size = region->length;
418 
419  return OCXL_OK;
420 }
421 
422 
434 inline static ocxl_err mmio_check(ocxl_mmio_h region, off_t offset, size_t size)
435 {
436  if (!region) {
438  errmsg(NULL, rc, "MMIO region is invalid");
439  return rc;
440  }
441 
442  if (!region->start) {
444  errmsg(region->afu, rc, "MMIO region has already been unmapped");
445  return rc;
446  }
447 
448  if (offset >= (off_t)(region->length - (size - 1))) {
450  errmsg(region->afu, rc, "%s MMIO access of 0x%016lx exceeds limit of 0x%016lx",
451  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
452  offset, region->length);
453  return rc;
454  }
455 
456  return OCXL_OK;
457 }
458 
477 inline static ocxl_err mmio_read32_native(ocxl_mmio_h region, off_t offset, uint32_t *out)
478 {
479  ocxl_err ret = mmio_check(region, offset, 4);
480  if (ret != OCXL_OK) {
481  return ret;
482  }
483 
484  __sync_synchronize();
485  *out = *(volatile uint32_t *)(region->start + offset);
486  __sync_synchronize();
487 
488  TRACE(region->afu, "%s MMIO Read32@0x%04lx=0x%08x",
489  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
490  offset, *out);
491 
492  return OCXL_OK;
493 }
494 
513 inline static ocxl_err mmio_read64_native(ocxl_mmio_h region, off_t offset, uint64_t *out)
514 {
515  ocxl_err ret = mmio_check(region, offset, 8);
516  if (ret != OCXL_OK) {
517  return ret;
518  }
519 
520  __sync_synchronize();
521  *out = *(volatile uint64_t *)(region->start + offset);
522  __sync_synchronize();
523 
524  TRACE(region->afu, "%s MMIO Read64@0x%04lx=0x%016lx",
525  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
526  offset, *out);
527 
528  return OCXL_OK;
529 }
530 
551 inline static ocxl_err mmio_write32_native(ocxl_mmio_h region, off_t offset, uint32_t value)
552 {
553  ocxl_err ret = mmio_check(region, offset, 4);
554  if (ret != OCXL_OK) {
555  return ret;
556  }
557 
558  TRACE(region->afu, "%s MMIO Write32@0x%04lx=0x%08x",
559  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
560  offset, value);
561 
562  volatile uint32_t *addr = (uint32_t *)(region->start + offset);
563 
564  __sync_synchronize();
565  *addr = value;
566  __sync_synchronize();
567 
568  return OCXL_OK;
569 }
570 
589 inline static ocxl_err mmio_write64_native(ocxl_mmio_h region, off_t offset, uint64_t value)
590 {
591  ocxl_err ret = mmio_check(region, offset, 8);
592  if (ret != OCXL_OK) {
593  return ret;
594  }
595 
596  TRACE(region->afu, "%s MMIO Write64@0x%04lx=0x%016lx",
597  region->type == OCXL_GLOBAL_MMIO ? "Global" : "Per-PASID",
598  offset, value);
599 
600  volatile uint64_t *addr = (uint64_t *)(region->start + offset);
601 
602  __sync_synchronize();
603  *addr = value;
604  __sync_synchronize();
605 
606  return OCXL_OK;
607 }
608 
627 ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
628 {
629  uint32_t val;
630  ocxl_err ret = mmio_read32_native(mmio, offset, &val);
631 
632  if (UNLIKELY(ret != OCXL_OK)) {
633  return ret;
634  }
635 
636  switch (endian) {
638  *out = be32toh(val);
639  break;
640 
642  *out = le32toh(val);
643  break;
644  default:
645  *out = val;
646  break;
647  }
648 
649  return OCXL_OK;
650 }
651 
671 ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
672 {
673  uint64_t val;
674  ocxl_err ret = mmio_read64_native(mmio, offset, &val);
675 
676  if (UNLIKELY(ret != OCXL_OK)) {
677  return ret;
678  }
679 
680  switch (endian) {
682  *out = be64toh(val);
683  break;
684 
686  *out = le64toh(val);
687  break;
688  default:
689  *out = val;
690  break;
691  }
692 
693  return OCXL_OK;
694 }
695 
715 ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
716 {
717  switch (endian) {
719  value = htobe32(value);
720  break;
721 
723  value = htole32(value);
724  break;
725  default:
726  break;
727  }
728 
729  return mmio_write32_native(mmio, offset, value);
730 }
731 
750 ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
751 {
752  switch (endian) {
754  value = htobe64(value);
755  break;
756 
758  value = htole64(value);
759  break;
760  default:
761  break;
762  }
763 
764  return mmio_write64_native(mmio, offset, value);
765 }
766 
767 
AFU data is little-endian.
Definition: libocxl.h:49
ocxl_err ocxl_mmio_map_advanced(ocxl_afu_h afu, ocxl_mmio_type type, size_t size, int prot, uint64_t flags, off_t offset, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition: mmio.c:260
ocxl_err ocxl_mmio_read32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t *out)
Read a 32-bit value from an AFU's MMIO region & convert endianness.
Definition: mmio.c:627
ocxl_err ocxl_mmio_get_info(ocxl_mmio_h region, void **address, size_t *size)
Get the address & size of a mapped MMIO region.
Definition: mmio.c:408
ocxl_err
Potential return values from ocxl_* functions.
Definition: libocxl.h:92
size_t ocxl_mmio_size(ocxl_afu_h afu, ocxl_mmio_type type)
Get the size of an MMIO region for an AFU.
Definition: mmio.c:380
The action requested falls outside the permitted area.
Definition: libocxl.h:100
The call requires an open context on the AFU.
Definition: libocxl.h:96
ocxl_err ocxl_mmio_read64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t *out)
Read a 64-bit value from an AFU's MMIO region & convert endianness.
Definition: mmio.c:671
void ocxl_mmio_unmap(ocxl_mmio_h region)
Unmap an MMIO region from an AFU.
Definition: mmio.c:333
ocxl_err ocxl_mmio_map(ocxl_afu_h afu, ocxl_mmio_type type, ocxl_mmio_h *region)
Map an MMIO area of an AFU.
Definition: mmio.c:321
One or more arguments are invalid.
Definition: libocxl.h:102
ocxl_endian
Defines the endianness of an AFU MMIO area.
Definition: libocxl.h:47
The OpenCAPI device is not available.
Definition: libocxl.h:95
ocxl_mmio_type
Defines the type of an MMIO area.
Definition: libocxl.h:56
ocxl_err ocxl_mmio_write32(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint32_t value)
Convert endianness and write a 32-bit value to an AFU's MMIO region.
Definition: mmio.c:715
struct ocxl_afu * ocxl_afu_h
A handle for an AFU.
Definition: libocxl.h:74
AFU data is big-endian.
Definition: libocxl.h:48
ocxl_err global_mmio_open(ocxl_afu *afu)
Open the global MMIO descriptor on an AFU.
Definition: mmio.c:96
An out of memory error occurred.
Definition: libocxl.h:94
ocxl_err ocxl_mmio_write64(ocxl_mmio_h mmio, off_t offset, ocxl_endian endian, uint64_t value)
Convert endianness and write a 64-bit value to an AFU's MMIO region.
Definition: mmio.c:750
int ocxl_mmio_get_fd(ocxl_afu_h afu, ocxl_mmio_type type)
Get a file descriptor for an MMIO area of an AFU.
Definition: mmio.c:357
struct ocxl_mmio_area * ocxl_mmio_h
A handle for an MMIO region on an AFU.
Definition: libocxl.h:86
The call succeeded.
Definition: libocxl.h:93