Line data    Source code

       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2020 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  */
       5             : 
       6             : #ifndef __IDXD_H__
       7             : #define __IDXD_H__
       8             : 
       9             : #include "spdk/stdinc.h"
      10             : 
      11             : #include "spdk/idxd.h"
      12             : #include "spdk/queue.h"
      13             : #include "spdk/mmio.h"
      14             : #include "spdk/idxd_spec.h"
      15             : 
      16             : #ifdef __cplusplus
      17             : extern "C" {
      18             : #endif
      19             : 
      20             : /* TODO: get the gcc intrinsic to work. */
      21             : #define nop() asm volatile ("nop")
      22           0 : static inline void movdir64b(void *dst, const void *src)
      23             : {
      24           0 :         asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
      25           0 :                      : "=m"(*(char *)dst)
      26           0 :                      : "d"(src), "a"(dst));
      27           0 : }
      28             : 
      29             : #define IDXD_REGISTER_TIMEOUT_US                50
      30             : #define IDXD_DRAIN_TIMEOUT_US                   500000
      31             : 
      32             : #define WQ_MODE_DEDICATED       1
      33             : 
      34             : /* TODO: consider setting the max per batch limit via RPC. */
      35             : 
      36             : #define LOG2_WQ_MAX_XFER        30 /* 2^30 = 1073741824 */
      37             : #define WQ_PRIORITY_1           1
      38             : #define IDXD_MAX_QUEUES         64
      39             : 
      40             : enum idxd_dev {
      41             :         IDXD_DEV_TYPE_DSA       = 0,
      42             :         IDXD_DEV_TYPE_IAA       = 1,
      43             : };
      44             : 
      45             : /* Each pre-allocated batch structure goes on a per channel list and
      46             :  * contains the memory for both user descriptors.
      47             :  */
      48             : struct idxd_batch {
      49             :         struct idxd_hw_desc             *user_desc;
      50             :         struct idxd_ops                 *user_ops;
      51             :         uint64_t                        user_desc_addr;
      52             :         uint16_t                        index;
      53             :         uint16_t                        refcnt;
      54             :         uint16_t                        size;
      55             :         struct spdk_idxd_io_channel     *chan;
      56             :         TAILQ_ENTRY(idxd_batch)         link;
      57             : };
      58             : 
      59             : struct device_config {
      60             :         uint8_t         config_num;
      61             :         uint8_t         num_groups;
      62             :         uint16_t        total_wqs;
      63             :         uint16_t        total_engines;
      64             : };
      65             : 
      66             : struct idxd_ops;
      67             : 
      68             : struct spdk_idxd_io_channel {
      69             :         struct spdk_idxd_device                 *idxd;
      70             :         /* The portal is the address that we write descriptors to for submission. */
      71             :         void                                    *portal;
      72             :         uint32_t                                portal_offset;
      73             : 
      74             :         bool                                    pasid_enabled;
      75             : 
      76             :         /* The currently open batch */
      77             :         struct idxd_batch                       *batch;
      78             : 
      79             :         /*
      80             :          * User descriptors (those included in a batch) are managed independently from
      81             :          * data descriptors and are located in the batch structure.
      82             :          */
      83             :         void                                    *desc_base;
      84             :         STAILQ_HEAD(, idxd_ops)                 ops_pool;
      85             :         /* Current list of outstanding operations to poll. */
      86             :         STAILQ_HEAD(op_head, idxd_ops)          ops_outstanding;
      87             :         void                                    *ops_base;
      88             : 
      89             :         TAILQ_HEAD(, idxd_batch)                batch_pool;
      90             :         void                                    *batch_base;
      91             : };
      92             : 
      93             : struct pci_dev_id {
      94             :         int vendor_id;
      95             :         int device_id;
      96             : };
      97             : 
      98             : /*
      99             :  * This struct wraps the hardware completion record which is 32 bytes in
     100             :  * size and must be 32 byte aligned.
     101             :  */
     102             : struct idxd_ops {
     103             :         union {
     104             :                 struct dsa_hw_comp_record       hw;
     105             :                 struct iaa_hw_comp_record       iaa_hw;
     106             :         };
     107             :         void                            *cb_arg;
     108             :         spdk_idxd_req_cb                cb_fn;
     109             :         struct idxd_batch               *batch;
     110             :         struct idxd_hw_desc             *desc;
     111             :         union {
     112             :                 uint32_t                *crc_dst;
     113             :                 uint32_t                *output_size;
     114             :         };
     115             :         struct idxd_ops                 *parent;
     116             :         uint32_t                        count;
     117             :         STAILQ_ENTRY(idxd_ops)          link;
     118             : };
     119             : SPDK_STATIC_ASSERT(sizeof(struct idxd_ops) == 128, "size mismatch");
     120             : 
     121             : struct spdk_idxd_impl {
     122             :         const char *name;
     123             :         int (*probe)(void *cb_ctx, spdk_idxd_attach_cb attach_cb,
     124             :                      spdk_idxd_probe_cb probe_cb);
     125             :         void (*destruct)(struct spdk_idxd_device *idxd);
     126             :         void (*dump_sw_error)(struct spdk_idxd_device *idxd, void *portal);
     127             :         char *(*portal_get_addr)(struct spdk_idxd_device *idxd);
     128             : 
     129             :         STAILQ_ENTRY(spdk_idxd_impl) link;
     130             : };
     131             : 
     132             : struct spdk_idxd_device {
     133             :         struct spdk_idxd_impl           *impl;
     134             :         void                            *portal;
     135             :         uint32_t                        socket_id;
     136             :         uint32_t                        num_channels;
     137             :         uint32_t                        total_wq_size;
     138             :         uint32_t                        chan_per_device;
     139             :         uint16_t                        batch_size;
     140             :         pthread_mutex_t                 num_channels_lock;
     141             :         bool                            pasid_enabled;
     142             :         enum idxd_dev                   type;
     143             :         struct iaa_aecs                 *aecs;
     144             :         uint64_t                        aecs_addr;
     145             :         uint32_t                        version;
     146             : };
     147             : 
     148             : void idxd_impl_register(struct spdk_idxd_impl *impl);
     149             : 
     150             : #define SPDK_IDXD_IMPL_REGISTER(name, impl) \
     151             : static void __attribute__((constructor)) idxd_impl_register_##name(void) \
     152             : { \
     153             :         idxd_impl_register(impl); \
     154             : }
     155             : 
     156             : #ifdef __cplusplus
     157             : }
     158             : #endif
     159             : 
     160             : #endif /* __IDXD_H__ */