pktmbuf.h

Go to the documentation of this file.

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2019-2022 Intel Corporation.
 * Copyright (c) 2019-2020 6WIND S.A.
 */
 
#ifndef _PKTMBUF_H_
#define _PKTMBUF_H_
 
#include <errno.h>                        // for EINVAL
#include <stdint.h>                       // for uint16_t, uint32_t, uint8_t, UINT...
#include <stdio.h>                        // for NULL, FILE
#include <bsd/string.h>                   // for strlcpy
#include <cne_atomic.h>                   // for atomic_uint_least16_t
#include <cne_common.h>                   // for CNDP_API, CNE_STD_C11, __cne_alwa...
#include <cne_branch_prediction.h>        // for likely, unlikely
#include <cne_log.h>                      // for CNE_ASSERT
#include <cne_mmap.h>                     // for mmap_type_t
#include <cne_prefetch.h>                 // for cne_prefetch0
#include <mempool.h>                      // for mempool_t, mempool_get, mempool_g...
 
#include "pktmbuf_ops.h"        // for mbuf_ops_t
#include "pktmbuf_offload.h"
 
#ifdef __cplusplus
extern "C" {
#endif
 
enum {
    DEFAULT_BURST_SIZE   = 64,          
    DEFAULT_MBUF_COUNT   = (16 * 1024), 
    DEFAULT_MBUF_SIZE    = 2048,        
    PKTMBUF_PENDING_SZ   = 128,         
    PKTMBUF_INFO_NAME_SZ = 24,          
};
 
struct pktmbuf_info_s;
struct pktmbuf_s;
 
typedef struct pktmbuf_pool_cfg {
    char *addr;              
    uint32_t bufcnt;         
    uint32_t bufsz;          
    uint32_t cache_sz;       
    uint32_t metadata_bufsz; 
    char *metadata;          
    mbuf_ops_t *ops;         
} pktmbuf_pool_cfg_t;
 
typedef struct pktmbuf_info_s {
    TAILQ_ENTRY(pktmbuf_info_s) next; 
    char name[PKTMBUF_INFO_NAME_SZ];  
    void *addr;                       
    void *pd;                         
    mbuf_ops_t ops;                   
    uint32_t bufcnt;                  
    uint32_t bufsz;                   
    uint32_t cache_sz;                
    uint32_t metadata_bufsz;          
    char *metadata;                   
} pktmbuf_info_t;
 
#define CNE_MBUF_DEFAULT_DATAROOM (2 * 1024)
#define CNE_MBUF_DEFAULT_BUF_SIZE CNE_MBUF_DEFAULT_DATAROOM
 
struct pktmbuf_s {
    void *pooldata;      
    void *buf_addr;      
    uint32_t hash;       
    uint32_t meta_index; 
    uint16_t data_off;   
    uint16_t lport;      
    uint16_t buf_len;    
    uint16_t data_len;   
    /*
     * The packet type, which is the combination of outer/inner L2, L3, L4
     * and tunnel types. The packet_type is about data really present in the
     * pktmbuf. Example: if vlan stripping is enabled, a received vlan packet
     * would have CNE_PTYPE_L2_ETHER and not CNE_PTYPE_L2_VLAN because the
     * vlan is stripped from the data.
     */
    CNE_STD_C11
    union {
        uint32_t packet_type; 
        CNE_STD_C11
        struct {
            uint8_t l2_type : 4;  
            uint8_t l3_type : 4;  
            uint8_t l4_type : 4;  
            uint8_t tun_type : 4; 
            CNE_STD_C11
            union {
                uint8_t inner_esp_next_proto;
                CNE_STD_C11
                struct {
                    uint8_t inner_l2_type : 4; 
                    uint8_t inner_l3_type : 4; 
                };
            };
            uint8_t inner_l4_type : 4; 
        };
    };
 
    CNE_STD_C11
    union {
        CNE_ATOMIC(uint_least16_t) refcnt_atomic; 
        uint16_t refcnt;                          
    };
 
    uint16_t rsvd16;
 
    /* fields to support TX offloads */
    CNE_STD_C11
    union {
        uint64_t tx_offload; 
        CNE_STD_C11
        struct {
            uint64_t l2_len : CNE_MBUF_L2_LEN_BITS;
            uint64_t l3_len : CNE_MBUF_L3_LEN_BITS;       
            uint64_t l4_len : CNE_MBUF_L4_LEN_BITS;       
            uint64_t tso_segsz : CNE_MBUF_TSO_SEGSZ_BITS; 
            /*
             * Fields for Tx offloading of tunnels.
             * These are undefined for packets which don't request
             * any tunnel offloads (outer IP or UDP checksum,
             * tunnel TSO).
             *
             * PMDs should not use these fields unconditionally
             * when calculating offsets.
             *
             * Applications are expected to set appropriate tunnel
             * offload flags when they fill in these fields.
             */
            uint64_t outer_l3_len : CNE_MBUF_OUTL3_LEN_BITS; 
            uint64_t outer_l2_len : CNE_MBUF_OUTL2_LEN_BITS; 
            /* uint64_t unused:CNE_MBUF_TXOFLD_UNUSED_BITS; */
        };
    };
 
    uint64_t ol_flags; 
    CNE_STD_C11
    union {               
        void *userptr;    
        uint64_t udata64; 
    };
} __cne_cache_aligned;
 
typedef struct pktmbuf_s pktmbuf_t;
 
typedef struct pktmbuf_pending {
    uint16_t nb_pending; 
    uint16_t pending_sz; 
    void *pooldata;      
    pktmbuf_t *pending[PKTMBUF_PENDING_SZ]; 
} pktmbuf_pending_t;
 
CNDP_API void pktmbuf_destroy(pktmbuf_info_t *pi);
 
CNDP_API pktmbuf_info_t *pktmbuf_pool_create(char *addr, uint32_t bufcnt, uint32_t bufsz,
                                             uint32_t cache_sz, mbuf_ops_t *ops);
 
CNDP_API pktmbuf_info_t *pktmbuf_pool_cfg_create(const pktmbuf_pool_cfg_t *cfg);
 
typedef int (*pktmbuf_cb_t)(pktmbuf_info_t *pi, pktmbuf_t *buf, uint32_t sz, uint32_t idx,
                            void *ud);
 
CNDP_API int pktmbuf_iterate(pktmbuf_info_t *pi, pktmbuf_cb_t cb, void *ud);
 
#define pktmbuf_hash(m) ((m)->hash)
 
#define pktmbuf_pooldata(m) ((m)->pooldata)
 
#define pktmbuf_buf_addr(m) ((m)->buf_addr)
 
#define pktmbuf_port(m) ((m)->lport)
 
#define pktmbuf_buf_len(m) ((m)->buf_len)
 
#define pktmbuf_tx_offload(m) ((m)->tx_offload)
 
#define pktmbuf_userptr(m) ((m)->userptr)
 
#define pktmbuf_udata64(m) ((m)->udata64)
 
#define pktmbuf_data_len(m) ((m)->data_len)
 
#define pktmbuf_data_off(m) ((m)->data_off)
 
#define pktmbuf_refcnt(m) ((m)->refcnt)
 
#define pktmbuf_mtod_offset(m, t, o) ((t)((char *)pktmbuf_buf_addr(m) + pktmbuf_data_off(m) + (o)))
 
#define pktmbuf_mtod(m, t) pktmbuf_mtod_offset(m, t, 0)
 
#define pktmbuf_meta_index(m) ((m)->meta_index)
 
static inline void
pktmbuf_prefetch(pktmbuf_t *m)
{
    cne_prefetch0(m);
}
 
static inline char *
pktmbuf_data_addr_default(pktmbuf_t *mb)
{
    /* gcc complains about calling this function even
     * when not using it.
     */
    return (char *)pktmbuf_buf_addr(mb) + CNE_PKTMBUF_HEADROOM;
}
 
#ifdef CNE_LIBCNE_MBUF_DEBUG
 
#define __pktmbuf_sanity_check(m, is_h) pktmbuf_sanity_check(m, is_h)
 
#else /*  CNE_LIBCNE_MBUF_DEBUG */
 
// clang-format off
#define __pktmbuf_sanity_check(m, is_h) do { } while (0)
// clang-format on
 
#endif /*  CNE_LIBCNE_MBUF_DEBUG */
 
static inline uint16_t
pktmbuf_refcnt_read(const pktmbuf_t *m)
{
    return atomic_load_explicit(&m->refcnt_atomic, CNE_MEMORY_ORDER(relaxed));
}
 
static inline void
pktmbuf_refcnt_set(pktmbuf_t *m, uint16_t new_value)
{
    atomic_store_explicit(&m->refcnt_atomic, new_value, CNE_MEMORY_ORDER(relaxed));
}
 
/* internal */
static inline uint16_t
__pktmbuf_refcnt_update(pktmbuf_t *m, int16_t value)
{
    return atomic_fetch_add_explicit(&m->refcnt_atomic, (uint16_t)value, CNE_MEMORY_ORDER(relaxed));
}
 
static inline uint16_t
pktmbuf_refcnt_update(pktmbuf_t *m, int16_t value)
{
    uint16_t refcnt;
 
    /*
     * The atomic_add is an expensive operation, so we don't want to
     * call it in the case where we know we are the unique holder of
     * this mbuf (i.e. ref_cnt == 1). Otherwise, an atomic
     * operation has to be used because concurrent accesses on the
     * reference counter can occur.
     */
    refcnt = pktmbuf_refcnt_read(m);
    if (likely(refcnt == 1)) {
        refcnt = refcnt + value;
        pktmbuf_refcnt_set(m, refcnt);
        return (uint16_t)refcnt;
    }
 
    __pktmbuf_refcnt_update(m, value);
 
    return pktmbuf_refcnt_read(m);
}
 
// clang-format off
#define CNE_MBUF_PREFETCH_TO_FREE(m) do {       \
        if ((m) != NULL)                        \
            cne_prefetch0(m);                   \
    } while (0)
// clang-format on
 
CNDP_API void pktmbuf_sanity_check(const pktmbuf_t *m, int is_header);
 
CNDP_API int pktmbuf_check(const pktmbuf_t *m, int is_header, const char **reason);
 
static inline void
pktmbuf_info_name_set(pktmbuf_info_t *pi, const char *str)
{
    if (pi) {
        if (!str)
            memset(pi->name, 0, sizeof(pi->name));
        else
            strlcpy(pi->name, str, sizeof(pi->name));
    }
}
 
static inline const char *
pktmbuf_info_name_get(pktmbuf_info_t *pi)
{
    return (!pi) ? NULL : pi->name;
}
 
static inline void
pktmbuf_reset_headroom(pktmbuf_t *m)
{
    m->data_off = (uint16_t)CNE_MIN((uint16_t)CNE_PKTMBUF_HEADROOM, (uint16_t)pktmbuf_buf_len(m));
}
 
#define CNE_MBUF_INVALID_PORT UINT16_MAX
 
static inline void
pktmbuf_reset(pktmbuf_t *m)
{
    pktmbuf_reset_headroom(m);
 
    pktmbuf_data_len(m) = 0;
    pktmbuf_port(m)     = CNE_MBUF_INVALID_PORT;
    m->packet_type      = 0;
    m->tx_offload       = 0;
    m->ol_flags         = 0;
    m->hash             = 0;
}
 
static inline int
pktmbuf_alloc_bulk(pktmbuf_info_t *pi, pktmbuf_t **mbufs, unsigned count)
{
    return (!pi) ? -EINVAL : pi->ops.mbuf_alloc(pi, mbufs, count);
}
 
static inline pktmbuf_t *
pktmbuf_alloc(pktmbuf_info_t *pi)
{
    pktmbuf_t *m = NULL;
 
    if (pktmbuf_alloc_bulk(pi, &m, 1) <= 0)
        return NULL;
 
    pktmbuf_reset(m);
 
    return m;
}
 
/* internal */
static inline void
__pktmbuf_copy_hdr(pktmbuf_t *mdst, const pktmbuf_t *msrc)
{
    pktmbuf_port(mdst) = pktmbuf_port(msrc);
}
 
static __cne_always_inline pktmbuf_t *
pktmbuf_refcnt_free(pktmbuf_t *m)
{
    if (m) {
        if (likely(pktmbuf_refcnt_read(m) == 1))
            return m;
        else if (__pktmbuf_refcnt_update(m, -1) == 0) {
            pktmbuf_refcnt_set(m, 1);
            return m;
        }
    }
    return NULL;
}
 
static __cne_always_inline void
pktmbuf_free(pktmbuf_t *m)
{
    m = pktmbuf_refcnt_free(m);
    if (likely(m != NULL)) {
        pktmbuf_info_t *pi = (pktmbuf_info_t *)m->pooldata;
 
        (void)pi->ops.mbuf_free(pi, &m, 1);
    }
}
 
static __cne_always_inline void
__pktmbuf_flush_pending(pktmbuf_pending_t *p)
{
    pktmbuf_info_t *pi = (pktmbuf_info_t *)p->pooldata;
 
    (void)pi->ops.mbuf_free(pi, p->pending, p->nb_pending);
    p->nb_pending = 0;
}
 
static __cne_always_inline void
__pktmbuf_free_bulk(pktmbuf_pending_t *p, pktmbuf_t *m)
{
    m = pktmbuf_refcnt_free(m);
 
    if (likely(m != NULL)) {
        if (p->nb_pending == p->pending_sz || (p->nb_pending > 0 && m->pooldata != p->pooldata)) {
            __pktmbuf_flush_pending(p);
            p->pooldata = m->pooldata;
        }
 
        p->pending[p->nb_pending++] = m;
    }
}
 
static inline void
pktmbuf_free_bulk(pktmbuf_t **mbufs, unsigned int count)
{
    pktmbuf_pending_t pend;
 
    memset(&pend, 0, sizeof(pend));
    pend.pending_sz = PKTMBUF_PENDING_SZ;
 
    if (count == 0)
        return;
 
    pend.pooldata = mbufs[0]->pooldata;
 
    for (unsigned int idx = 0; idx < count; idx++)
        __pktmbuf_free_bulk(&pend, mbufs[idx]);
 
    if (pend.nb_pending)
        __pktmbuf_flush_pending(&pend);
}
 
CNDP_API pktmbuf_t *pktmbuf_copy(const pktmbuf_t *m, pktmbuf_info_t *pi, uint32_t offset,
                                 uint32_t length);
 
static inline uint16_t
pktmbuf_headroom(const pktmbuf_t *m)
{
    __pktmbuf_sanity_check(m, 0);
    return pktmbuf_data_off(m);
}
 
static inline uint16_t
pktmbuf_tailroom(const pktmbuf_t *m)
{
    __pktmbuf_sanity_check(m, 0);
    return (uint16_t)(pktmbuf_buf_len(m) - pktmbuf_data_off(m) - pktmbuf_data_len(m));
}
 
static inline char *
pktmbuf_mtod_last(const pktmbuf_t *m)
{
    __pktmbuf_sanity_check(m, 0);
    return pktmbuf_mtod_offset(m, char *, pktmbuf_data_len(m));
}
 
static inline char *
pktmbuf_mtod_end(const pktmbuf_t *m)
{
    __pktmbuf_sanity_check(m, 0);
    return (char *)CNE_PTR_ADD((char *)(uintptr_t)m, pktmbuf_buf_len(m) + sizeof(pktmbuf_t));
}
 
static inline char *
pktmbuf_prepend(pktmbuf_t *m, uint16_t len)
{
    __pktmbuf_sanity_check(m, 1);
 
    if (unlikely(len > pktmbuf_headroom(m)))
        return NULL;
 
    /* NB: elaborating the subtraction like this instead of using
     *     -= allows us to ensure the result type is uint16_t
     *     avoiding compiler warnings on gcc 8.1 at least */
    pktmbuf_data_off(m) = (uint16_t)(pktmbuf_data_off(m) - len);
    pktmbuf_data_len(m) = (uint16_t)(pktmbuf_data_len(m) + len);
 
    return pktmbuf_mtod(m, char *);
}
 
static inline char *
pktmbuf_adj_offset(pktmbuf_t *m, int16_t len)
{
    uint16_t alen = abs(len);
 
    __pktmbuf_sanity_check(m, 1);
 
    if (likely(len >= 0)) {
        if (unlikely(alen > pktmbuf_data_len(m)))
            return NULL;
        if (unlikely((alen + pktmbuf_data_off(m)) > pktmbuf_buf_len(m)))
            return NULL;
 
        /* NB: elaborating the subtraction like this instead of using
         *     -= allows us to ensure the result type is uint16_t
         *     avoiding compiler warnings on gcc 8.1 at least */
        pktmbuf_data_off(m) = (uint16_t)(pktmbuf_data_off(m) + alen);
        pktmbuf_data_len(m) = (uint16_t)(pktmbuf_data_len(m) - alen);
    } else {
        if (unlikely(alen > pktmbuf_data_off(m)))
            return NULL;
 
        /* NB: elaborating the subtraction like this instead of using
         *     -= allows us to ensure the result type is uint16_t
         *     avoiding compiler warnings on gcc 8.1 at least */
        pktmbuf_data_off(m) = (uint16_t)(pktmbuf_data_off(m) - alen);
        pktmbuf_data_len(m) = (uint16_t)(pktmbuf_data_len(m) + alen);
    }
    return pktmbuf_mtod(m, char *);
}
 
#define pktmbuf_adjust(m, t, len) (t) pktmbuf_adj_offset(m, len)
 
static inline char *
pktmbuf_append(pktmbuf_t *m, uint16_t len)
{
    void *tail;
 
    __pktmbuf_sanity_check(m, 1);
 
    if (unlikely(len > pktmbuf_tailroom(m)))
        return NULL;
 
    tail                = (char *)pktmbuf_buf_addr(m) + pktmbuf_data_off(m) + pktmbuf_data_len(m);
    pktmbuf_data_len(m) = (uint16_t)(pktmbuf_data_len(m) + len);
    return (char *)tail;
}
 
static inline int
pktmbuf_trim(pktmbuf_t *m, uint16_t len)
{
    __pktmbuf_sanity_check(m, 1);
 
    if (unlikely(len > pktmbuf_data_len(m)))
        return -1;
 
    pktmbuf_data_len(m) = (uint16_t)(pktmbuf_data_len(m) - len);
 
    return 0;
}
 
const void *__pktmbuf_read(const pktmbuf_t *m, uint32_t off, uint32_t len, void *buf);
 
static inline const void *
pktmbuf_read(const pktmbuf_t *m, uint32_t off, uint32_t len, void *buf)
{
    if (likely(off + len <= pktmbuf_data_len(m)))
        return pktmbuf_mtod_offset(m, char *, off);
    else
        return __pktmbuf_read(m, off, len, buf);
}
 
const void *__pktmbuf_write(const void *buf, uint32_t len, pktmbuf_t *m, uint32_t off);
 
static inline const void *
pktmbuf_write(const void *buf, uint32_t len, pktmbuf_t *m, uint32_t off)
{
    if (likely((len + off) > pktmbuf_tailroom(m)))
        return NULL;
    else
        return __pktmbuf_write(buf, len, m, off);
}
 
static inline uint16_t
pktmbuf_data_room_size(mempool_t *mp)
{
    return mempool_objsz(mp) - sizeof(pktmbuf_t);
}
 
CNDP_API pktmbuf_t *pktmbuf_clone(pktmbuf_t *md, pktmbuf_info_t *pi);
 
CNDP_API void pktmbuf_dump(const char *msg, const pktmbuf_t *m, unsigned dump_len);
 
const char *cne_get_rx_ol_flag_name(uint64_t mask);
 
int cne_get_rx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
 
const char *cne_get_tx_ol_flag_name(uint64_t mask);
 
int cne_get_tx_ol_flag_list(uint64_t mask, char *buf, size_t buflen);
 
static inline void *
pktmbuf_metadata(const pktmbuf_t *m)
{
    pktmbuf_info_t *p;
 
    if (!m)
        return NULL;
 
    if (((p = (pktmbuf_info_t *)m->pooldata) != NULL) && p->metadata)
        return CNE_PTR_ADD(p->metadata, (m->meta_index * p->metadata_bufsz));
    else
        return CNE_PTR_ADD(m, sizeof(pktmbuf_t)); /* default to metadata in pktmbuf headroom */
}
 
static inline int32_t
pktmbuf_metadata_bufsz(const pktmbuf_t *m)
{
    pktmbuf_info_t *p;
 
    if (!m)
        return -1;
 
    if (((p = (pktmbuf_info_t *)m->pooldata) != NULL) && p->metadata)
        return (int32_t)p->metadata_bufsz;
    else
        return (int32_t)pktmbuf_headroom(m);
}
 
CNDP_API int pktmbuf_pool_cfg(pktmbuf_pool_cfg_t *c, void *addr, uint32_t bufcnt, uint32_t bufsz,
                              uint32_t cache_sz, void *metadata, uint32_t metadata_bufsz,
                              mbuf_ops_t *ops);
 
CNDP_API void pktmbuf_info_dump(void);
 
#ifdef __cplusplus
}
#endif
 
#endif /* _PKTMBUF_H_ */