Hallo, dies ist ein Test.

PWD: /www/data-lst1/unixsoft/unixsoft/kaempfer/.public_html

Running in File Mode
Relative path: ./../../../../../../usr/include/umem_impl.h
Real path: /usr/include/umem_impl.h
Zurück

/*
 * Copyright (c) 2002, 2023, Oracle and/or its affiliates.
 */

#ifndef _UMEM_IMPL_H
#define	_UMEM_IMPL_H

#include <umem.h>

#include <sys/sysmacros.h>
#include <sys/time.h>
#include <sys/vmem.h>
#include <uadi_impl.h>
#include <thread.h>

#ifdef	__cplusplus
extern "C" {
#endif

/*
 * umem memory allocator: implementation-private data structures
 */

#define	UMEM_ALLOC_STR "umem_alloc_"
#define	UMEM_ALLOC_CACHE(cp) (((cp)->cache_cflags & UMC_UMEM_ALLOC) != 0 || \
	strncmp((cp)->cache_name, UMEM_ALLOC_STR, strlen(UMEM_ALLOC_STR)) == 0)

/*
 * Internal flags for umem_cache_create
 */
#define	UMC_QCACHE	0x00100000
#define	UMC_INTERNAL	0x80000000

/*
 * Cache flags
 */
typedef enum umem_flags_e {
	UMF_AUDIT	= 0x00000001,	/* transaction auditing */
	UMF_DEADBEEF	= 0x00000002,	/* deadbeef checking */
	UMF_REDZONE	= 0x00000004,	/* redzone checking */
	UMF_CONTENTS	= 0x00000008,	/* freed-buffer content logging */
	UMF_CHECKSIGNAL	= 0x00000010,	/* abort when in signal context */
	UMF_NOMAGAZINE	= 0x00000020,	/* disable per-cpu magazines */
	UMF_FIREWALL	= 0x00000040,	/* put all bufs before unmapped pages */
	UMF_LITE	= 0x00000100,	/* lightweight debugging */

UMF_HASH	= 0x00000200,	/* cache has hash table */
	UMF_RANDOMIZE	= 0x00000400,	/* randomize other umem_flags */
	UMF_ADI		= 0x00000800	/* protected with ADI */
} umem_flags_t;

#define	UMEM_STACK_DEPTH	umem_stack_depth

#define	UMEM_FREE_PATTERN		0xdeadbeefdeadbeefULL
#define	UMEM_UNINITIALIZED_PATTERN	0xbaddcafebaddcafeULL
#define	UMEM_REDZONE_PATTERN		0xfeedfacefeedfaceULL
#define	UMEM_REDZONE_BYTE		0xbb

#define	UMEM_FATAL_FLAGS	(UMEM_NOFAIL)
#define	UMEM_SLEEP_FLAGS	(0)

/*
 * Redzone size encodings for umem_alloc() / umem_free().  We encode the
 * allocation size, rather than storing it directly, so that umem_free()
 * can distinguish frees of the wrong size from redzone violations.
 */
#define	UMEM_SIZE_ENCODE(x)	(251 * (x) + 1)
#define	UMEM_SIZE_DECODE(x)	((x) / 251)
#define	UMEM_SIZE_VALID(x)	((x) % 251 == 1)

/*
 * The bufctl (buffer control) structure keeps some minimal information
 * about each buffer: its address, its slab, and its current linkage,
 * which is either on the slab's freelist (if the buffer is free), or
 * on the cache's buf-to-bufctl hash table (if the buffer is allocated).
 * In the case of non-hashed, or "raw", caches (the common case), only
 * the freelist linkage is necessary: the buffer address is at a fixed
 * offset from the bufctl address, and the slab is at the end of the page.
 *
 * NOTE: bc_next must be the first field; raw buffers have linkage only.
 */
typedef struct umem_bufctl {
	struct umem_bufctl	*bc_next;	/* next bufctl struct */
	void			*bc_addr;	/* address of buffer */
	struct umem_slab	*bc_slab;	/* controlling slab */
} umem_bufctl_t;

/*
 * The UMF_AUDIT version of the bufctl structure.  The beginning of this
 * structure must be identical to the normal bufctl structure so that
 * pointers are interchangeable.
 */

#define	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(frames) \
	((size_t)(&((umem_bufctl_audit_t *)0)->bc_stack[frames]))

/*
 * umem_bufctl_audits must be allocated from a UMC_NOHASH cache, so we
 * require that 2 of them, plus 2 buftags, plus a umem_slab_t, all fit on
 * a single page.
 *
 * For ILP32, this is about 1000 frames.
 * For LP64, this is about 490 frames.
 */

#define	UMEM_BUFCTL_AUDIT_ALIGN	32

#define	UMEM_BUFCTL_AUDIT_MAX_SIZE					\
	(P2ALIGN((PAGESIZE - sizeof (umem_slab_t))/2 -			\
	    sizeof (umem_buftag_t), UMEM_BUFCTL_AUDIT_ALIGN))

#define	UMEM_MAX_STACK_DEPTH						\
	((UMEM_BUFCTL_AUDIT_MAX_SIZE -					\
	    UMEM_BUFCTL_AUDIT_SIZE_DEPTH(0)) / sizeof (uintptr_t))

typedef struct umem_bufctl_audit {
	struct umem_bufctl	*bc_next;	/* next bufctl struct */
	void			*bc_addr;	/* address of buffer */
	struct umem_slab	*bc_slab;	/* controlling slab */
	umem_cache_t		*bc_cache;	/* controlling cache */
	hrtime_t		bc_timestamp;	/* transaction time */
	thread_t		bc_thread;	/* thread doing transaction */
	struct umem_bufctl	*bc_lastlog;	/* last log entry */
	void			*bc_contents;	/* contents at last free */
	uint8_t			bc_depth;	/* stack depth */
	uint8_t			bc_adiver;	/* buffer ADI version */
	uint16_t		bc_pad;		/* available space! */
	uintptr_t		bc_stack[1];	/* pc stack */
} umem_bufctl_audit_t;

#define	UMEM_LOCAL_BUFCTL_AUDIT(bcpp)					\
		*(bcpp) = (umem_bufctl_audit_t *)			\
		    alloca(UMEM_BUFCTL_AUDIT_SIZE)

#define	UMEM_BUFCTL_AUDIT_SIZE						\
	UMEM_BUFCTL_AUDIT_SIZE_DEPTH(UMEM_STACK_DEPTH)

/*
 * A umem_buftag structure is appended to each buffer whenever any of the
 * UMF_BUFTAG flags (UMF_DEADBEEF, UMF_REDZONE, UMF_VERIFY) are set.
 */
typedef struct umem_buftag {
	uint64_t		bt_redzone;	/* 64-bit redzone pattern */
	umem_bufctl_t		*bt_bufctl;	/* bufctl */
	intptr_t		bt_bxstat;	/* bufctl ^ (alloc/free) */
} umem_buftag_t;

/* These functions are defined in umem.c and mdb/../libumem/umem.c */
#define	UMEM_BUFTAG(cp, buf)	umem_buftag_impl((cp), (void *)(buf))

#define	UMEM_BUFCTL(cp, buf)	umem_bufctl_impl((cp), (void *)(buf))

#if defined(__sparcv9) && !defined(UMEM_STANDALONE)

#define	UMEM_BUF_OFFSET(cp, buf)	\
	(void *)((uintptr_t)buf + cp->cache_chunkunused)

/*
 * If ADI is enabled, we adjust umem_[z]alloc() buffers so that they are
 * still UMEM_ALIGN-aligned, but are as close as possible to the end of
 * the buffer, so that the buftag is immediately after the buffer.  Since
 * the buftag has a different ADI tag, we'll catch buffer overruns immediately.
 *
 * Note that 64-byte aligned buffers will be allocated out of 64-byte-multiple
 * caches, so this adjustment won't change that alignment.
 */
#define	UMEM_ALLOC_BUF_ADJ(cp, size)					\
	(((cp)->cache_flags & UMF_ADI) ?				\
	    P2ALIGN((cp)->cache_bufsize - (size), UMEM_ALIGN) : 0)

/*
 * The above adjustment means that the redzone byte (which immediately follows
 * the end of the buffer) will be the first byte of the buftag if size is
 * a umem-align multiple and UMF_ADI is set.  Since the buftag has a separate
 * ADI version, we need to use the buftag pointer in that case.
 */
#define	UMEM_ALLOC_RZBYTE(cp, buf, btp, size)			\
	((((cp)->cache_flags & UMF_ADI) &&			\
	    IS_P2ALIGNED((size), UMEM_ALIGN)) ?			\
	    (uint8_t *)(btp) : (&((uint8_t *)(buf))[(size)]))

#define	UMEM_BUF(cp, bcp)	uadi_buffer(cp, (umem_bufctl_t *)bcp)

#define	UMEM_SLAB(cp, buf)	uadi_slab(cp, buf)

#else  /* __sparcv9 && !UMEM_STANDALONE */

#define	UMEM_BUF_OFFSET(cp, buf)	(void *)(buf)

#define	UMEM_ALLOC_BUF_ADJ(cp, size)	(0)

#define	UMEM_ALLOC_RZBYTE(cp, buf, btp, size)				\
	(&((uint8_t *)buf)[(size)])

#define	UMEM_BUF(cp, bcp)		\
	((void *)((uintptr_t)(bcp) - (cp)->cache_bufctl))

#define	UMEM_SLAB(cp, buf)		\
	((umem_slab_t *)P2END((uintptr_t)(buf), (cp)->cache_slabsize) - 1)

#endif /* __sparcv9 && !UMEM_STANDALONE */

#define	UMEM_CPU_CACHE(cp, cpu)		\
	(umem_cpu_cache_t *)((uintptr_t)cp + cpu->cpu_cache_offset)

#define	UMEM_MAGAZINE_VALID(cp, mp)	\
	(((umem_slab_t *)P2END((uintptr_t)(mp), PAGESIZE) - 1)->slab_cache == \
	    (cp)->cache_magtype->mt_cache)

#define	UMEM_NORM_ADDR(cp, buf)		\
	(((cp)->cache_flags & UMF_ADI) ? ADI_NORMALIZE_ADDR(buf) : buf)

#define	UMEM_SLAB_OFFSET(sp, buf)					\
	((size_t)((uintptr_t)UMEM_NORM_ADDR((sp)->slab_cache, buf) -	\
		(uintptr_t)(sp)->slab_base))

#define	UMEM_SLAB_MEMBER(sp, buf)	\
	(UMEM_SLAB_OFFSET(sp, buf) < (sp)->slab_cache->cache_slabsize)

#define	UMEM_BUFTAG_ALLOC	0xa110c8edUL
#define	UMEM_BUFTAG_FREE	0xf4eef4eeUL

typedef struct umem_slab {
	struct umem_cache	*slab_cache;	/* controlling cache */
	void			*slab_base;	/* base of allocated memory */
	struct umem_slab	*slab_next;	/* next slab on freelist */
	struct umem_slab	*slab_prev;	/* prev slab on freelist */
	struct umem_bufctl	*slab_head;	/* first free buffer */
	long			slab_refcnt;	/* outstanding allocations */
	long			slab_chunks;	/* chunks (bufs) in this slab */
} umem_slab_t;

#define	UMEM_HASH_INITIAL	64

#define	UMEM_HASH(cp, buf)	\
	((cp)->cache_hash_table +	\
	(((uintptr_t)UMEM_NORM_ADDR(cp, buf) >> (cp)->cache_hash_shift) &   \
	(cp)->cache_hash_mask))

/* Version of UMEM_HASH() for mdb which takes a normalized address */
#define	MDB_UMEM_HASH(cp, buf)						\
	((cp)->cache_hash_table +					\
	(((uintptr_t)(buf) >> (cp)->cache_hash_shift) &			\
	(cp)->cache_hash_mask))

typedef struct umem_magazine {
	void	*mag_next;
	void	*mag_round[1];		/* one or more rounds */
} umem_magazine_t;

/*
 * The magazine types for fast per-cpu allocation
 */
typedef struct umem_magtype {
	int		mt_magsize;	/* magazine size (number of rounds) */
	int		mt_align;	/* magazine alignment */
	size_t		mt_minbuf;	/* all smaller buffers don't qualify */
	size_t		mt_maxbuf;	/* no larger buffers qualify */
	umem_cache_t	*mt_cache;	/* magazine cache */
} umem_magtype_t;

#define	UMEM_CPU_PAD		P2NPHASE(sizeof (mutex_t) + \
	2 * sizeof (uint_t) + 2 * sizeof (void *) + sizeof (uint64_t) +	\
	4 * sizeof (int) +	sizeof (uint16_t), _FALSE_SHARING_ALIGN)
#define	UMEM_CACHE_SIZE(ncpus)	\
	((size_t)(&((umem_cache_t *)0)->cache_cpu[ncpus]))

typedef struct umem_cpu_cache {
	mutex_t		cc_lock;	/* protects this cpu's local cache */
	uint_t		cc_alloc;	/* allocations from this cpu */
	uint_t		cc_free;	/* frees to this cpu */
	umem_magazine_t	*cc_loaded;	/* the currently loaded magazine */
	umem_magazine_t	*cc_ploaded;	/* the previously loaded magazine */
	uint64_t	cc_adi_vermap;	/* ADI version bitmap */
	int		cc_rounds;	/* number of objects in loaded mag */
	int		cc_prounds;	/* number of objects in previous mag */
	int		cc_magsize;	/* number of rounds in a full mag */
	int		cc_flags;	/* CPU-local copy of cache_flags */
	uint16_t	cc_adi_bufsize; /* encoded cache_bufsize for ADI */
	char		cc_pad[UMEM_CPU_PAD]; /* for nice alignment (32-bit) */
} umem_cpu_cache_t;

#define	UMEM_CPU_CACHE_SIZE	(sizeof (umem_cpu_cache_t))

/*
 * The magazine lists used in the depot.
 */
typedef struct umem_maglist {
	umem_magazine_t	*ml_list;	/* magazine list */
	long		ml_total;	/* number of magazines */
	long		ml_min;		/* min since last update */
	long		ml_reaplimit;	/* max reapable magazines */
	uint64_t	ml_alloc;	/* allocations from this list */
} umem_maglist_t;

#define	UMEM_CACHE_NAMELEN	31

struct umem_cache {
	/*
	 * Statistics
	 */
	uint64_t	cache_slab_create;	/* slab creates */
	uint64_t	cache_slab_destroy;	/* slab destroys */
	uint64_t	cache_slab_alloc;	/* slab layer allocations */
	uint64_t	cache_slab_free;	/* slab layer frees */
	uint64_t	cache_alloc_fail;	/* total failed allocations */
	uint64_t	cache_buftotal;		/* total buffers */
	uint64_t	cache_bufmax;		/* max buffers ever */
	uint64_t	cache_rescale;		/* # of hash table rescales */
	uint64_t	cache_lookup_depth;	/* hash lookup depth */
	uint64_t	cache_depot_contention;	/* mutex contention count */
	uint64_t	cache_depot_contention_prev; /* previous snapshot */

/*
	 * Cache properties
	 */
	char		cache_name[UMEM_CACHE_NAMELEN + 1];
	size_t		cache_bufsize;		/* object size */
	size_t		cache_bufsize_adi;	/* object cacheline size */
	size_t		cache_align;		/* object alignment */
	umem_constructor_t *cache_constructor;
	umem_destructor_t *cache_destructor;
	umem_reclaim_t	*cache_reclaim;
	umem_reclaim_t	*cache_postreclaim;
	void		*cache_private;		/* opaque arg to callbacks */
	vmem_t		*cache_arena;		/* vmem source for slabs */
	int		cache_cflags;		/* cache creation flags */
	int		cache_flags;		/* various cache state info */
	int		cache_uflags;		/* UMU_* flags */
	uint32_t	cache_mtbf;		/* induced alloc failure rate */
	umem_cache_t	*cache_next;		/* forward cache linkage */
	umem_cache_t	*cache_prev;		/* backward cache linkage */
	umem_cache_t	*cache_unext;		/* next in update list */
	umem_cache_t	*cache_uprev;		/* prev in update list */
	uint32_t	cache_cpu_mask;		/* mask for cpu offset */
	uint8_t		cache_chunkunused;	/* bytes at chunk start */

/*
	 * Padding to keep slab slayer layer word aligned.
	 */
	uint8_t	cache_pad1[3];

/*
	 * Slab layer
	 */
	mutex_t		cache_lock;		/* protects slab layer */
	size_t		cache_chunksize;	/* buf + alignment [+ debug] */
	size_t		cache_slabsize;		/* size of a slab */
	size_t		cache_bufctl;		/* buf-to-bufctl distance */
	size_t		cache_buftag;		/* buf-to-buftag distance */
	size_t		cache_verify;		/* bytes to verify */
	size_t		cache_contents;		/* bytes of saved content */
	size_t		cache_color;		/* next slab color */
	size_t		cache_mincolor;		/* maximum slab color */
	size_t		cache_maxcolor;		/* maximum slab color */
	size_t		cache_hash_shift;	/* get to interesting bits */
	size_t		cache_hash_mask;	/* hash table mask */
	umem_slab_t	*cache_freelist;	/* slab free list */
	umem_slab_t	cache_nullslab;		/* end of freelist marker */
	umem_cache_t	*cache_bufctl_cache;	/* source of bufctls */
	umem_bufctl_t	**cache_hash_table;	/* hash table base */
	uint64_t	cache_adi_vermap;	/* ADI version bitmap */

/*
	 * Padding to put per-CPU layer on its own cacheline.  We put the
	 * padding here, so that the depot layer is more likely to be on its
	 * own cacheline, and not have interference with the slab layer.
	 */
	uint8_t		cache_pad2[36];

/*
	 * Depot layer
	 */
	mutex_t		cache_depot_lock;	/* protects depot */
	umem_magtype_t	*cache_magtype;		/* magazine type */
	umem_maglist_t	cache_full;		/* full magazines */
	umem_maglist_t	cache_empty;		/* empty magazines */
#ifdef _LP64
	uint8_t		cache_pad3[8];		/* align cache_cpu to 16B */
#endif
	/*
	 * Per-CPU layer
	 */
	umem_cpu_cache_t cache_cpu[1];		/* cache_cpu_mask + 1 entries */
};

typedef struct umem_cpu_log_header {
	mutex_t		clh_lock;
	char		*clh_current;
	size_t		clh_avail;
	int		clh_chunk;
	int		clh_hits;
	char		clh_pad[64 - sizeof (mutex_t) - sizeof (char *) -
				sizeof (size_t) - 2 * sizeof (int)];
} umem_cpu_log_header_t;

typedef struct umem_log_header {
	mutex_t		lh_lock;
	char		*lh_base;
	int		*lh_free;
	size_t		lh_chunksize;
	int		lh_nchunks;
	int		lh_head;
	int		lh_tail;
	int		lh_hits;
	umem_cpu_log_header_t lh_cpu[1];	/* actually umem_max_ncpus */
} umem_log_header_t;

typedef struct umem_cpu {
	uint32_t cpu_cache_offset;
	uint32_t cpu_number;
} umem_cpu_t;

#define	UMEM_MIN_MAXBUF			16384
#define	UMEM_MAXBUF			1048576

#ifdef _LP64
#define	UMEM_ALIGN			16	/* min guaranteed alignment */
#define	UMEM_ALIGN_SHIFT		4	/* log2(UMEM_ALIGN) */
#else
#define	UMEM_ALIGN			8	/* min guaranteed alignment */
#define	UMEM_ALIGN_SHIFT		3	/* log2(UMEM_ALIGN) */
#endif /* _LP64 */

#define	UMEM_ALLOC_TABLE_MAX		(UMEM_MIN_MAXBUF >> UMEM_ALIGN_SHIFT)

#define	UMEM_BIG_MULTIPLE		8192
#define	UMEM_BIG_SHIFT			13	/* log2(UMEM_BIG_MULTIPLE) */
#define	UMEM_BIG_ALLOC_TABLE_MAX	(UMEM_MAXBUF >> UMEM_BIG_SHIFT)

#define	UMEM_VOID_FRACTION	8	/* never waste more than 1/8 of slab */

#define	MALLOC_MAGIC			0x3a10c000 /* 8-byte tag */
#define	MEMALIGN_MAGIC			0x3e3a1000

#ifdef _LP64
#define	MALLOC_OVERSIZE_MAGIC		0x06e47000 /* 16-byte tag, _LP64 */
#endif

#define	UMEM_MALLOC_ENCODE(type, sz)	(uint32_t)((type) - (sz))
#define	UMEM_MALLOC_DECODE(stat, sz)	(uint32_t)((stat) + (sz))
#define	UMEM_FREE_PATTERN_32		(uint32_t)(UMEM_FREE_PATTERN)

#define	UMU_MAGAZINE_RESIZE	0x00000001
#define	UMU_HASH_RESCALE	0x00000002
#define	UMU_REAP		0x00000004
#define	UMU_NOTIFY		0x08000000
#define	UMU_ACTIVE		0x80000000

typedef enum umem_ready_e {
	UMEM_READY_INIT_FAILED		= -1,
	UMEM_READY_STARTUP		= 1,
	UMEM_READY_INITING		= 2,
	UMEM_READY			= 3
} umem_ready_t;

/*
 * Internal flag for umem_alloc
 * Only malloc may use UMF_MALLOC
 */
#define	UMEM_MALLOC	0x1000
extern void umem_free_internal(void *, size_t, int);

#ifdef UMEM_STANDALONE
extern void umem_startup(caddr_t, size_t, size_t, caddr_t, caddr_t);
extern int umem_add(caddr_t, size_t);
#endif

typedef enum umem_err {
	UMERR_MODIFIED = 0,
	UMERR_REDZONE,
	UMERR_DUPFREE,
	UMERR_BADADDR,
	UMERR_BADBUFTAG,
	UMERR_BADBUFCTL,
	UMERR_BADCACHE,
	UMERR_BADSIZE,
	UMERR_BADBASE,
	UMERR_ADI_BADOFFSET
} umem_err_t;

typedef struct umem_abort_info_s {
	hrtime_t	ump_timestamp;	/* timestamp of error */
	umem_err_t	ump_error;	/* type of umem error (UMERR_*) */
	void		*ump_buffer;	/* buffer that induced abort */
	void		*ump_realbuf;	/* real start address for buffer */
	umem_cache_t	*ump_cache;	/* buffer's cache according to client */
	umem_cache_t	*ump_realcache;	/* actual cache containing buffer */
	umem_slab_t	*ump_slab;	/* slab accoring to umem_findslab() */
	umem_bufctl_t	*ump_bufctl;	/* bufctl */
} umem_abort_info_t;

#ifdef	__cplusplus
}
#endif

#endif	/* _UMEM_IMPL_H */