/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2018, Joyent, Inc.
 */

/*
 * Kernel asynchronous I/O.
 * This is only for raw devices now (as of Nov. 1993).
 */

#include <sys/types.h>
#include <sys/errno.h>
#include <sys/conf.h>
#include <sys/file.h>
#include <sys/fs/snode.h>
#include <sys/unistd.h>
#include <sys/cmn_err.h>
#include <vm/as.h>
#include <vm/faultcode.h>
#include <sys/sysmacros.h>
#include <sys/procfs.h>
#include <sys/kmem.h>
#include <sys/autoconf.h>
#include <sys/ddi_impldefs.h>
#include <sys/sunddi.h>
#include <sys/aio_impl.h>
#include <sys/debug.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/vmsystm.h>
#include <sys/contract/process_impl.h>

/*
 * external entry point.
 */
#ifdef _LP64
static int64_t kaioc(long, long, long, long, long, long);
#endif
static int kaio(ulong_t *, rval_t *);


#define	AIO_64	0
#define	AIO_32	1
#define	AIO_LARGEFILE	2

/*
 * implementation specific functions (private)
 */
#ifdef _LP64
static int alio(int, aiocb_t **, int, struct sigevent *);
#endif
static int aionotify(void);
static int aioinit(void);
static int aiostart(void);
static void alio_cleanup(aio_t *, aiocb_t **, int, int);
static int (*check_vp(struct vnode *, int))(vnode_t *, struct aio_req *,
    cred_t *);
static void lio_set_error(aio_req_t *, int portused);
static aio_t *aio_aiop_alloc();
static int aio_req_alloc(aio_req_t **, aio_result_t *);
static int aio_lio_alloc(aio_lio_t **);
static aio_req_t *aio_req_done(void *);
static aio_req_t *aio_req_remove(aio_req_t *);
static int aio_req_find(aio_result_t *, aio_req_t **);
static int aio_hash_insert(struct aio_req_t *, aio_t *);
static int aio_req_setup(aio_req_t **, aio_t *, aiocb_t *,
    aio_result_t *, vnode_t *, int);
static int aio_cleanup_thread(aio_t *);
static aio_lio_t *aio_list_get(aio_result_t *);
static void lio_set_uerror(void *, int);
extern void aio_zerolen(aio_req_t *);
static int aiowait(struct timeval *, int, long	*);
static int aiowaitn(void *, uint_t, uint_t *, timespec_t *);
static int aio_unlock_requests(caddr_t iocblist, int iocb_index,
    aio_req_t *reqlist, aio_t *aiop, model_t model);
static int aio_reqlist_concat(aio_t *aiop, aio_req_t **reqlist, int max);
static int aiosuspend(void *, int, struct  timespec *, int,
    long	*, int);
static int aliowait(int, void *, int, void *, int);
static int aioerror(void *, int);
static int aio_cancel(int, void *, long	*, int);
static int arw(int, int, char *, int, offset_t, aio_result_t *, int);
static int aiorw(int, void *, int, int);

static int alioLF(int, void *, int, void *);
static int aio_req_setupLF(aio_req_t **, aio_t *, aiocb64_32_t *,
    aio_result_t *, vnode_t *, int);
static int alio32(int, void *, int, void *);
static int driver_aio_write(vnode_t *vp, struct aio_req *aio, cred_t *cred_p);
static int driver_aio_read(vnode_t *vp, struct aio_req *aio, cred_t *cred_p);

#ifdef  _SYSCALL32_IMPL
static void aiocb_LFton(aiocb64_32_t *, aiocb_t *);
void	aiocb_32ton(aiocb32_t *, aiocb_t *);
#endif /* _SYSCALL32_IMPL */

/*
 * implementation specific functions (external)
 */
void aio_req_free(aio_t *, aio_req_t *);

/*
 * Event Port framework
 */

void aio_req_free_port(aio_t *, aio_req_t *);
static int aio_port_callback(void *, int *, pid_t, int, void *);

/*
 * This is the loadable module wrapper.
 */
#include <sys/modctl.h>
#include <sys/syscall.h>

#ifdef _LP64

static struct sysent kaio_sysent = {
	6,
	SE_NOUNLOAD | SE_64RVAL | SE_ARGC,
	(int (*)())kaioc
};

#ifdef _SYSCALL32_IMPL
static struct sysent kaio_sysent32 = {
	7,
	SE_NOUNLOAD | SE_64RVAL,
	kaio
};
#endif  /* _SYSCALL32_IMPL */

#else   /* _LP64 */

static struct sysent kaio_sysent = {
	7,
	SE_NOUNLOAD | SE_32RVAL1,
	kaio
};

#endif  /* _LP64 */

/*
 * Module linkage information for the kernel.
 */

static struct modlsys modlsys = {
	&mod_syscallops,
	"kernel Async I/O",
	&kaio_sysent
};

#ifdef  _SYSCALL32_IMPL
static struct modlsys modlsys32 = {
	&mod_syscallops32,
	"kernel Async I/O for 32 bit compatibility",
	&kaio_sysent32
};
#endif  /* _SYSCALL32_IMPL */


static struct modlinkage modlinkage = {
	MODREV_1,
	&modlsys,
#ifdef  _SYSCALL32_IMPL
	&modlsys32,
#endif
	NULL
};

int
_init(void)
{
	int retval;

	if ((retval = mod_install(&modlinkage)) != 0)
		return (retval);

	return (0);
}

int
_fini(void)
{
	int retval;

	retval = mod_remove(&modlinkage);

	return (retval);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

#ifdef	_LP64
static int64_t
kaioc(
	long	a0,
	long	a1,
	long	a2,
	long	a3,
	long	a4,
	long	a5)
{
	int	error;
	long	rval = 0;

	switch ((int)a0 & ~AIO_POLL_BIT) {
	case AIOREAD:
		error = arw((int)a0, (int)a1, (char *)a2, (int)a3,
		    (offset_t)a4, (aio_result_t *)a5, FREAD);
		break;
	case AIOWRITE:
		error = arw((int)a0, (int)a1, (char *)a2, (int)a3,
		    (offset_t)a4, (aio_result_t *)a5, FWRITE);
		break;
	case AIOWAIT:
		error = aiowait((struct timeval *)a1, (int)a2, &rval);
		break;
	case AIOWAITN:
		error = aiowaitn((void *)a1, (uint_t)a2, (uint_t *)a3,
		    (timespec_t *)a4);
		break;
	case AIONOTIFY:
		error = aionotify();
		break;
	case AIOINIT:
		error = aioinit();
		break;
	case AIOSTART:
		error = aiostart();
		break;
	case AIOLIO:
		error = alio((int)a1, (aiocb_t **)a2, (int)a3,
		    (struct sigevent *)a4);
		break;
	case AIOLIOWAIT:
		error = aliowait((int)a1, (void *)a2, (int)a3,
		    (struct sigevent *)a4, AIO_64);
		break;
	case AIOSUSPEND:
		error = aiosuspend((void *)a1, (int)a2, (timespec_t *)a3,
		    (int)a4, &rval, AIO_64);
		break;
	case AIOERROR:
		error = aioerror((void *)a1, AIO_64);
		break;
	case AIOAREAD:
		error = aiorw((int)a0, (void *)a1, FREAD, AIO_64);
		break;
	case AIOAWRITE:
		error = aiorw((int)a0, (void *)a1, FWRITE, AIO_64);
		break;
	case AIOCANCEL:
		error = aio_cancel((int)a1, (void *)a2, &rval, AIO_64);
		break;

	/*
	 * The large file related stuff is valid only for
	 * 32 bit kernel and not for 64 bit kernel
	 * On 64 bit kernel we convert large file calls
	 * to regular 64bit calls.
	 */

	default:
		error = EINVAL;
	}
	if (error)
		return ((int64_t)set_errno(error));
	return (rval);
}
#endif

static int
kaio(
	ulong_t *uap,
	rval_t *rvp)
{
	long rval = 0;
	int	error = 0;
	offset_t	off;


	rvp->r_vals = 0;
#if defined(_LITTLE_ENDIAN)
	off = ((uoff_t)uap[5] << 32) | (uoff_t)uap[4];
#else
	off = ((uoff_t)uap[4] << 32) | (uoff_t)uap[5];
#endif

	switch (uap[0] & ~AIO_POLL_BIT) {
	/*
	 * It must be the 32 bit system call on 64 bit kernel
	 */
	case AIOREAD:
		return (arw((int)uap[0], (int)uap[1], (char *)uap[2],
		    (int)uap[3], off, (aio_result_t *)uap[6], FREAD));
	case AIOWRITE:
		return (arw((int)uap[0], (int)uap[1], (char *)uap[2],
		    (int)uap[3], off, (aio_result_t *)uap[6], FWRITE));
	case AIOWAIT:
		error = aiowait((struct	timeval *)uap[1], (int)uap[2],
		    &rval);
		break;
	case AIOWAITN:
		error = aiowaitn((void *)uap[1], (uint_t)uap[2],
		    (uint_t *)uap[3], (timespec_t *)uap[4]);
		break;
	case AIONOTIFY:
		return (aionotify());
	case AIOINIT:
		return (aioinit());
	case AIOSTART:
		return (aiostart());
	case AIOLIO:
		return (alio32((int)uap[1], (void *)uap[2], (int)uap[3],
		    (void *)uap[4]));
	case AIOLIOWAIT:
		return (aliowait((int)uap[1], (void *)uap[2],
		    (int)uap[3], (struct sigevent *)uap[4], AIO_32));
	case AIOSUSPEND:
		error = aiosuspend((void *)uap[1], (int)uap[2],
		    (timespec_t *)uap[3], (int)uap[4],
		    &rval, AIO_32);
		break;
	case AIOERROR:
		return (aioerror((void *)uap[1], AIO_32));
	case AIOAREAD:
		return (aiorw((int)uap[0], (void *)uap[1],
		    FREAD, AIO_32));
	case AIOAWRITE:
		return (aiorw((int)uap[0], (void *)uap[1],
		    FWRITE, AIO_32));
	case AIOCANCEL:
		error = (aio_cancel((int)uap[1], (void *)uap[2], &rval,
		    AIO_32));
		break;
	case AIOLIO64:
		return (alioLF((int)uap[1], (void *)uap[2],
		    (int)uap[3], (void *)uap[4]));
	case AIOLIOWAIT64:
		return (aliowait(uap[1], (void *)uap[2],
		    (int)uap[3], (void *)uap[4], AIO_LARGEFILE));
	case AIOSUSPEND64:
		error = aiosuspend((void *)uap[1], (int)uap[2],
		    (timespec_t *)uap[3], (int)uap[4], &rval,
		    AIO_LARGEFILE);
		break;
	case AIOERROR64:
		return (aioerror((void *)uap[1], AIO_LARGEFILE));
	case AIOAREAD64:
		return (aiorw((int)uap[0], (void *)uap[1], FREAD,
		    AIO_LARGEFILE));
	case AIOAWRITE64:
		return (aiorw((int)uap[0], (void *)uap[1], FWRITE,
		    AIO_LARGEFILE));
	case AIOCANCEL64:
		error = (aio_cancel((int)uap[1], (void *)uap[2],
		    &rval, AIO_LARGEFILE));
		break;
	default:
		return (EINVAL);
	}

	rvp->r_val1 = rval;
	return (error);
}

/*
 * wake up LWPs in this process that are sleeping in
 * aiowait().
 */
static int
aionotify(void)
{
	aio_t	*aiop;

	aiop = curproc->p_aio;
	if (aiop == NULL)
		return (0);

	mutex_enter(&aiop->aio_mutex);
	aiop->aio_notifycnt++;
	cv_broadcast(&aiop->aio_waitcv);
	mutex_exit(&aiop->aio_mutex);

	return (0);
}

static int
timeval2reltime(struct timeval *timout, timestruc_t *rqtime,
    timestruc_t **rqtp, int *blocking)
{
#ifdef	_SYSCALL32_IMPL
	struct timeval32 wait_time_32;
#endif
	struct timeval wait_time;
	model_t	model = get_udatamodel();

	*rqtp = NULL;
	if (timout == NULL) {		/* wait indefinitely */
		*blocking = 1;
		return (0);
	}

	/*
	 * Need to correctly compare with the -1 passed in for a user
	 * address pointer, with both 32 bit and 64 bit apps.
	 */
	if (model == DATAMODEL_NATIVE) {
		if ((intptr_t)timout == (intptr_t)-1) {	/* don't wait */
			*blocking = 0;
			return (0);
		}

		if (copyin(timout, &wait_time, sizeof (wait_time)))
			return (EFAULT);
	}
#ifdef	_SYSCALL32_IMPL
	else {
		/*
		 * -1 from a 32bit app. It will not get sign extended.
		 * don't wait if -1.
		 */
		if ((intptr_t)timout == (intptr_t)((uint32_t)-1)) {
			*blocking = 0;
			return (0);
		}

		if (copyin(timout, &wait_time_32, sizeof (wait_time_32)))
			return (EFAULT);
		TIMEVAL32_TO_TIMEVAL(&wait_time, &wait_time_32);
	}
#endif  /* _SYSCALL32_IMPL */

	if (wait_time.tv_sec == 0 && wait_time.tv_usec == 0) {	/* don't wait */
		*blocking = 0;
		return (0);
	}

	if (wait_time.tv_sec < 0 ||
	    wait_time.tv_usec < 0 || wait_time.tv_usec >= MICROSEC)
		return (EINVAL);

	rqtime->tv_sec = wait_time.tv_sec;
	rqtime->tv_nsec = wait_time.tv_usec * 1000;
	*rqtp = rqtime;
	*blocking = 1;

	return (0);
}

static int
timespec2reltime(timespec_t *timout, timestruc_t *rqtime,
    timestruc_t **rqtp, int *blocking)
{
#ifdef	_SYSCALL32_IMPL
	timespec32_t wait_time_32;
#endif
	model_t	model = get_udatamodel();

	*rqtp = NULL;
	if (timout == NULL) {
		*blocking = 1;
		return (0);
	}

	if (model == DATAMODEL_NATIVE) {
		if (copyin(timout, rqtime, sizeof (*rqtime)))
			return (EFAULT);
	}
#ifdef	_SYSCALL32_IMPL
	else {
		if (copyin(timout, &wait_time_32, sizeof (wait_time_32)))
			return (EFAULT);
		TIMESPEC32_TO_TIMESPEC(rqtime, &wait_time_32);
	}
#endif  /* _SYSCALL32_IMPL */

	if (rqtime->tv_sec == 0 && rqtime->tv_nsec == 0) {
		*blocking = 0;
		return (0);
	}

	if (rqtime->tv_sec < 0 ||
	    rqtime->tv_nsec < 0 || rqtime->tv_nsec >= NANOSEC)
		return (EINVAL);

	*rqtp = rqtime;
	*blocking = 1;

	return (0);
}

/*ARGSUSED*/
static int
aiowait(
	struct timeval	*timout,
	int	dontblockflg,
	long	*rval)
{
	int 		error;
	aio_t		*aiop;
	aio_req_t	*reqp;
	clock_t		status;
	int		blocking;
	int		timecheck;
	timestruc_t	rqtime;
	timestruc_t	*rqtp;

	aiop = curproc->p_aio;
	if (aiop == NULL)
		return (EINVAL);

	/*
	 * Establish the absolute future time for the timeout.
	 */
	error = timeval2reltime(timout, &rqtime, &rqtp, &blocking);
	if (error)
		return (error);
	if (rqtp) {
		timestruc_t now;
		timecheck = timechanged;
		gethrestime(&now);
		timespecadd(rqtp, &now);
	}

	mutex_enter(&aiop->aio_mutex);
	for (;;) {
		/* process requests on poll queue */
		if (aiop->aio_pollq) {
			mutex_exit(&aiop->aio_mutex);
			aio_cleanup(0);
			mutex_enter(&aiop->aio_mutex);
		}
		if ((reqp = aio_req_remove(NULL)) != NULL) {
			*rval = (long)reqp->aio_req_resultp;
			break;
		}
		/* user-level done queue might not be empty */
		if (aiop->aio_notifycnt > 0) {
			aiop->aio_notifycnt--;
			*rval = 1;
			break;
		}
		/* don't block if no outstanding aio */
		if (aiop->aio_outstanding == 0 && dontblockflg) {
			error = EINVAL;
			break;
		}
		if (blocking) {
			status = cv_waituntil_sig(&aiop->aio_waitcv,
			    &aiop->aio_mutex, rqtp, timecheck);

			if (status > 0)		/* check done queue again */
				continue;
			if (status == 0) {	/* interrupted by a signal */
				error = EINTR;
				*rval = -1;
			} else {		/* timer expired */
				error = ETIME;
			}
		}
		break;
	}
	mutex_exit(&aiop->aio_mutex);
	if (reqp) {
		aphysio_unlock(reqp);
		aio_copyout_result(reqp);
		mutex_enter(&aiop->aio_mutex);
		aio_req_free(aiop, reqp);
		mutex_exit(&aiop->aio_mutex);
	}
	return (error);
}

/*
 * aiowaitn can be used to reap completed asynchronous requests submitted with
 * lio_listio, aio_read or aio_write.
 * This function only reaps asynchronous raw I/Os.
 */

/*ARGSUSED*/
static int
aiowaitn(void *uiocb, uint_t nent, uint_t *nwait, timespec_t *timout)
{
	int 		error = 0;
	aio_t		*aiop;
	aio_req_t	*reqlist = NULL;
	caddr_t		iocblist = NULL;	/* array of iocb ptr's */
	uint_t		waitcnt, cnt = 0;	/* iocb cnt */
	size_t		iocbsz;			/* users iocb size */
	size_t		riocbsz;		/* returned iocb size */
	int		iocb_index = 0;
	model_t		model = get_udatamodel();
	int		blocking = 1;
	int		timecheck;
	timestruc_t	rqtime;
	timestruc_t	*rqtp;

	aiop = curproc->p_aio;
	if (aiop == NULL || nent == 0 || nent > _AIO_LISTIO_MAX)
		return (EINVAL);

	if (aiop->aio_outstanding == 0)
		return (EAGAIN);

	if (copyin(nwait, &waitcnt, sizeof (uint_t)))
		return (EFAULT);

	/* set *nwait to zero, if we must return prematurely */
	if (copyout(&cnt, nwait, sizeof (uint_t)))
		return (EFAULT);

	if (waitcnt == 0) {
		blocking = 0;
		rqtp = NULL;
		waitcnt = nent;
	} else {
		error = timespec2reltime(timout, &rqtime, &rqtp, &blocking);
		if (error)
			return (error);
	}

	if (model == DATAMODEL_NATIVE)
		iocbsz = (sizeof (aiocb_t *) * nent);
#ifdef	_SYSCALL32_IMPL
	else
		iocbsz = (sizeof (caddr32_t) * nent);
#endif  /* _SYSCALL32_IMPL */

	/*
	 * Only one aio_waitn call is allowed at a time.
	 * The active aio_waitn will collect all requests
	 * out of the "done" list and if necessary it will wait
	 * for some/all pending requests to fulfill the nwait
	 * parameter.
	 * A second or further aio_waitn calls will sleep here
	 * until the active aio_waitn finishes and leaves the kernel
	 * If the second call does not block (poll), then return
	 * immediately with the error code : EAGAIN.
	 * If the second call should block, then sleep here, but
	 * do not touch the timeout. The timeout starts when this
	 * aio_waitn-call becomes active.
	 */

	mutex_enter(&aiop->aio_mutex);

	while (aiop->aio_flags & AIO_WAITN) {
		if (blocking == 0) {
			mutex_exit(&aiop->aio_mutex);
			return (EAGAIN);
		}

		/* block, no timeout */
		aiop->aio_flags |= AIO_WAITN_PENDING;
		if (!cv_wait_sig(&aiop->aio_waitncv, &aiop->aio_mutex)) {
			mutex_exit(&aiop->aio_mutex);
			return (EINTR);
		}
	}

	/*
	 * Establish the absolute future time for the timeout.
	 */
	if (rqtp) {
		timestruc_t now;
		timecheck = timechanged;
		gethrestime(&now);
		timespecadd(rqtp, &now);
	}

	if (iocbsz > aiop->aio_iocbsz && aiop->aio_iocb != NULL) {
		kmem_free(aiop->aio_iocb, aiop->aio_iocbsz);
		aiop->aio_iocb = NULL;
	}

	if (aiop->aio_iocb == NULL) {
		iocblist = kmem_zalloc(iocbsz, KM_NOSLEEP);
		if (iocblist == NULL) {
			mutex_exit(&aiop->aio_mutex);
			return (ENOMEM);
		}
		aiop->aio_iocb = (aiocb_t **)iocblist;
		aiop->aio_iocbsz = iocbsz;
	} else {
		iocblist = (char *)aiop->aio_iocb;
	}

	aiop->aio_waitncnt = waitcnt;
	aiop->aio_flags |= AIO_WAITN;

	for (;;) {
		/* push requests on poll queue to done queue */
		if (aiop->aio_pollq) {
			mutex_exit(&aiop->aio_mutex);
			aio_cleanup(0);
			mutex_enter(&aiop->aio_mutex);
		}

		/* check for requests on done queue */
		if (aiop->aio_doneq) {
			cnt += aio_reqlist_concat(aiop, &reqlist, nent - cnt);
			aiop->aio_waitncnt = waitcnt - cnt;
		}

		/* user-level done queue might not be empty */
		if (aiop->aio_notifycnt > 0) {
			aiop->aio_notifycnt--;
			error = 0;
			break;
		}

		/*
		 * if we are here second time as a result of timer
		 * expiration, we reset error if there are enough
		 * aiocb's to satisfy request.
		 * We return also if all requests are already done
		 * and we picked up the whole done queue.
		 */

		if ((cnt >= waitcnt) || (cnt > 0 && aiop->aio_pending == 0 &&
		    aiop->aio_doneq == NULL)) {
			error = 0;
			break;
		}

		if ((cnt < waitcnt) && blocking) {
			int rval = cv_waituntil_sig(&aiop->aio_waitcv,
			    &aiop->aio_mutex, rqtp, timecheck);
			if (rval > 0)
				continue;
			if (rval < 0) {
				error = ETIME;
				blocking = 0;
				continue;
			}
			error = EINTR;
		}
		break;
	}

	mutex_exit(&aiop->aio_mutex);

	if (cnt > 0) {

		iocb_index = aio_unlock_requests(iocblist, iocb_index, reqlist,
		    aiop, model);

		if (model == DATAMODEL_NATIVE)
			riocbsz = (sizeof (aiocb_t *) * cnt);
#ifdef	_SYSCALL32_IMPL
		else
			riocbsz = (sizeof (caddr32_t) * cnt);
#endif  /* _SYSCALL32_IMPL */

		if (copyout(iocblist, uiocb, riocbsz) ||
		    copyout(&cnt, nwait, sizeof (uint_t)))
			error = EFAULT;
	}

	/* check if there is another thread waiting for execution */
	mutex_enter(&aiop->aio_mutex);
	aiop->aio_flags &= ~AIO_WAITN;
	if (aiop->aio_flags & AIO_WAITN_PENDING) {
		aiop->aio_flags &= ~AIO_WAITN_PENDING;
		cv_signal(&aiop->aio_waitncv);
	}
	mutex_exit(&aiop->aio_mutex);

	return (error);
}

/*
 * aio_unlock_requests
 * copyouts the result of the request as well as the return value.
 * It builds the list of completed asynchronous requests,
 * unlocks the allocated memory ranges and
 * put the aio request structure back into the free list.
 */

static int
aio_unlock_requests(
	caddr_t	iocblist,
	int	iocb_index,
	aio_req_t *reqlist,
	aio_t	*aiop,
	model_t	model)
{
	aio_req_t	*reqp, *nreqp;

	if (model == DATAMODEL_NATIVE) {
		for (reqp = reqlist; reqp != NULL;  reqp = nreqp) {
			(((caddr_t *)iocblist)[iocb_index++]) =
			    reqp->aio_req_iocb.iocb;
			nreqp = reqp->aio_req_next;
			aphysio_unlock(reqp);
			aio_copyout_result(reqp);
			mutex_enter(&aiop->aio_mutex);
			aio_req_free(aiop, reqp);
			mutex_exit(&aiop->aio_mutex);
		}
	}
#ifdef	_SYSCALL32_IMPL
	else {
		for (reqp = reqlist; reqp != NULL;  reqp = nreqp) {
			((caddr32_t *)iocblist)[iocb_index++] =
			    reqp->aio_req_iocb.iocb32;
			nreqp = reqp->aio_req_next;
			aphysio_unlock(reqp);
			aio_copyout_result(reqp);
			mutex_enter(&aiop->aio_mutex);
			aio_req_free(aiop, reqp);
			mutex_exit(&aiop->aio_mutex);
		}
	}
#endif	/* _SYSCALL32_IMPL */
	return (iocb_index);
}

/*
 * aio_reqlist_concat
 * moves "max" elements from the done queue to the reqlist queue and removes
 * the AIO_DONEQ flag.
 * - reqlist queue is a simple linked list
 * - done queue is a double linked list
 */

static int
aio_reqlist_concat(aio_t *aiop, aio_req_t **reqlist, int max)
{
	aio_req_t *q2, *q2work, *list;
	int count = 0;

	list = *reqlist;
	q2 = aiop->aio_doneq;
	q2work = q2;
	while (max-- > 0) {
		q2work->aio_req_flags &= ~AIO_DONEQ;
		q2work = q2work->aio_req_next;
		count++;
		if (q2work == q2)
			break;
	}

	if (q2work == q2) {
		/* all elements revised */
		q2->aio_req_prev->aio_req_next = list;
		list = q2;
		aiop->aio_doneq = NULL;
	} else {
		/*
		 * max < elements in the doneq
		 * detach only the required amount of elements
		 * out of the doneq
		 */
		q2work->aio_req_prev->aio_req_next = list;
		list = q2;

		aiop->aio_doneq = q2work;
		q2work->aio_req_prev = q2->aio_req_prev;
		q2->aio_req_prev->aio_req_next = q2work;
	}
	*reqlist = list;
	return (count);
}

/*ARGSUSED*/
static int
aiosuspend(
	void	*aiocb,
	int	nent,
	struct	timespec	*timout,
	int	flag,
	long	*rval,
	int	run_mode)
{
	int 		error;
	aio_t		*aiop;
	aio_req_t	*reqp, *found, *next;
	caddr_t		cbplist = NULL;
	aiocb_t		*cbp, **ucbp;
#ifdef	_SYSCALL32_IMPL
	aiocb32_t	*cbp32;
	caddr32_t	*ucbp32;
#endif  /* _SYSCALL32_IMPL */
	aiocb64_32_t	*cbp64;
	int		rv;
	int		i;
	size_t		ssize;
	model_t		model = get_udatamodel();
	int		blocking;
	int		timecheck;
	timestruc_t	rqtime;
	timestruc_t	*rqtp;

	aiop = curproc->p_aio;
	if (aiop == NULL || nent <= 0 || nent > _AIO_LISTIO_MAX)
		return (EINVAL);

	/*
	 * Establish the absolute future time for the timeout.
	 */
	error = timespec2reltime(timout, &rqtime, &rqtp, &blocking);
	if (error)
		return (error);
	if (rqtp) {
		timestruc_t now;
		timecheck = timechanged;
		gethrestime(&now);
		timespecadd(rqtp, &now);
	}

	/*
	 * If we are not blocking and there's no IO complete
	 * skip aiocb copyin.
	 */
	if (!blocking && (aiop->aio_pollq == NULL) &&
	    (aiop->aio_doneq == NULL)) {
		return (EAGAIN);
	}

	if (model == DATAMODEL_NATIVE)
		ssize = (sizeof (aiocb_t *) * nent);
#ifdef	_SYSCALL32_IMPL
	else
		ssize = (sizeof (caddr32_t) * nent);
#endif  /* _SYSCALL32_IMPL */

	cbplist = kmem_alloc(ssize, KM_NOSLEEP);
	if (cbplist == NULL)
		return (ENOMEM);

	if (copyin(aiocb, cbplist, ssize)) {
		error = EFAULT;
		goto done;
	}

	found = NULL;
	/*
	 * we need to get the aio_cleanupq_mutex since we call
	 * aio_req_done().
	 */
	mutex_enter(&aiop->aio_cleanupq_mutex);
	mutex_enter(&aiop->aio_mutex);
	for (;;) {
		/* push requests on poll queue to done queue */
		if (aiop->aio_pollq) {
			mutex_exit(&aiop->aio_mutex);
			mutex_exit(&aiop->aio_cleanupq_mutex);
			aio_cleanup(0);
			mutex_enter(&aiop->aio_cleanupq_mutex);
			mutex_enter(&aiop->aio_mutex);
		}
		/* check for requests on done queue */
		if (aiop->aio_doneq) {
			if (model == DATAMODEL_NATIVE)
				ucbp = (aiocb_t **)cbplist;
#ifdef	_SYSCALL32_IMPL
			else
				ucbp32 = (caddr32_t *)cbplist;
#endif  /* _SYSCALL32_IMPL */
			for (i = 0; i < nent; i++) {
				if (model == DATAMODEL_NATIVE) {
					if ((cbp = *ucbp++) == NULL)
						continue;
					if (run_mode != AIO_LARGEFILE)
						reqp = aio_req_done(
						    &cbp->aio_resultp);
					else {
						cbp64 = (aiocb64_32_t *)cbp;
						reqp = aio_req_done(
						    &cbp64->aio_resultp);
					}
				}
#ifdef	_SYSCALL32_IMPL
				else {
					if (run_mode == AIO_32) {
						if ((cbp32 =
						    (aiocb32_t *)(uintptr_t)
						    *ucbp32++) == NULL)
							continue;
						reqp = aio_req_done(
						    &cbp32->aio_resultp);
					} else if (run_mode == AIO_LARGEFILE) {
						if ((cbp64 =
						    (aiocb64_32_t *)(uintptr_t)
						    *ucbp32++) == NULL)
							continue;
						reqp = aio_req_done(
						    &cbp64->aio_resultp);
					}

				}
#endif  /* _SYSCALL32_IMPL */
				if (reqp) {
					reqp->aio_req_next = found;
					found = reqp;
				}
				if (aiop->aio_doneq == NULL)
					break;
			}
			if (found)
				break;
		}
		if (aiop->aio_notifycnt > 0) {
			/*
			 * nothing on the kernel's queue. the user
			 * has notified the kernel that it has items
			 * on a user-level queue.
			 */
			aiop->aio_notifycnt--;
			*rval = 1;
			error = 0;
			break;
		}
		/* don't block if nothing is outstanding */
		if (aiop->aio_outstanding == 0) {
			error = EAGAIN;
			break;
		}
		if (blocking) {
			/*
			 * drop the aio_cleanupq_mutex as we are
			 * going to block.
			 */
			mutex_exit(&aiop->aio_cleanupq_mutex);
			rv = cv_waituntil_sig(&aiop->aio_waitcv,
			    &aiop->aio_mutex, rqtp, timecheck);
			/*
			 * we have to drop aio_mutex and
			 * grab it in the right order.
			 */
			mutex_exit(&aiop->aio_mutex);
			mutex_enter(&aiop->aio_cleanupq_mutex);
			mutex_enter(&aiop->aio_mutex);
			if (rv > 0)	/* check done queue again */
				continue;
			if (rv == 0)	/* interrupted by a signal */
				error = EINTR;
			else		/* timer expired */
				error = ETIME;
		} else {
			error = EAGAIN;
		}
		break;
	}
	mutex_exit(&aiop->aio_mutex);
	mutex_exit(&aiop->aio_cleanupq_mutex);
	for (reqp = found; reqp != NULL; reqp = next) {
		next = reqp->aio_req_next;
		aphysio_unlock(reqp);
		aio_copyout_result(reqp);
		mutex_enter(&aiop->aio_mutex);
		aio_req_free(aiop, reqp);
		mutex_exit(&aiop->aio_mutex);
	}
done:
	kmem_free(cbplist, ssize);
	return (error);
}

/*
 * initialize aio by allocating an aio_t struct for this
 * process.
 */
static int
aioinit(void)
{
	proc_t *p = curproc;
	aio_t *aiop;
	mutex_enter(&p->p_lock);
	if ((aiop = p->p_aio) == NULL) {
		aiop = aio_aiop_alloc();
		p->p_aio = aiop;
	}
	mutex_exit(&p->p_lock);
	if (aiop == NULL)
		return (ENOMEM);
	return (0);
}

/*
 * start a special thread that will cleanup after aio requests
 * that are preventing a segment from being unmapped. as_unmap()
 * blocks until all phsyio to this segment is completed. this
 * doesn't happen until all the pages in this segment are not
 * SOFTLOCKed. Some pages will be SOFTLOCKed when there are aio
 * requests still outstanding. this special thread will make sure
 * that these SOFTLOCKed pages will eventually be SOFTUNLOCKed.
 *
 * this function will return an error if the process has only
 * one LWP. the assumption is that the caller is a separate LWP
 * that remains blocked in the kernel for the life of this process.
 */
static int
aiostart(void)
{
	proc_t *p = curproc;
	aio_t *aiop;
	int first, error = 0;

	if (p->p_lwpcnt == 1)
		return (EDEADLK);
	mutex_enter(&p->p_lock);
	if ((aiop = p->p_aio) == NULL)
		error = EINVAL;
	else {
		first = aiop->aio_ok;
		if (aiop->aio_ok == 0)
			aiop->aio_ok = 1;
	}
	mutex_exit(&p->p_lock);
	if (error == 0 && first == 0) {
		return (aio_cleanup_thread(aiop));
		/* should return only to exit */
	}
	return (error);
}

/*
 * Associate an aiocb with a port.
 * This function is used by aiorw() to associate a transaction with a port.
 * Allocate an event port structure (port_alloc_event()) and store the
 * delivered user pointer (portnfy_user) in the portkev_user field of the
 * port_kevent_t structure..
 * The aio_req_portkev pointer in the aio_req_t structure was added to identify
 * the port association.
 */

static int
aio_req_assoc_port_rw(port_notify_t *pntfy, aiocb_t *cbp,
    aio_req_t *reqp, int event)
{
	port_kevent_t	*pkevp = NULL;
	int		error;

	error = port_alloc_event(pntfy->portnfy_port, PORT_ALLOC_DEFAULT,
	    PORT_SOURCE_AIO, &pkevp);
	if (error) {
		if ((error == ENOMEM) || (error == EAGAIN))
			error = EAGAIN;
		else
			error = EINVAL;
	} else {
		port_init_event(pkevp, (uintptr_t)cbp, pntfy->portnfy_user,
		    aio_port_callback, reqp);
		pkevp->portkev_events = event;
		reqp->aio_req_portkev = pkevp;
		reqp->aio_req_port = pntfy->portnfy_port;
	}
	return (error);
}

#ifdef _LP64

/*
 * Asynchronous list IO. A chain of aiocb's are copied in
 * one at a time. If the aiocb is invalid, it is skipped.
 * For each aiocb, the appropriate driver entry point is
 * called. Optimize for the common case where the list
 * of requests is to the same file descriptor.
 *
 * One possible optimization is to define a new driver entry
 * point that supports a list of IO requests. Whether this
 * improves performance depends somewhat on the driver's
 * locking strategy. Processing a list could adversely impact
 * the driver's interrupt latency.
 */
static int
alio(
	int		mode_arg,
	aiocb_t		**aiocb_arg,
	int		nent,
	struct sigevent	*sigev)
{
	file_t		*fp;
	file_t		*prev_fp = NULL;
	int		prev_mode = -1;
	struct vnode	*vp;
	aio_lio_t	*head;
	aio_req_t	*reqp;
	aio_t		*aiop;
	caddr_t		cbplist;
	aiocb_t		cb;
	aiocb_t		*aiocb = &cb;
	aiocb_t		*cbp;
	aiocb_t		**ucbp;
	struct sigevent sigevk;
	sigqueue_t	*sqp;
	int		(*aio_func)();
	int		mode;
	int		error = 0;
	int		aio_errors = 0;
	int		i;
	size_t		ssize;
	int		deadhead = 0;
	int		aio_notsupported = 0;
	int		lio_head_port;
	int		aio_port;
	int		aio_thread;
	port_kevent_t	*pkevtp = NULL;
	int		portused = 0;
	port_notify_t	pnotify;
	int		event;

	aiop = curproc->p_aio;
	if (aiop == NULL || nent <= 0 || nent > _AIO_LISTIO_MAX)
		return (EINVAL);

	ssize = (sizeof (aiocb_t *) * nent);
	cbplist = kmem_alloc(ssize, KM_SLEEP);
	ucbp = (aiocb_t **)cbplist;

	if (copyin(aiocb_arg, cbplist, ssize) ||
	    (sigev && copyin(sigev, &sigevk, sizeof (struct sigevent)))) {
		kmem_free(cbplist, ssize);
		return (EFAULT);
	}

	/* Event Ports  */
	if (sigev &&
	    (sigevk.sigev_notify == SIGEV_THREAD ||
	    sigevk.sigev_notify == SIGEV_PORT)) {
		if (sigevk.sigev_notify == SIGEV_THREAD) {
			pnotify.portnfy_port = sigevk.sigev_signo;
			pnotify.portnfy_user = sigevk.sigev_value.sival_ptr;
		} else if (copyin(sigevk.sigev_value.sival_ptr,
		    &pnotify, sizeof (pnotify))) {
			kmem_free(cbplist, ssize);
			return (EFAULT);
		}
		error = port_alloc_event(pnotify.portnfy_port,
		    PORT_ALLOC_DEFAULT, PORT_SOURCE_AIO, &pkevtp);
		if (error) {
			if (error == ENOMEM || error == EAGAIN)
				error = EAGAIN;
			else
				error = EINVAL;
			kmem_free(cbplist, ssize);
			return (error);
		}
		lio_head_port = pnotify.portnfy_port;
		portused = 1;
	}

	/*
	 * a list head should be allocated if notification is
	 * enabled for this list.
	 */
	head = NULL;

	if (mode_arg == LIO_WAIT || sigev) {
		mutex_enter(&aiop->aio_mutex);
		error = aio_lio_alloc(&head);
		mutex_exit(&aiop->aio_mutex);
		if (error)
			goto done;
		deadhead = 1;
		head->lio_nent = nent;
		head->lio_refcnt = nent;
		head->lio_port = -1;
		head->lio_portkev = NULL;
		if (sigev && sigevk.sigev_notify == SIGEV_SIGNAL &&
		    sigevk.sigev_signo > 0 && sigevk.sigev_signo < NSIG) {
			sqp = kmem_zalloc(sizeof (sigqueue_t), KM_NOSLEEP);
			if (sqp == NULL) {
				error = EAGAIN;
				goto done;
			}
			sqp->sq_func = NULL;
			sqp->sq_next = NULL;
			sqp->sq_info.si_code = SI_ASYNCIO;
			sqp->sq_info.si_pid = curproc->p_pid;
			sqp->sq_info.si_ctid = PRCTID(curproc);
			sqp->sq_info.si_zoneid = getzoneid();
			sqp->sq_info.si_uid = crgetuid(curproc->p_cred);
			sqp->sq_info.si_signo = sigevk.sigev_signo;
			sqp->sq_info.si_value = sigevk.sigev_value;
			head->lio_sigqp = sqp;
		} else {
			head->lio_sigqp = NULL;
		}
		if (pkevtp) {
			/*
			 * Prepare data to send when list of aiocb's
			 * has completed.
			 */
			port_init_event(pkevtp, (uintptr_t)sigev,
			    (void *)(uintptr_t)pnotify.portnfy_user,
			    NULL, head);
			pkevtp->portkev_events = AIOLIO;
			head->lio_portkev = pkevtp;
			head->lio_port = pnotify.portnfy_port;
		}
	}

	for (i = 0; i < nent; i++, ucbp++) {

		cbp = *ucbp;
		/* skip entry if it can't be copied. */
		if (cbp == NULL || copyin(cbp, aiocb, sizeof (*aiocb))) {
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			continue;
		}

		/* skip if opcode for aiocb is LIO_NOP */
		mode = aiocb->aio_lio_opcode;
		if (mode == LIO_NOP) {
			cbp = NULL;
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			continue;
		}

		/* increment file descriptor's ref count. */
		if ((fp = getf(aiocb->aio_fildes)) == NULL) {
			lio_set_uerror(&cbp->aio_resultp, EBADF);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		/*
		 * check the permission of the partition
		 */
		if ((fp->f_flag & mode) == 0) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, EBADF);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		/*
		 * common case where requests are to the same fd
		 * for the same r/w operation.
		 * for UFS, need to set EBADFD
		 */
		vp = fp->f_vnode;
		if (fp != prev_fp || mode != prev_mode) {
			aio_func = check_vp(vp, mode);
			if (aio_func == NULL) {
				prev_fp = NULL;
				releasef(aiocb->aio_fildes);
				lio_set_uerror(&cbp->aio_resultp, EBADFD);
				aio_notsupported++;
				if (head) {
					mutex_enter(&aiop->aio_mutex);
					head->lio_nent--;
					head->lio_refcnt--;
					mutex_exit(&aiop->aio_mutex);
				}
				continue;
			} else {
				prev_fp = fp;
				prev_mode = mode;
			}
		}

		error = aio_req_setup(&reqp, aiop, aiocb,
		    &cbp->aio_resultp, vp, 0);
		if (error) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, error);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		reqp->aio_req_lio = head;
		deadhead = 0;

		/*
		 * Set the errno field now before sending the request to
		 * the driver to avoid a race condition
		 */
		(void) suword32(&cbp->aio_resultp.aio_errno,
		    EINPROGRESS);

		reqp->aio_req_iocb.iocb = (caddr_t)cbp;

		event = (mode == LIO_READ)? AIOAREAD : AIOAWRITE;
		aio_port = (aiocb->aio_sigevent.sigev_notify == SIGEV_PORT);
		aio_thread = (aiocb->aio_sigevent.sigev_notify == SIGEV_THREAD);
		if (aio_port | aio_thread) {
			port_kevent_t *lpkevp;
			/*
			 * Prepare data to send with each aiocb completed.
			 */
			if (aio_port) {
				void *paddr =
				    aiocb->aio_sigevent.sigev_value.sival_ptr;
				if (copyin(paddr, &pnotify, sizeof (pnotify)))
					error = EFAULT;
			} else {	/* aio_thread */
				pnotify.portnfy_port =
				    aiocb->aio_sigevent.sigev_signo;
				pnotify.portnfy_user =
				    aiocb->aio_sigevent.sigev_value.sival_ptr;
			}
			if (error)
				/* EMPTY */;
			else if (pkevtp != NULL &&
			    pnotify.portnfy_port == lio_head_port)
				error = port_dup_event(pkevtp, &lpkevp,
				    PORT_ALLOC_DEFAULT);
			else
				error = port_alloc_event(pnotify.portnfy_port,
				    PORT_ALLOC_DEFAULT, PORT_SOURCE_AIO,
				    &lpkevp);
			if (error == 0) {
				port_init_event(lpkevp, (uintptr_t)cbp,
				    (void *)(uintptr_t)pnotify.portnfy_user,
				    aio_port_callback, reqp);
				lpkevp->portkev_events = event;
				reqp->aio_req_portkev = lpkevp;
				reqp->aio_req_port = pnotify.portnfy_port;
			}
		}

		/*
		 * send the request to driver.
		 */
		if (error == 0) {
			if (aiocb->aio_nbytes == 0) {
				clear_active_fd(aiocb->aio_fildes);
				aio_zerolen(reqp);
				continue;
			}
			error = (*aio_func)(vp, (aio_req_t *)&reqp->aio_req,
			    CRED());
		}

		/*
		 * the fd's ref count is not decremented until the IO has
		 * completed unless there was an error.
		 */
		if (error) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, error);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			if (error == ENOTSUP)
				aio_notsupported++;
			else
				aio_errors++;
			lio_set_error(reqp, portused);
		} else {
			clear_active_fd(aiocb->aio_fildes);
		}
	}

	if (aio_notsupported) {
		error = ENOTSUP;
	} else if (aio_errors) {
		/*
		 * return EIO if any request failed
		 */
		error = EIO;
	}

	if (mode_arg == LIO_WAIT) {
		mutex_enter(&aiop->aio_mutex);
		while (head->lio_refcnt > 0) {
			if (!cv_wait_sig(&head->lio_notify, &aiop->aio_mutex)) {
				mutex_exit(&aiop->aio_mutex);
				error = EINTR;
				goto done;
			}
		}
		mutex_exit(&aiop->aio_mutex);
		alio_cleanup(aiop, (aiocb_t **)cbplist, nent, AIO_64);
	}

done:
	kmem_free(cbplist, ssize);
	if (deadhead) {
		if (head->lio_sigqp)
			kmem_free(head->lio_sigqp, sizeof (sigqueue_t));
		if (head->lio_portkev)
			port_free_event(head->lio_portkev);
		kmem_free(head, sizeof (aio_lio_t));
	}
	return (error);
}

#endif /* _LP64 */

/*
 * Asynchronous list IO.
 * If list I/O is called with LIO_WAIT it can still return
 * before all the I/O's are completed if a signal is caught
 * or if the list include UFS I/O requests. If this happens,
 * libaio will call aliowait() to wait for the I/O's to
 * complete
 */
/*ARGSUSED*/
static int
aliowait(
	int	mode,
	void	*aiocb,
	int	nent,
	void	*sigev,
	int	run_mode)
{
	aio_lio_t	*head;
	aio_t		*aiop;
	caddr_t		cbplist;
	aiocb_t		*cbp, **ucbp;
#ifdef	_SYSCALL32_IMPL
	aiocb32_t	*cbp32;
	caddr32_t	*ucbp32;
	aiocb64_32_t	*cbp64;
#endif
	int		error = 0;
	int		i;
	size_t		ssize = 0;
	model_t		model = get_udatamodel();

	aiop = curproc->p_aio;
	if (aiop == NULL || nent <= 0 || nent > _AIO_LISTIO_MAX)
		return (EINVAL);

	if (model == DATAMODEL_NATIVE)
		ssize = (sizeof (aiocb_t *) * nent);
#ifdef	_SYSCALL32_IMPL
	else
		ssize = (sizeof (caddr32_t) * nent);
#endif  /* _SYSCALL32_IMPL */

	if (ssize == 0)
		return (EINVAL);

	cbplist = kmem_alloc(ssize, KM_SLEEP);

	if (model == DATAMODEL_NATIVE)
		ucbp = (aiocb_t **)cbplist;
#ifdef	_SYSCALL32_IMPL
	else
		ucbp32 = (caddr32_t *)cbplist;
#endif  /* _SYSCALL32_IMPL */

	if (copyin(aiocb, cbplist, ssize)) {
		error = EFAULT;
		goto done;
	}

	/*
	 * To find the list head, we go through the
	 * list of aiocb structs, find the request
	 * its for, then get the list head that reqp
	 * points to
	 */
	head = NULL;

	for (i = 0; i < nent; i++) {
		if (model == DATAMODEL_NATIVE) {
			/*
			 * Since we are only checking for a NULL pointer
			 * Following should work on both native data sizes
			 * as well as for largefile aiocb.
			 */
			if ((cbp = *ucbp++) == NULL)
				continue;
			if (run_mode != AIO_LARGEFILE)
				if (head = aio_list_get(&cbp->aio_resultp))
					break;
			else {
				/*
				 * This is a case when largefile call is
				 * made on 32 bit kernel.
				 * Treat each pointer as pointer to
				 * aiocb64_32
				 */
				if (head = aio_list_get((aio_result_t *)
				    &(((aiocb64_32_t *)cbp)->aio_resultp)))
					break;
			}
		}
#ifdef	_SYSCALL32_IMPL
		else {
			if (run_mode == AIO_LARGEFILE) {
				if ((cbp64 = (aiocb64_32_t *)
				    (uintptr_t)*ucbp32++) == NULL)
					continue;
				if (head = aio_list_get((aio_result_t *)
				    &cbp64->aio_resultp))
					break;
			} else if (run_mode == AIO_32) {
				if ((cbp32 = (aiocb32_t *)
				    (uintptr_t)*ucbp32++) == NULL)
					continue;
				if (head = aio_list_get((aio_result_t *)
				    &cbp32->aio_resultp))
					break;
			}
		}
#endif	/* _SYSCALL32_IMPL */
	}

	if (head == NULL) {
		error = EINVAL;
		goto done;
	}

	mutex_enter(&aiop->aio_mutex);
	while (head->lio_refcnt > 0) {
		if (!cv_wait_sig(&head->lio_notify, &aiop->aio_mutex)) {
			mutex_exit(&aiop->aio_mutex);
			error = EINTR;
			goto done;
		}
	}
	mutex_exit(&aiop->aio_mutex);
	alio_cleanup(aiop, (aiocb_t **)cbplist, nent, run_mode);
done:
	kmem_free(cbplist, ssize);
	return (error);
}

aio_lio_t *
aio_list_get(aio_result_t *resultp)
{
	aio_lio_t	*head = NULL;
	aio_t		*aiop;
	aio_req_t 	**bucket;
	aio_req_t 	*reqp;
	long		index;

	aiop = curproc->p_aio;
	if (aiop == NULL)
		return (NULL);

	if (resultp) {
		index = AIO_HASH(resultp);
		bucket = &aiop->aio_hash[index];
		for (reqp = *bucket; reqp != NULL;
		    reqp = reqp->aio_hash_next) {
			if (reqp->aio_req_resultp == resultp) {
				head = reqp->aio_req_lio;
				return (head);
			}
		}
	}
	return (NULL);
}


static void
lio_set_uerror(void *resultp, int error)
{
	/*
	 * the resultp field is a pointer to where the
	 * error should be written out to the user's
	 * aiocb.
	 *
	 */
	if (get_udatamodel() == DATAMODEL_NATIVE) {
		(void) sulword(&((aio_result_t *)resultp)->aio_return,
		    (ssize_t)-1);
		(void) suword32(&((aio_result_t *)resultp)->aio_errno, error);
	}
#ifdef	_SYSCALL32_IMPL
	else {
		(void) suword32(&((aio_result32_t *)resultp)->aio_return,
		    (uint_t)-1);
		(void) suword32(&((aio_result32_t *)resultp)->aio_errno, error);
	}
#endif  /* _SYSCALL32_IMPL */
}

/*
 * do cleanup completion for all requests in list. memory for
 * each request is also freed.
 */
static void
alio_cleanup(aio_t *aiop, aiocb_t **cbp, int nent, int run_mode)
{
	int i;
	aio_req_t *reqp;
	aio_result_t *resultp;
	aiocb64_32_t *aiocb_64;

	for (i = 0; i < nent; i++) {
		if (get_udatamodel() == DATAMODEL_NATIVE) {
			if (cbp[i] == NULL)
				continue;
			if (run_mode == AIO_LARGEFILE) {
				aiocb_64 = (aiocb64_32_t *)cbp[i];
				resultp = (aio_result_t *)
				    &aiocb_64->aio_resultp;
			} else
				resultp = &cbp[i]->aio_resultp;
		}
#ifdef	_SYSCALL32_IMPL
		else {
			aiocb32_t *aiocb_32;
			caddr32_t *cbp32;

			cbp32 = (caddr32_t *)cbp;
			if (cbp32[i] == (uintptr_t)NULL)
				continue;
			if (run_mode == AIO_32) {
				aiocb_32 = (aiocb32_t *)(uintptr_t)cbp32[i];
				resultp = (aio_result_t *)&aiocb_32->
				    aio_resultp;
			} else if (run_mode == AIO_LARGEFILE) {
				aiocb_64 = (aiocb64_32_t *)(uintptr_t)cbp32[i];
				resultp = (aio_result_t *)&aiocb_64->
				    aio_resultp;
			}
		}
#endif  /* _SYSCALL32_IMPL */
		/*
		 * we need to get the aio_cleanupq_mutex since we call
		 * aio_req_done().
		 */
		mutex_enter(&aiop->aio_cleanupq_mutex);
		mutex_enter(&aiop->aio_mutex);
		reqp = aio_req_done(resultp);
		mutex_exit(&aiop->aio_mutex);
		mutex_exit(&aiop->aio_cleanupq_mutex);
		if (reqp != NULL) {
			aphysio_unlock(reqp);
			aio_copyout_result(reqp);
			mutex_enter(&aiop->aio_mutex);
			aio_req_free(aiop, reqp);
			mutex_exit(&aiop->aio_mutex);
		}
	}
}

/*
 * Write out the results for an aio request that is done.
 */
static int
aioerror(void *cb, int run_mode)
{
	aio_result_t *resultp;
	aio_t *aiop;
	aio_req_t *reqp;
	int retval;

	aiop = curproc->p_aio;
	if (aiop == NULL || cb == NULL)
		return (EINVAL);

	if (get_udatamodel() == DATAMODEL_NATIVE) {
		if (run_mode == AIO_LARGEFILE)
			resultp = (aio_result_t *)&((aiocb64_32_t *)cb)->
			    aio_resultp;
		else
			resultp = &((aiocb_t *)cb)->aio_resultp;
	}
#ifdef	_SYSCALL32_IMPL
	else {
		if (run_mode == AIO_LARGEFILE)
			resultp = (aio_result_t *)&((aiocb64_32_t *)cb)->
			    aio_resultp;
		else if (run_mode == AIO_32)
			resultp = (aio_result_t *)&((aiocb32_t *)cb)->
			    aio_resultp;
	}
#endif  /* _SYSCALL32_IMPL */
	/*
	 * we need to get the aio_cleanupq_mutex since we call
	 * aio_req_find().
	 */
	mutex_enter(&aiop->aio_cleanupq_mutex);
	mutex_enter(&aiop->aio_mutex);
	retval = aio_req_find(resultp, &reqp);
	mutex_exit(&aiop->aio_mutex);
	mutex_exit(&aiop->aio_cleanupq_mutex);
	if (retval == 0) {
		aphysio_unlock(reqp);
		aio_copyout_result(reqp);
		mutex_enter(&aiop->aio_mutex);
		aio_req_free(aiop, reqp);
		mutex_exit(&aiop->aio_mutex);
		return (0);
	} else if (retval == 1)
		return (EINPROGRESS);
	else if (retval == 2)
		return (EINVAL);
	return (0);
}

/*
 * 	aio_cancel - if no requests outstanding,
 *			return AIO_ALLDONE
 *			else
 *			return AIO_NOTCANCELED
 */
static int
aio_cancel(
	int	fildes,
	void 	*cb,
	long	*rval,
	int	run_mode)
{
	aio_t *aiop;
	void *resultp;
	int index;
	aio_req_t **bucket;
	aio_req_t *ent;


	/*
	 * Verify valid file descriptor
	 */
	if ((getf(fildes)) == NULL) {
		return (EBADF);
	}
	releasef(fildes);

	aiop = curproc->p_aio;
	if (aiop == NULL)
		return (EINVAL);

	if (aiop->aio_outstanding == 0) {
		*rval = AIO_ALLDONE;
		return (0);
	}

	mutex_enter(&aiop->aio_mutex);
	if (cb != NULL) {
		if (get_udatamodel() == DATAMODEL_NATIVE) {
			if (run_mode == AIO_LARGEFILE)
				resultp = (aio_result_t *)&((aiocb64_32_t *)cb)
				    ->aio_resultp;
			else
				resultp = &((aiocb_t *)cb)->aio_resultp;
		}
#ifdef	_SYSCALL32_IMPL
		else {
			if (run_mode == AIO_LARGEFILE)
				resultp = (aio_result_t *)&((aiocb64_32_t *)cb)
				    ->aio_resultp;
			else if (run_mode == AIO_32)
				resultp = (aio_result_t *)&((aiocb32_t *)cb)
				    ->aio_resultp;
		}
#endif  /* _SYSCALL32_IMPL */
		index = AIO_HASH(resultp);
		bucket = &aiop->aio_hash[index];
		for (ent = *bucket; ent != NULL; ent = ent->aio_hash_next) {
			if (ent->aio_req_resultp == resultp) {
				if ((ent->aio_req_flags & AIO_PENDING) == 0) {
					mutex_exit(&aiop->aio_mutex);
					*rval = AIO_ALLDONE;
					return (0);
				}
				mutex_exit(&aiop->aio_mutex);
				*rval = AIO_NOTCANCELED;
				return (0);
			}
		}
		mutex_exit(&aiop->aio_mutex);
		*rval = AIO_ALLDONE;
		return (0);
	}

	for (index = 0; index < AIO_HASHSZ; index++) {
		bucket = &aiop->aio_hash[index];
		for (ent = *bucket; ent != NULL; ent = ent->aio_hash_next) {
			if (ent->aio_req_fd == fildes) {
				if ((ent->aio_req_flags & AIO_PENDING) != 0) {
					mutex_exit(&aiop->aio_mutex);
					*rval = AIO_NOTCANCELED;
					return (0);
				}
			}
		}
	}
	mutex_exit(&aiop->aio_mutex);
	*rval = AIO_ALLDONE;
	return (0);
}

/*
 * solaris version of asynchronous read and write
 */
static int
arw(
	int	opcode,
	int	fdes,
	char	*bufp,
	int	bufsize,
	offset_t	offset,
	aio_result_t	*resultp,
	int		mode)
{
	file_t		*fp;
	int		error;
	struct vnode	*vp;
	aio_req_t	*reqp;
	aio_t		*aiop;
	int		(*aio_func)();
#ifdef _LP64
	aiocb_t		aiocb;
#else
	aiocb64_32_t	aiocb64;
#endif

	aiop = curproc->p_aio;
	if (aiop == NULL)
		return (EINVAL);

	if ((fp = getf(fdes)) == NULL) {
		return (EBADF);
	}

	/*
	 * check the permission of the partition
	 */
	if ((fp->f_flag & mode) == 0) {
		releasef(fdes);
		return (EBADF);
	}

	vp = fp->f_vnode;
	aio_func = check_vp(vp, mode);
	if (aio_func == NULL) {
		releasef(fdes);
		return (EBADFD);
	}
#ifdef _LP64
	aiocb.aio_fildes = fdes;
	aiocb.aio_buf = bufp;
	aiocb.aio_nbytes = bufsize;
	aiocb.aio_offset = offset;
	aiocb.aio_sigevent.sigev_notify = 0;
	error = aio_req_setup(&reqp, aiop, &aiocb, resultp, vp, 1);
#else
	aiocb64.aio_fildes = fdes;
	aiocb64.aio_buf = (caddr32_t)bufp;
	aiocb64.aio_nbytes = bufsize;
	aiocb64.aio_offset = offset;
	aiocb64.aio_sigevent.sigev_notify = 0;
	error = aio_req_setupLF(&reqp, aiop, &aiocb64, resultp, vp, 1);
#endif
	if (error) {
		releasef(fdes);
		return (error);
	}

	/*
	 * enable polling on this request if the opcode has
	 * the AIO poll bit set
	 */
	if (opcode & AIO_POLL_BIT)
		reqp->aio_req_flags |= AIO_POLL;

	if (bufsize == 0) {
		clear_active_fd(fdes);
		aio_zerolen(reqp);
		return (0);
	}
	/*
	 * send the request to driver.
	 */
	error = (*aio_func)(vp, (aio_req_t *)&reqp->aio_req, CRED());
	/*
	 * the fd is stored in the aio_req_t by aio_req_setup(), and
	 * is released by the aio_cleanup_thread() when the IO has
	 * completed.
	 */
	if (error) {
		releasef(fdes);
		mutex_enter(&aiop->aio_mutex);
		aio_req_free(aiop, reqp);
		aiop->aio_pending--;
		if (aiop->aio_flags & AIO_REQ_BLOCK)
			cv_signal(&aiop->aio_cleanupcv);
		mutex_exit(&aiop->aio_mutex);
		return (error);
	}
	clear_active_fd(fdes);
	return (0);
}

/*
 * posix version of asynchronous read and write
 */
static int
aiorw(
	int		opcode,
	void		*aiocb_arg,
	int		mode,
	int		run_mode)
{
#ifdef _SYSCALL32_IMPL
	aiocb32_t	aiocb32;
	struct	sigevent32 *sigev32;
	port_notify32_t	pntfy32;
#endif
	aiocb64_32_t	aiocb64;
	aiocb_t		aiocb;
	file_t		*fp;
	int		error, fd;
	size_t		bufsize;
	struct vnode	*vp;
	aio_req_t	*reqp;
	aio_t		*aiop;
	int		(*aio_func)();
	aio_result_t	*resultp;
	struct	sigevent *sigev;
	model_t		model;
	int		aio_use_port = 0;
	port_notify_t	pntfy;

	model = get_udatamodel();
	aiop = curproc->p_aio;
	if (aiop == NULL)
		return (EINVAL);

	if (model == DATAMODEL_NATIVE) {
		if (run_mode != AIO_LARGEFILE) {
			if (copyin(aiocb_arg, &aiocb, sizeof (aiocb_t)))
				return (EFAULT);
			bufsize = aiocb.aio_nbytes;
			resultp = &(((aiocb_t *)aiocb_arg)->aio_resultp);
			if ((fp = getf(fd = aiocb.aio_fildes)) == NULL) {
				return (EBADF);
			}
			sigev = &aiocb.aio_sigevent;
		} else {
			/*
			 * We come here only when we make largefile
			 * call on 32 bit kernel using 32 bit library.
			 */
			if (copyin(aiocb_arg, &aiocb64, sizeof (aiocb64_32_t)))
				return (EFAULT);
			bufsize = aiocb64.aio_nbytes;
			resultp = (aio_result_t *)&(((aiocb64_32_t *)aiocb_arg)
			    ->aio_resultp);
			if ((fp = getf(fd = aiocb64.aio_fildes)) == NULL)
				return (EBADF);
			sigev = (struct sigevent *)&aiocb64.aio_sigevent;
		}

		if (sigev->sigev_notify == SIGEV_PORT) {
			if (copyin((void *)sigev->sigev_value.sival_ptr,
			    &pntfy, sizeof (port_notify_t))) {
				releasef(fd);
				return (EFAULT);
			}
			aio_use_port = 1;
		} else if (sigev->sigev_notify == SIGEV_THREAD) {
			pntfy.portnfy_port = aiocb.aio_sigevent.sigev_signo;
			pntfy.portnfy_user =
			    aiocb.aio_sigevent.sigev_value.sival_ptr;
			aio_use_port = 1;
		}
	}
#ifdef	_SYSCALL32_IMPL
	else {
		if (run_mode == AIO_32) {
			/* 32 bit system call is being made on 64 bit kernel */
			if (copyin(aiocb_arg, &aiocb32, sizeof (aiocb32_t)))
				return (EFAULT);

			bufsize = aiocb32.aio_nbytes;
			aiocb_32ton(&aiocb32, &aiocb);
			resultp = (aio_result_t *)&(((aiocb32_t *)aiocb_arg)->
			    aio_resultp);
			if ((fp = getf(fd = aiocb32.aio_fildes)) == NULL) {
				return (EBADF);
			}
			sigev32 = &aiocb32.aio_sigevent;
		} else if (run_mode == AIO_LARGEFILE) {
			/*
			 * We come here only when we make largefile
			 * call on 64 bit kernel using 32 bit library.
			 */
			if (copyin(aiocb_arg, &aiocb64, sizeof (aiocb64_32_t)))
				return (EFAULT);
			bufsize = aiocb64.aio_nbytes;
			aiocb_LFton(&aiocb64, &aiocb);
			resultp = (aio_result_t *)&(((aiocb64_32_t *)aiocb_arg)
			    ->aio_resultp);
			if ((fp = getf(fd = aiocb64.aio_fildes)) == NULL)
				return (EBADF);
			sigev32 = &aiocb64.aio_sigevent;
		}

		if (sigev32->sigev_notify == SIGEV_PORT) {
			if (copyin(
			    (void *)(uintptr_t)sigev32->sigev_value.sival_ptr,
			    &pntfy32, sizeof (port_notify32_t))) {
				releasef(fd);
				return (EFAULT);
			}
			pntfy.portnfy_port = pntfy32.portnfy_port;
			pntfy.portnfy_user = (void *)(uintptr_t)
			    pntfy32.portnfy_user;
			aio_use_port = 1;
		} else if (sigev32->sigev_notify == SIGEV_THREAD) {
			pntfy.portnfy_port = sigev32->sigev_signo;
			pntfy.portnfy_user = (void *)(uintptr_t)
			    sigev32->sigev_value.sival_ptr;
			aio_use_port = 1;
		}
	}
#endif  /* _SYSCALL32_IMPL */

	/*
	 * check the permission of the partition
	 */

	if ((fp->f_flag & mode) == 0) {
		releasef(fd);
		return (EBADF);
	}

	vp = fp->f_vnode;
	aio_func = check_vp(vp, mode);
	if (aio_func == NULL) {
		releasef(fd);
		return (EBADFD);
	}
	if (run_mode == AIO_LARGEFILE)
		error = aio_req_setupLF(&reqp, aiop, &aiocb64, resultp, vp, 0);
	else
		error = aio_req_setup(&reqp, aiop, &aiocb, resultp, vp, 0);

	if (error) {
		releasef(fd);
		return (error);
	}
	/*
	 * enable polling on this request if the opcode has
	 * the AIO poll bit set
	 */
	if (opcode & AIO_POLL_BIT)
		reqp->aio_req_flags |= AIO_POLL;

	if (model == DATAMODEL_NATIVE)
		reqp->aio_req_iocb.iocb = aiocb_arg;
#ifdef  _SYSCALL32_IMPL
	else
		reqp->aio_req_iocb.iocb32 = (caddr32_t)(uintptr_t)aiocb_arg;
#endif

	if (aio_use_port) {
		int event = (run_mode == AIO_LARGEFILE)?
		    ((mode == FREAD)? AIOAREAD64 : AIOAWRITE64) :
		    ((mode == FREAD)? AIOAREAD : AIOAWRITE);
		error = aio_req_assoc_port_rw(&pntfy, aiocb_arg, reqp, event);
	}

	/*
	 * send the request to driver.
	 */
	if (error == 0) {
		if (bufsize == 0) {
			clear_active_fd(fd);
			aio_zerolen(reqp);
			return (0);
		}
		error = (*aio_func)(vp, (aio_req_t *)&reqp->aio_req, CRED());
	}

	/*
	 * the fd is stored in the aio_req_t by aio_req_setup(), and
	 * is released by the aio_cleanup_thread() when the IO has
	 * completed.
	 */
	if (error) {
		releasef(fd);
		mutex_enter(&aiop->aio_mutex);
		if (aio_use_port)
			aio_deq(&aiop->aio_portpending, reqp);
		aio_req_free(aiop, reqp);
		aiop->aio_pending--;
		if (aiop->aio_flags & AIO_REQ_BLOCK)
			cv_signal(&aiop->aio_cleanupcv);
		mutex_exit(&aiop->aio_mutex);
		return (error);
	}
	clear_active_fd(fd);
	return (0);
}


/*
 * set error for a list IO entry that failed.
 */
static void
lio_set_error(aio_req_t *reqp, int portused)
{
	aio_t *aiop = curproc->p_aio;

	if (aiop == NULL)
		return;

	mutex_enter(&aiop->aio_mutex);
	if (portused)
		aio_deq(&aiop->aio_portpending, reqp);
	aiop->aio_pending--;
	/* request failed, AIO_PHYSIODONE set to aviod physio cleanup. */
	reqp->aio_req_flags |= AIO_PHYSIODONE;
	/*
	 * Need to free the request now as its never
	 * going to get on the done queue
	 *
	 * Note: aio_outstanding is decremented in
	 *	 aio_req_free()
	 */
	aio_req_free(aiop, reqp);
	if (aiop->aio_flags & AIO_REQ_BLOCK)
		cv_signal(&aiop->aio_cleanupcv);
	mutex_exit(&aiop->aio_mutex);
}

/*
 * check if a specified request is done, and remove it from
 * the done queue. otherwise remove anybody from the done queue
 * if NULL is specified.
 */
static aio_req_t *
aio_req_done(void *resultp)
{
	aio_req_t **bucket;
	aio_req_t *ent;
	aio_t *aiop = curproc->p_aio;
	long index;

	ASSERT(MUTEX_HELD(&aiop->aio_cleanupq_mutex));
	ASSERT(MUTEX_HELD(&aiop->aio_mutex));

	if (resultp) {
		index = AIO_HASH(resultp);
		bucket = &aiop->aio_hash[index];
		for (ent = *bucket; ent != NULL; ent = ent->aio_hash_next) {
			if (ent->aio_req_resultp == (aio_result_t *)resultp) {
				if (ent->aio_req_flags & AIO_DONEQ) {
					return (aio_req_remove(ent));
				}
				return (NULL);
			}
		}
		/* no match, resultp is invalid */
		return (NULL);
	}
	return (aio_req_remove(NULL));
}

/*
 * determine if a user-level resultp pointer is associated with an
 * active IO request. Zero is returned when the request is done,
 * and the request is removed from the done queue. Only when the
 * return value is zero, is the "reqp" pointer valid. One is returned
 * when the request is inprogress. Two is returned when the request
 * is invalid.
 */
static int
aio_req_find(aio_result_t *resultp, aio_req_t **reqp)
{
	aio_req_t **bucket;
	aio_req_t *ent;
	aio_t *aiop = curproc->p_aio;
	long index;

	ASSERT(MUTEX_HELD(&aiop->aio_cleanupq_mutex));
	ASSERT(MUTEX_HELD(&aiop->aio_mutex));

	index = AIO_HASH(resultp);
	bucket = &aiop->aio_hash[index];
	for (ent = *bucket; ent != NULL; ent = ent->aio_hash_next) {
		if (ent->aio_req_resultp == resultp) {
			if (ent->aio_req_flags & AIO_DONEQ) {
				*reqp = aio_req_remove(ent);
				return (0);
			}
			return (1);
		}
	}
	/* no match, resultp is invalid */
	return (2);
}

/*
 * remove a request from the done queue.
 */
static aio_req_t *
aio_req_remove(aio_req_t *reqp)
{
	aio_t *aiop = curproc->p_aio;

	ASSERT(MUTEX_HELD(&aiop->aio_mutex));

	if (reqp != NULL) {
		ASSERT(reqp->aio_req_flags & AIO_DONEQ);
		if (reqp->aio_req_next == reqp) {
			/* only one request on queue */
			if (reqp ==  aiop->aio_doneq) {
				aiop->aio_doneq = NULL;
			} else {
				ASSERT(reqp == aiop->aio_cleanupq);
				aiop->aio_cleanupq = NULL;
			}
		} else {
			reqp->aio_req_next->aio_req_prev = reqp->aio_req_prev;
			reqp->aio_req_prev->aio_req_next = reqp->aio_req_next;
			/*
			 * The request can be either on the aio_doneq or the
			 * aio_cleanupq
			 */
			if (reqp == aiop->aio_doneq)
				aiop->aio_doneq = reqp->aio_req_next;

			if (reqp == aiop->aio_cleanupq)
				aiop->aio_cleanupq = reqp->aio_req_next;
		}
		reqp->aio_req_flags &= ~AIO_DONEQ;
		reqp->aio_req_next = NULL;
		reqp->aio_req_prev = NULL;
	} else if ((reqp = aiop->aio_doneq) != NULL) {
		ASSERT(reqp->aio_req_flags & AIO_DONEQ);
		if (reqp == reqp->aio_req_next) {
			/* only one request on queue */
			aiop->aio_doneq = NULL;
		} else {
			reqp->aio_req_prev->aio_req_next = reqp->aio_req_next;
			reqp->aio_req_next->aio_req_prev = reqp->aio_req_prev;
			aiop->aio_doneq = reqp->aio_req_next;
		}
		reqp->aio_req_flags &= ~AIO_DONEQ;
		reqp->aio_req_next = NULL;
		reqp->aio_req_prev = NULL;
	}
	if (aiop->aio_doneq == NULL && (aiop->aio_flags & AIO_WAITN))
		cv_broadcast(&aiop->aio_waitcv);
	return (reqp);
}

static int
aio_req_setup(
	aio_req_t	**reqpp,
	aio_t 		*aiop,
	aiocb_t 	*arg,
	aio_result_t 	*resultp,
	vnode_t		*vp,
	int		old_solaris_req)
{
	sigqueue_t	*sqp = NULL;
	aio_req_t 	*reqp;
	struct uio 	*uio;
	struct sigevent *sigev;
	int		error;

	sigev = &arg->aio_sigevent;
	if (sigev->sigev_notify == SIGEV_SIGNAL &&
	    sigev->sigev_signo > 0 && sigev->sigev_signo < NSIG) {
		sqp = kmem_zalloc(sizeof (sigqueue_t), KM_NOSLEEP);
		if (sqp == NULL)
			return (EAGAIN);
		sqp->sq_func = NULL;
		sqp->sq_next = NULL;
		sqp->sq_info.si_code = SI_ASYNCIO;
		sqp->sq_info.si_pid = curproc->p_pid;
		sqp->sq_info.si_ctid = PRCTID(curproc);
		sqp->sq_info.si_zoneid = getzoneid();
		sqp->sq_info.si_uid = crgetuid(curproc->p_cred);
		sqp->sq_info.si_signo = sigev->sigev_signo;
		sqp->sq_info.si_value = sigev->sigev_value;
	}

	mutex_enter(&aiop->aio_mutex);

	if (aiop->aio_flags & AIO_REQ_BLOCK) {
		mutex_exit(&aiop->aio_mutex);
		if (sqp)
			kmem_free(sqp, sizeof (sigqueue_t));
		return (EIO);
	}
	/*
	 * get an aio_reqp from the free list or allocate one
	 * from dynamic memory.
	 */
	if (error = aio_req_alloc(&reqp, resultp)) {
		mutex_exit(&aiop->aio_mutex);
		if (sqp)
			kmem_free(sqp, sizeof (sigqueue_t));
		return (error);
	}
	aiop->aio_pending++;
	aiop->aio_outstanding++;
	reqp->aio_req_flags = AIO_PENDING;
	if (old_solaris_req) {
		/* this is an old solaris aio request */
		reqp->aio_req_flags |= AIO_SOLARIS;
		aiop->aio_flags |= AIO_SOLARIS_REQ;
	}
	if (sigev->sigev_notify == SIGEV_THREAD ||
	    sigev->sigev_notify == SIGEV_PORT)
		aio_enq(&aiop->aio_portpending, reqp, 0);
	mutex_exit(&aiop->aio_mutex);
	/*
	 * initialize aio request.
	 */
	reqp->aio_req_fd = arg->aio_fildes;
	reqp->aio_req_sigqp = sqp;
	reqp->aio_req_iocb.iocb = NULL;
	reqp->aio_req_lio = NULL;
	reqp->aio_req_buf.b_file = vp;
	uio = reqp->aio_req.aio_uio;
	uio->uio_iovcnt = 1;
	uio->uio_iov->iov_base = (caddr_t)arg->aio_buf;
	uio->uio_iov->iov_len = arg->aio_nbytes;
	uio->uio_loffset = arg->aio_offset;
	*reqpp = reqp;
	return (0);
}

/*
 * Allocate p_aio struct.
 */
static aio_t *
aio_aiop_alloc(void)
{
	aio_t	*aiop;

	ASSERT(MUTEX_HELD(&curproc->p_lock));

	aiop = kmem_zalloc(sizeof (struct aio), KM_NOSLEEP);
	if (aiop) {
		mutex_init(&aiop->aio_mutex, NULL, MUTEX_DEFAULT, NULL);
		mutex_init(&aiop->aio_cleanupq_mutex, NULL, MUTEX_DEFAULT,
		    NULL);
		mutex_init(&aiop->aio_portq_mutex, NULL, MUTEX_DEFAULT, NULL);
	}
	return (aiop);
}

/*
 * Allocate an aio_req struct.
 */
static int
aio_req_alloc(aio_req_t **nreqp, aio_result_t *resultp)
{
	aio_req_t *reqp;
	aio_t *aiop = curproc->p_aio;

	ASSERT(MUTEX_HELD(&aiop->aio_mutex));

	if ((reqp = aiop->aio_free) != NULL) {
		aiop->aio_free = reqp->aio_req_next;
		bzero(reqp, sizeof (*reqp));
	} else {
		/*
		 * Check whether memory is getting tight.
		 * This is a temporary mechanism to avoid memory
		 * exhaustion by a single process until we come up
		 * with a per process solution such as setrlimit().
		 */
		if (freemem < desfree)
			return (EAGAIN);
		reqp = kmem_zalloc(sizeof (struct aio_req_t), KM_NOSLEEP);
		if (reqp == NULL)
			return (EAGAIN);
	}
	reqp->aio_req.aio_uio = &reqp->aio_req_uio;
	reqp->aio_req.aio_uio->uio_iov = &reqp->aio_req_iov;
	reqp->aio_req.aio_private = reqp;
	reqp->aio_req_buf.b_offset = -1;
	reqp->aio_req_resultp = resultp;
	if (aio_hash_insert(reqp, aiop)) {
		reqp->aio_req_next = aiop->aio_free;
		aiop->aio_free = reqp;
		return (EBUSY);
	}
	*nreqp = reqp;
	return (0);
}

/*
 * Allocate an aio_lio_t struct.
 */
static int
aio_lio_alloc(aio_lio_t **head)
{
	aio_lio_t *liop;
	aio_t *aiop = curproc->p_aio;

	ASSERT(MUTEX_HELD(&aiop->aio_mutex));

	if ((liop = aiop->aio_lio_free) != NULL) {
		aiop->aio_lio_free = liop->lio_next;
	} else {
		/*
		 * Check whether memory is getting tight.
		 * This is a temporary mechanism to avoid memory
		 * exhaustion by a single process until we come up
		 * with a per process solution such as setrlimit().
		 */
		if (freemem < desfree)
			return (EAGAIN);

		liop = kmem_zalloc(sizeof (aio_lio_t), KM_NOSLEEP);
		if (liop == NULL)
			return (EAGAIN);
	}
	*head = liop;
	return (0);
}

/*
 * this is a special per-process thread that is only activated if
 * the process is unmapping a segment with outstanding aio. normally,
 * the process will have completed the aio before unmapping the
 * segment. If the process does unmap a segment with outstanding aio,
 * this special thread will guarentee that the locked pages due to
 * aphysio() are released, thereby permitting the segment to be
 * unmapped. In addition to this, the cleanup thread is woken up
 * during DR operations to release the locked pages.
 */

static int
aio_cleanup_thread(aio_t *aiop)
{
	proc_t *p = curproc;
	struct as *as = p->p_as;
	int poked = 0;
	kcondvar_t *cvp;
	int exit_flag = 0;
	int rqclnup = 0;

	sigfillset(&curthread->t_hold);
	sigdiffset(&curthread->t_hold, &cantmask);
	for (;;) {
		/*
		 * if a segment is being unmapped, and the current
		 * process's done queue is not empty, then every request
		 * on the doneq with locked resources should be forced
		 * to release their locks. By moving the doneq request
		 * to the cleanupq, aio_cleanup() will process the cleanupq,
		 * and place requests back onto the doneq. All requests
		 * processed by aio_cleanup() will have their physical
		 * resources unlocked.
		 */
		mutex_enter(&aiop->aio_mutex);
		if ((aiop->aio_flags & AIO_CLEANUP) == 0) {
			aiop->aio_flags |= AIO_CLEANUP;
			mutex_enter(&as->a_contents);
			if (aiop->aio_rqclnup) {
				aiop->aio_rqclnup = 0;
				rqclnup = 1;
			}
			mutex_exit(&as->a_contents);
			if (aiop->aio_doneq) {
				aio_req_t *doneqhead = aiop->aio_doneq;
				aiop->aio_doneq = NULL;
				aio_cleanupq_concat(aiop, doneqhead, AIO_DONEQ);
			}
		}
		mutex_exit(&aiop->aio_mutex);
		aio_cleanup(AIO_CLEANUP_THREAD);
		/*
		 * thread should block on the cleanupcv while
		 * AIO_CLEANUP is set.
		 */
		cvp = &aiop->aio_cleanupcv;
		mutex_enter(&aiop->aio_mutex);

		if (aiop->aio_pollq != NULL || aiop->aio_cleanupq != NULL ||
		    aiop->aio_notifyq != NULL ||
		    aiop->aio_portcleanupq != NULL) {
			mutex_exit(&aiop->aio_mutex);
			continue;
		}
		mutex_enter(&as->a_contents);

		/*
		 * AIO_CLEANUP determines when the cleanup thread
		 * should be active. This flag is set when
		 * the cleanup thread is awakened by as_unmap() or
		 * due to DR operations.
		 * The flag is cleared when the blocking as_unmap()
		 * that originally awakened us is allowed to
		 * complete. as_unmap() blocks when trying to
		 * unmap a segment that has SOFTLOCKed pages. when
		 * the segment's pages are all SOFTUNLOCKed,
		 * as->a_flags & AS_UNMAPWAIT should be zero.
		 *
		 * In case of cleanup request by DR, the flag is cleared
		 * once all the pending aio requests have been processed.
		 *
		 * The flag shouldn't be cleared right away if the
		 * cleanup thread was interrupted because the process
		 * is doing forkall(). This happens when cv_wait_sig()
		 * returns zero, because it was awakened by a pokelwps().
		 * If the process is not exiting, it must be doing forkall().
		 */
		if ((poked == 0) &&
		    ((!rqclnup && (AS_ISUNMAPWAIT(as) == 0)) ||
		    (aiop->aio_pending == 0))) {
			aiop->aio_flags &= ~(AIO_CLEANUP | AIO_CLEANUP_PORT);
			cvp = &as->a_cv;
			rqclnup = 0;
		}
		mutex_exit(&aiop->aio_mutex);
		if (poked) {
			/*
			 * If the process is exiting/killed, don't return
			 * immediately without waiting for pending I/O's
			 * and releasing the page locks.
			 */
			if (p->p_flag & (SEXITLWPS|SKILLED)) {
				/*
				 * If exit_flag is set, then it is
				 * safe to exit because we have released
				 * page locks of completed I/O's.
				 */
				if (exit_flag)
					break;

				mutex_exit(&as->a_contents);

				/*
				 * Wait for all the pending aio to complete.
				 */
				mutex_enter(&aiop->aio_mutex);
				aiop->aio_flags |= AIO_REQ_BLOCK;
				while (aiop->aio_pending != 0)
					cv_wait(&aiop->aio_cleanupcv,
					    &aiop->aio_mutex);
				mutex_exit(&aiop->aio_mutex);
				exit_flag = 1;
				continue;
			} else if (p->p_flag &
			    (SHOLDFORK|SHOLDFORK1|SHOLDWATCH)) {
				/*
				 * hold LWP until it
				 * is continued.
				 */
				mutex_exit(&as->a_contents);
				mutex_enter(&p->p_lock);
				stop(PR_SUSPENDED, SUSPEND_NORMAL);
				mutex_exit(&p->p_lock);
				poked = 0;
				continue;
			}
		} else {
			/*
			 * When started this thread will sleep on as->a_cv.
			 * as_unmap will awake this thread if the
			 * segment has SOFTLOCKed pages (poked = 0).
			 * 1. pokelwps() awakes this thread =>
			 *    break the loop to check SEXITLWPS, SHOLDFORK, etc
			 * 2. as_unmap awakes this thread =>
			 *    to break the loop it is necessary that
			 *    - AS_UNMAPWAIT is set (as_unmap is waiting for
			 *	memory to be unlocked)
			 *    - AIO_CLEANUP is not set
			 *	(if AIO_CLEANUP is set we have to wait for
			 *	pending requests. aio_done will send a signal
			 *	for every request which completes to continue
			 *	unmapping the corresponding address range)
			 * 3. A cleanup request will wake this thread up, ex.
			 *    by the DR operations. The aio_rqclnup flag will
			 *    be set.
			 */
			while (poked == 0) {
				/*
				 * The clean up requests that came in
				 * after we had just cleaned up, couldn't
				 * be causing the unmap thread to block - as
				 * unmap event happened first.
				 * Let aio_done() wake us up if it sees a need.
				 */
				if (aiop->aio_rqclnup &&
				    (aiop->aio_flags & AIO_CLEANUP) == 0)
					break;
				poked = !cv_wait_sig(cvp, &as->a_contents);
				if (AS_ISUNMAPWAIT(as) == 0)
					cv_signal(cvp);
				if (aiop->aio_outstanding != 0)
					break;
			}
		}
		mutex_exit(&as->a_contents);
	}
exit:
	mutex_exit(&as->a_contents);
	ASSERT((curproc->p_flag & (SEXITLWPS|SKILLED)));
	aston(curthread);	/* make thread do post_syscall */
	return (0);
}

/*
 * save a reference to a user's outstanding aio in a hash list.
 */
static int
aio_hash_insert(
	aio_req_t *aio_reqp,
	aio_t *aiop)
{
	long index;
	aio_result_t *resultp = aio_reqp->aio_req_resultp;
	aio_req_t *current;
	aio_req_t **nextp;

	index = AIO_HASH(resultp);
	nextp = &aiop->aio_hash[index];
	while ((current = *nextp) != NULL) {
		if (current->aio_req_resultp == resultp)
			return (DUPLICATE);
		nextp = &current->aio_hash_next;
	}
	*nextp = aio_reqp;
	aio_reqp->aio_hash_next = NULL;
	return (0);
}

static int
(*check_vp(struct vnode *vp, int mode))(vnode_t *, struct aio_req *,
    cred_t *)
{
	struct snode *sp;
	dev_t		dev;
	struct cb_ops  	*cb;
	major_t		major;
	int		(*aio_func)();

	dev = vp->v_rdev;
	major = getmajor(dev);

	/*
	 * return NULL for requests to files and STREAMs so
	 * that libaio takes care of them.
	 */
	if (vp->v_type == VCHR) {
		/* no stream device for kaio */
		if (STREAMSTAB(major)) {
			return (NULL);
		}
	} else {
		return (NULL);
	}

	/*
	 * Check old drivers which do not have async I/O entry points.
	 */
	if (devopsp[major]->devo_rev < 3)
		return (NULL);

	cb = devopsp[major]->devo_cb_ops;

	if (cb->cb_rev < 1)
		return (NULL);

	/*
	 * Check whether this device is a block device.
	 * Kaio is not supported for devices like tty.
	 */
	if (cb->cb_strategy == nodev || cb->cb_strategy == NULL)
		return (NULL);

	if (mode & FREAD)
		aio_func = (cb->cb_aread == nodev) ? NULL : driver_aio_read;
	else
		aio_func = (cb->cb_awrite == nodev) ? NULL : driver_aio_write;

	/*
	 * Do we need this ?
	 * nodev returns ENXIO anyway.
	 */
	if (aio_func == nodev)
		return (NULL);

	sp = VTOS(vp);
	smark(sp, SACC);
	return (aio_func);
}

/*
 * Clustering: We want check_vp to return a function prototyped
 * correctly that will be common to both PXFS and regular case.
 * We define this intermediate function that will do the right
 * thing for driver cases.
 */

static int
driver_aio_write(vnode_t *vp, struct aio_req *aio, cred_t *cred_p)
{
	dev_t dev;
	struct cb_ops  	*cb;

	ASSERT(vp->v_type == VCHR);
	dev = VTOS(vp)->s_dev;
	ASSERT(STREAMSTAB(getmajor(dev)) == NULL);

	cb = devopsp[getmajor(dev)]->devo_cb_ops;

	ASSERT(cb->cb_awrite != nodev);
	return ((*cb->cb_awrite)(dev, aio, cred_p));
}

/*
 * Clustering: We want check_vp to return a function prototyped
 * correctly that will be common to both PXFS and regular case.
 * We define this intermediate function that will do the right
 * thing for driver cases.
 */

static int
driver_aio_read(vnode_t *vp, struct aio_req *aio, cred_t *cred_p)
{
	dev_t dev;
	struct cb_ops  	*cb;

	ASSERT(vp->v_type == VCHR);
	dev = VTOS(vp)->s_dev;
	ASSERT(!STREAMSTAB(getmajor(dev)));

	cb = devopsp[getmajor(dev)]->devo_cb_ops;

	ASSERT(cb->cb_aread != nodev);
	return ((*cb->cb_aread)(dev, aio, cred_p));
}

/*
 * This routine is called when a largefile call is made by a 32bit
 * process on a ILP32 or LP64 kernel. All 64bit processes are large
 * file by definition and will call alio() instead.
 */
static int
alioLF(
	int		mode_arg,
	void		*aiocb_arg,
	int		nent,
	void		*sigev)
{
	file_t		*fp;
	file_t		*prev_fp = NULL;
	int		prev_mode = -1;
	struct vnode	*vp;
	aio_lio_t	*head;
	aio_req_t	*reqp;
	aio_t		*aiop;
	caddr_t		cbplist;
	aiocb64_32_t	cb64;
	aiocb64_32_t	*aiocb = &cb64;
	aiocb64_32_t	*cbp;
	caddr32_t	*ucbp;
#ifdef _LP64
	aiocb_t		aiocb_n;
#endif
	struct sigevent32	sigevk;
	sigqueue_t	*sqp;
	int		(*aio_func)();
	int		mode;
	int		error = 0;
	int		aio_errors = 0;
	int		i;
	size_t		ssize;
	int		deadhead = 0;
	int		aio_notsupported = 0;
	int		lio_head_port;
	int		aio_port;
	int		aio_thread;
	port_kevent_t	*pkevtp = NULL;
	int		portused = 0;
	port_notify32_t	pnotify;
	int		event;

	aiop = curproc->p_aio;
	if (aiop == NULL || nent <= 0 || nent > _AIO_LISTIO_MAX)
		return (EINVAL);

	ASSERT(get_udatamodel() == DATAMODEL_ILP32);

	ssize = (sizeof (caddr32_t) * nent);
	cbplist = kmem_alloc(ssize, KM_SLEEP);
	ucbp = (caddr32_t *)cbplist;

	if (copyin(aiocb_arg, cbplist, ssize) ||
	    (sigev && copyin(sigev, &sigevk, sizeof (sigevk)))) {
		kmem_free(cbplist, ssize);
		return (EFAULT);
	}

	/* Event Ports  */
	if (sigev &&
	    (sigevk.sigev_notify == SIGEV_THREAD ||
	    sigevk.sigev_notify == SIGEV_PORT)) {
		if (sigevk.sigev_notify == SIGEV_THREAD) {
			pnotify.portnfy_port = sigevk.sigev_signo;
			pnotify.portnfy_user = sigevk.sigev_value.sival_ptr;
		} else if (copyin(
		    (void *)(uintptr_t)sigevk.sigev_value.sival_ptr,
		    &pnotify, sizeof (pnotify))) {
			kmem_free(cbplist, ssize);
			return (EFAULT);
		}
		error = port_alloc_event(pnotify.portnfy_port,
		    PORT_ALLOC_DEFAULT, PORT_SOURCE_AIO, &pkevtp);
		if (error) {
			if (error == ENOMEM || error == EAGAIN)
				error = EAGAIN;
			else
				error = EINVAL;
			kmem_free(cbplist, ssize);
			return (error);
		}
		lio_head_port = pnotify.portnfy_port;
		portused = 1;
	}

	/*
	 * a list head should be allocated if notification is
	 * enabled for this list.
	 */
	head = NULL;

	if (mode_arg == LIO_WAIT || sigev) {
		mutex_enter(&aiop->aio_mutex);
		error = aio_lio_alloc(&head);
		mutex_exit(&aiop->aio_mutex);
		if (error)
			goto done;
		deadhead = 1;
		head->lio_nent = nent;
		head->lio_refcnt = nent;
		head->lio_port = -1;
		head->lio_portkev = NULL;
		if (sigev && sigevk.sigev_notify == SIGEV_SIGNAL &&
		    sigevk.sigev_signo > 0 && sigevk.sigev_signo < NSIG) {
			sqp = kmem_zalloc(sizeof (sigqueue_t), KM_NOSLEEP);
			if (sqp == NULL) {
				error = EAGAIN;
				goto done;
			}
			sqp->sq_func = NULL;
			sqp->sq_next = NULL;
			sqp->sq_info.si_code = SI_ASYNCIO;
			sqp->sq_info.si_pid = curproc->p_pid;
			sqp->sq_info.si_ctid = PRCTID(curproc);
			sqp->sq_info.si_zoneid = getzoneid();
			sqp->sq_info.si_uid = crgetuid(curproc->p_cred);
			sqp->sq_info.si_signo = sigevk.sigev_signo;
			sqp->sq_info.si_value.sival_int =
			    sigevk.sigev_value.sival_int;
			head->lio_sigqp = sqp;
		} else {
			head->lio_sigqp = NULL;
		}
		if (pkevtp) {
			/*
			 * Prepare data to send when list of aiocb's
			 * has completed.
			 */
			port_init_event(pkevtp, (uintptr_t)sigev,
			    (void *)(uintptr_t)pnotify.portnfy_user,
			    NULL, head);
			pkevtp->portkev_events = AIOLIO64;
			head->lio_portkev = pkevtp;
			head->lio_port = pnotify.portnfy_port;
		}
	}

	for (i = 0; i < nent; i++, ucbp++) {

		cbp = (aiocb64_32_t *)(uintptr_t)*ucbp;
		/* skip entry if it can't be copied. */
		if (cbp == NULL || copyin(cbp, aiocb, sizeof (*aiocb))) {
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			continue;
		}

		/* skip if opcode for aiocb is LIO_NOP */
		mode = aiocb->aio_lio_opcode;
		if (mode == LIO_NOP) {
			cbp = NULL;
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			continue;
		}

		/* increment file descriptor's ref count. */
		if ((fp = getf(aiocb->aio_fildes)) == NULL) {
			lio_set_uerror(&cbp->aio_resultp, EBADF);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		/*
		 * check the permission of the partition
		 */
		if ((fp->f_flag & mode) == 0) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, EBADF);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		/*
		 * common case where requests are to the same fd
		 * for the same r/w operation
		 * for UFS, need to set EBADFD
		 */
		vp = fp->f_vnode;
		if (fp != prev_fp || mode != prev_mode) {
			aio_func = check_vp(vp, mode);
			if (aio_func == NULL) {
				prev_fp = NULL;
				releasef(aiocb->aio_fildes);
				lio_set_uerror(&cbp->aio_resultp, EBADFD);
				aio_notsupported++;
				if (head) {
					mutex_enter(&aiop->aio_mutex);
					head->lio_nent--;
					head->lio_refcnt--;
					mutex_exit(&aiop->aio_mutex);
				}
				continue;
			} else {
				prev_fp = fp;
				prev_mode = mode;
			}
		}

#ifdef	_LP64
		aiocb_LFton(aiocb, &aiocb_n);
		error = aio_req_setup(&reqp, aiop, &aiocb_n,
		    (aio_result_t *)&cbp->aio_resultp, vp, 0);
#else
		error = aio_req_setupLF(&reqp, aiop, aiocb,
		    (aio_result_t *)&cbp->aio_resultp, vp, 0);
#endif  /* _LP64 */
		if (error) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, error);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		reqp->aio_req_lio = head;
		deadhead = 0;

		/*
		 * Set the errno field now before sending the request to
		 * the driver to avoid a race condition
		 */
		(void) suword32(&cbp->aio_resultp.aio_errno,
		    EINPROGRESS);

		reqp->aio_req_iocb.iocb32 = *ucbp;

		event = (mode == LIO_READ)? AIOAREAD64 : AIOAWRITE64;
		aio_port = (aiocb->aio_sigevent.sigev_notify == SIGEV_PORT);
		aio_thread = (aiocb->aio_sigevent.sigev_notify == SIGEV_THREAD);
		if (aio_port | aio_thread) {
			port_kevent_t *lpkevp;
			/*
			 * Prepare data to send with each aiocb completed.
			 */
			if (aio_port) {
				void *paddr = (void *)(uintptr_t)
				    aiocb->aio_sigevent.sigev_value.sival_ptr;
				if (copyin(paddr, &pnotify, sizeof (pnotify)))
					error = EFAULT;
			} else {	/* aio_thread */
				pnotify.portnfy_port =
				    aiocb->aio_sigevent.sigev_signo;
				pnotify.portnfy_user =
				    aiocb->aio_sigevent.sigev_value.sival_ptr;
			}
			if (error)
				/* EMPTY */;
			else if (pkevtp != NULL &&
			    pnotify.portnfy_port == lio_head_port)
				error = port_dup_event(pkevtp, &lpkevp,
				    PORT_ALLOC_DEFAULT);
			else
				error = port_alloc_event(pnotify.portnfy_port,
				    PORT_ALLOC_DEFAULT, PORT_SOURCE_AIO,
				    &lpkevp);
			if (error == 0) {
				port_init_event(lpkevp, (uintptr_t)*ucbp,
				    (void *)(uintptr_t)pnotify.portnfy_user,
				    aio_port_callback, reqp);
				lpkevp->portkev_events = event;
				reqp->aio_req_portkev = lpkevp;
				reqp->aio_req_port = pnotify.portnfy_port;
			}
		}

		/*
		 * send the request to driver.
		 */
		if (error == 0) {
			if (aiocb->aio_nbytes == 0) {
				clear_active_fd(aiocb->aio_fildes);
				aio_zerolen(reqp);
				continue;
			}
			error = (*aio_func)(vp, (aio_req_t *)&reqp->aio_req,
			    CRED());
		}

		/*
		 * the fd's ref count is not decremented until the IO has
		 * completed unless there was an error.
		 */
		if (error) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, error);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			if (error == ENOTSUP)
				aio_notsupported++;
			else
				aio_errors++;
			lio_set_error(reqp, portused);
		} else {
			clear_active_fd(aiocb->aio_fildes);
		}
	}

	if (aio_notsupported) {
		error = ENOTSUP;
	} else if (aio_errors) {
		/*
		 * return EIO if any request failed
		 */
		error = EIO;
	}

	if (mode_arg == LIO_WAIT) {
		mutex_enter(&aiop->aio_mutex);
		while (head->lio_refcnt > 0) {
			if (!cv_wait_sig(&head->lio_notify, &aiop->aio_mutex)) {
				mutex_exit(&aiop->aio_mutex);
				error = EINTR;
				goto done;
			}
		}
		mutex_exit(&aiop->aio_mutex);
		alio_cleanup(aiop, (aiocb_t **)cbplist, nent, AIO_LARGEFILE);
	}

done:
	kmem_free(cbplist, ssize);
	if (deadhead) {
		if (head->lio_sigqp)
			kmem_free(head->lio_sigqp, sizeof (sigqueue_t));
		if (head->lio_portkev)
			port_free_event(head->lio_portkev);
		kmem_free(head, sizeof (aio_lio_t));
	}
	return (error);
}

#ifdef  _SYSCALL32_IMPL
static void
aiocb_LFton(aiocb64_32_t *src, aiocb_t *dest)
{
	dest->aio_fildes = src->aio_fildes;
	dest->aio_buf = (void *)(uintptr_t)src->aio_buf;
	dest->aio_nbytes = (size_t)src->aio_nbytes;
	dest->aio_offset = (off_t)src->aio_offset;
	dest->aio_reqprio = src->aio_reqprio;
	dest->aio_sigevent.sigev_notify = src->aio_sigevent.sigev_notify;
	dest->aio_sigevent.sigev_signo = src->aio_sigevent.sigev_signo;

	/*
	 * See comment in sigqueue32() on handling of 32-bit
	 * sigvals in a 64-bit kernel.
	 */
	dest->aio_sigevent.sigev_value.sival_int =
	    (int)src->aio_sigevent.sigev_value.sival_int;
	dest->aio_sigevent.sigev_notify_function = (void (*)(union sigval))
	    (uintptr_t)src->aio_sigevent.sigev_notify_function;
	dest->aio_sigevent.sigev_notify_attributes = (pthread_attr_t *)
	    (uintptr_t)src->aio_sigevent.sigev_notify_attributes;
	dest->aio_sigevent.__sigev_pad2 = src->aio_sigevent.__sigev_pad2;
	dest->aio_lio_opcode = src->aio_lio_opcode;
	dest->aio_state = src->aio_state;
	dest->aio__pad[0] = src->aio__pad[0];
}
#endif

/*
 * This function is used only for largefile calls made by
 * 32 bit applications.
 */
static int
aio_req_setupLF(
	aio_req_t	**reqpp,
	aio_t		*aiop,
	aiocb64_32_t	*arg,
	aio_result_t	*resultp,
	vnode_t		*vp,
	int		old_solaris_req)
{
	sigqueue_t	*sqp = NULL;
	aio_req_t	*reqp;
	struct uio	*uio;
	struct sigevent32 *sigev;
	int 		error;

	sigev = &arg->aio_sigevent;
	if (sigev->sigev_notify == SIGEV_SIGNAL &&
	    sigev->sigev_signo > 0 && sigev->sigev_signo < NSIG) {
		sqp = kmem_zalloc(sizeof (sigqueue_t), KM_NOSLEEP);
		if (sqp == NULL)
			return (EAGAIN);
		sqp->sq_func = NULL;
		sqp->sq_next = NULL;
		sqp->sq_info.si_code = SI_ASYNCIO;
		sqp->sq_info.si_pid = curproc->p_pid;
		sqp->sq_info.si_ctid = PRCTID(curproc);
		sqp->sq_info.si_zoneid = getzoneid();
		sqp->sq_info.si_uid = crgetuid(curproc->p_cred);
		sqp->sq_info.si_signo = sigev->sigev_signo;
		sqp->sq_info.si_value.sival_int = sigev->sigev_value.sival_int;
	}

	mutex_enter(&aiop->aio_mutex);

	if (aiop->aio_flags & AIO_REQ_BLOCK) {
		mutex_exit(&aiop->aio_mutex);
		if (sqp)
			kmem_free(sqp, sizeof (sigqueue_t));
		return (EIO);
	}
	/*
	 * get an aio_reqp from the free list or allocate one
	 * from dynamic memory.
	 */
	if (error = aio_req_alloc(&reqp, resultp)) {
		mutex_exit(&aiop->aio_mutex);
		if (sqp)
			kmem_free(sqp, sizeof (sigqueue_t));
		return (error);
	}
	aiop->aio_pending++;
	aiop->aio_outstanding++;
	reqp->aio_req_flags = AIO_PENDING;
	if (old_solaris_req) {
		/* this is an old solaris aio request */
		reqp->aio_req_flags |= AIO_SOLARIS;
		aiop->aio_flags |= AIO_SOLARIS_REQ;
	}
	if (sigev->sigev_notify == SIGEV_THREAD ||
	    sigev->sigev_notify == SIGEV_PORT)
		aio_enq(&aiop->aio_portpending, reqp, 0);
	mutex_exit(&aiop->aio_mutex);
	/*
	 * initialize aio request.
	 */
	reqp->aio_req_fd = arg->aio_fildes;
	reqp->aio_req_sigqp = sqp;
	reqp->aio_req_iocb.iocb = NULL;
	reqp->aio_req_lio = NULL;
	reqp->aio_req_buf.b_file = vp;
	uio = reqp->aio_req.aio_uio;
	uio->uio_iovcnt = 1;
	uio->uio_iov->iov_base = (caddr_t)(uintptr_t)arg->aio_buf;
	uio->uio_iov->iov_len = arg->aio_nbytes;
	uio->uio_loffset = arg->aio_offset;
	*reqpp = reqp;
	return (0);
}

/*
 * This routine is called when a non largefile call is made by a 32bit
 * process on a ILP32 or LP64 kernel.
 */
static int
alio32(
	int		mode_arg,
	void		*aiocb_arg,
	int		nent,
	void		*sigev)
{
	file_t		*fp;
	file_t		*prev_fp = NULL;
	int		prev_mode = -1;
	struct vnode	*vp;
	aio_lio_t	*head;
	aio_req_t	*reqp;
	aio_t		*aiop;
	caddr_t		cbplist;
	aiocb_t		cb;
	aiocb_t		*aiocb = &cb;
#ifdef	_LP64
	aiocb32_t	*cbp;
	caddr32_t	*ucbp;
	aiocb32_t	cb32;
	aiocb32_t	*aiocb32 = &cb32;
	struct sigevent32	sigevk;
#else
	aiocb_t		*cbp, **ucbp;
	struct sigevent	sigevk;
#endif
	sigqueue_t	*sqp;
	int		(*aio_func)();
	int		mode;
	int		error = 0;
	int		aio_errors = 0;
	int		i;
	size_t		ssize;
	int		deadhead = 0;
	int		aio_notsupported = 0;
	int		lio_head_port;
	int		aio_port;
	int		aio_thread;
	port_kevent_t	*pkevtp = NULL;
	int		portused = 0;
#ifdef	_LP64
	port_notify32_t	pnotify;
#else
	port_notify_t	pnotify;
#endif
	int		event;

	aiop = curproc->p_aio;
	if (aiop == NULL || nent <= 0 || nent > _AIO_LISTIO_MAX)
		return (EINVAL);

#ifdef	_LP64
	ssize = (sizeof (caddr32_t) * nent);
#else
	ssize = (sizeof (aiocb_t *) * nent);
#endif
	cbplist = kmem_alloc(ssize, KM_SLEEP);
	ucbp = (void *)cbplist;

	if (copyin(aiocb_arg, cbplist, ssize) ||
	    (sigev && copyin(sigev, &sigevk, sizeof (struct sigevent32)))) {
		kmem_free(cbplist, ssize);
		return (EFAULT);
	}

	/* Event Ports  */
	if (sigev &&
	    (sigevk.sigev_notify == SIGEV_THREAD ||
	    sigevk.sigev_notify == SIGEV_PORT)) {
		if (sigevk.sigev_notify == SIGEV_THREAD) {
			pnotify.portnfy_port = sigevk.sigev_signo;
			pnotify.portnfy_user = sigevk.sigev_value.sival_ptr;
		} else if (copyin(
		    (void *)(uintptr_t)sigevk.sigev_value.sival_ptr,
		    &pnotify, sizeof (pnotify))) {
			kmem_free(cbplist, ssize);
			return (EFAULT);
		}
		error = port_alloc_event(pnotify.portnfy_port,
		    PORT_ALLOC_DEFAULT, PORT_SOURCE_AIO, &pkevtp);
		if (error) {
			if (error == ENOMEM || error == EAGAIN)
				error = EAGAIN;
			else
				error = EINVAL;
			kmem_free(cbplist, ssize);
			return (error);
		}
		lio_head_port = pnotify.portnfy_port;
		portused = 1;
	}

	/*
	 * a list head should be allocated if notification is
	 * enabled for this list.
	 */
	head = NULL;

	if (mode_arg == LIO_WAIT || sigev) {
		mutex_enter(&aiop->aio_mutex);
		error = aio_lio_alloc(&head);
		mutex_exit(&aiop->aio_mutex);
		if (error)
			goto done;
		deadhead = 1;
		head->lio_nent = nent;
		head->lio_refcnt = nent;
		head->lio_port = -1;
		head->lio_portkev = NULL;
		if (sigev && sigevk.sigev_notify == SIGEV_SIGNAL &&
		    sigevk.sigev_signo > 0 && sigevk.sigev_signo < NSIG) {
			sqp = kmem_zalloc(sizeof (sigqueue_t), KM_NOSLEEP);
			if (sqp == NULL) {
				error = EAGAIN;
				goto done;
			}
			sqp->sq_func = NULL;
			sqp->sq_next = NULL;
			sqp->sq_info.si_code = SI_ASYNCIO;
			sqp->sq_info.si_pid = curproc->p_pid;
			sqp->sq_info.si_ctid = PRCTID(curproc);
			sqp->sq_info.si_zoneid = getzoneid();
			sqp->sq_info.si_uid = crgetuid(curproc->p_cred);
			sqp->sq_info.si_signo = sigevk.sigev_signo;
			sqp->sq_info.si_value.sival_int =
			    sigevk.sigev_value.sival_int;
			head->lio_sigqp = sqp;
		} else {
			head->lio_sigqp = NULL;
		}
		if (pkevtp) {
			/*
			 * Prepare data to send when list of aiocb's has
			 * completed.
			 */
			port_init_event(pkevtp, (uintptr_t)sigev,
			    (void *)(uintptr_t)pnotify.portnfy_user,
			    NULL, head);
			pkevtp->portkev_events = AIOLIO;
			head->lio_portkev = pkevtp;
			head->lio_port = pnotify.portnfy_port;
		}
	}

	for (i = 0; i < nent; i++, ucbp++) {

		/* skip entry if it can't be copied. */
#ifdef	_LP64
		cbp = (aiocb32_t *)(uintptr_t)*ucbp;
		if (cbp == NULL || copyin(cbp, aiocb32, sizeof (*aiocb32)))
#else
		cbp = (aiocb_t *)*ucbp;
		if (cbp == NULL || copyin(cbp, aiocb, sizeof (*aiocb)))
#endif
		{
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			continue;
		}
#ifdef	_LP64
		/*
		 * copy 32 bit structure into 64 bit structure
		 */
		aiocb_32ton(aiocb32, aiocb);
#endif /* _LP64 */

		/* skip if opcode for aiocb is LIO_NOP */
		mode = aiocb->aio_lio_opcode;
		if (mode == LIO_NOP) {
			cbp = NULL;
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			continue;
		}

		/* increment file descriptor's ref count. */
		if ((fp = getf(aiocb->aio_fildes)) == NULL) {
			lio_set_uerror(&cbp->aio_resultp, EBADF);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		/*
		 * check the permission of the partition
		 */
		if ((fp->f_flag & mode) == 0) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, EBADF);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		/*
		 * common case where requests are to the same fd
		 * for the same r/w operation
		 * for UFS, need to set EBADFD
		 */
		vp = fp->f_vnode;
		if (fp != prev_fp || mode != prev_mode) {
			aio_func = check_vp(vp, mode);
			if (aio_func == NULL) {
				prev_fp = NULL;
				releasef(aiocb->aio_fildes);
				lio_set_uerror(&cbp->aio_resultp, EBADFD);
				aio_notsupported++;
				if (head) {
					mutex_enter(&aiop->aio_mutex);
					head->lio_nent--;
					head->lio_refcnt--;
					mutex_exit(&aiop->aio_mutex);
				}
				continue;
			} else {
				prev_fp = fp;
				prev_mode = mode;
			}
		}

		error = aio_req_setup(&reqp, aiop, aiocb,
		    (aio_result_t *)&cbp->aio_resultp, vp, 0);
		if (error) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, error);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			aio_errors++;
			continue;
		}

		reqp->aio_req_lio = head;
		deadhead = 0;

		/*
		 * Set the errno field now before sending the request to
		 * the driver to avoid a race condition
		 */
		(void) suword32(&cbp->aio_resultp.aio_errno,
		    EINPROGRESS);

		reqp->aio_req_iocb.iocb32 = (caddr32_t)(uintptr_t)cbp;

		event = (mode == LIO_READ)? AIOAREAD : AIOAWRITE;
		aio_port = (aiocb->aio_sigevent.sigev_notify == SIGEV_PORT);
		aio_thread = (aiocb->aio_sigevent.sigev_notify == SIGEV_THREAD);
		if (aio_port | aio_thread) {
			port_kevent_t *lpkevp;
			/*
			 * Prepare data to send with each aiocb completed.
			 */
#ifdef _LP64
			if (aio_port) {
				void *paddr = (void  *)(uintptr_t)
				    aiocb32->aio_sigevent.sigev_value.sival_ptr;
				if (copyin(paddr, &pnotify, sizeof (pnotify)))
					error = EFAULT;
			} else {	/* aio_thread */
				pnotify.portnfy_port =
				    aiocb32->aio_sigevent.sigev_signo;
				pnotify.portnfy_user =
				    aiocb32->aio_sigevent.sigev_value.sival_ptr;
			}
#else
			if (aio_port) {
				void *paddr =
				    aiocb->aio_sigevent.sigev_value.sival_ptr;
				if (copyin(paddr, &pnotify, sizeof (pnotify)))
					error = EFAULT;
			} else {	/* aio_thread */
				pnotify.portnfy_port =
				    aiocb->aio_sigevent.sigev_signo;
				pnotify.portnfy_user =
				    aiocb->aio_sigevent.sigev_value.sival_ptr;
			}
#endif
			if (error)
				/* EMPTY */;
			else if (pkevtp != NULL &&
			    pnotify.portnfy_port == lio_head_port)
				error = port_dup_event(pkevtp, &lpkevp,
				    PORT_ALLOC_DEFAULT);
			else
				error = port_alloc_event(pnotify.portnfy_port,
				    PORT_ALLOC_DEFAULT, PORT_SOURCE_AIO,
				    &lpkevp);
			if (error == 0) {
				port_init_event(lpkevp, (uintptr_t)cbp,
				    (void *)(uintptr_t)pnotify.portnfy_user,
				    aio_port_callback, reqp);
				lpkevp->portkev_events = event;
				reqp->aio_req_portkev = lpkevp;
				reqp->aio_req_port = pnotify.portnfy_port;
			}
		}

		/*
		 * send the request to driver.
		 */
		if (error == 0) {
			if (aiocb->aio_nbytes == 0) {
				clear_active_fd(aiocb->aio_fildes);
				aio_zerolen(reqp);
				continue;
			}
			error = (*aio_func)(vp, (aio_req_t *)&reqp->aio_req,
			    CRED());
		}

		/*
		 * the fd's ref count is not decremented until the IO has
		 * completed unless there was an error.
		 */
		if (error) {
			releasef(aiocb->aio_fildes);
			lio_set_uerror(&cbp->aio_resultp, error);
			if (head) {
				mutex_enter(&aiop->aio_mutex);
				head->lio_nent--;
				head->lio_refcnt--;
				mutex_exit(&aiop->aio_mutex);
			}
			if (error == ENOTSUP)
				aio_notsupported++;
			else
				aio_errors++;
			lio_set_error(reqp, portused);
		} else {
			clear_active_fd(aiocb->aio_fildes);
		}
	}

	if (aio_notsupported) {
		error = ENOTSUP;
	} else if (aio_errors) {
		/*
		 * return EIO if any request failed
		 */
		error = EIO;
	}

	if (mode_arg == LIO_WAIT) {
		mutex_enter(&aiop->aio_mutex);
		while (head->lio_refcnt > 0) {
			if (!cv_wait_sig(&head->lio_notify, &aiop->aio_mutex)) {
				mutex_exit(&aiop->aio_mutex);
				error = EINTR;
				goto done;
			}
		}
		mutex_exit(&aiop->aio_mutex);
		alio_cleanup(aiop, (aiocb_t **)cbplist, nent, AIO_32);
	}

done:
	kmem_free(cbplist, ssize);
	if (deadhead) {
		if (head->lio_sigqp)
			kmem_free(head->lio_sigqp, sizeof (sigqueue_t));
		if (head->lio_portkev)
			port_free_event(head->lio_portkev);
		kmem_free(head, sizeof (aio_lio_t));
	}
	return (error);
}


#ifdef  _SYSCALL32_IMPL
void
aiocb_32ton(aiocb32_t *src, aiocb_t *dest)
{
	dest->aio_fildes = src->aio_fildes;
	dest->aio_buf = (caddr_t)(uintptr_t)src->aio_buf;
	dest->aio_nbytes = (size_t)src->aio_nbytes;
	dest->aio_offset = (off_t)src->aio_offset;
	dest->aio_reqprio = src->aio_reqprio;
	dest->aio_sigevent.sigev_notify = src->aio_sigevent.sigev_notify;
	dest->aio_sigevent.sigev_signo = src->aio_sigevent.sigev_signo;

	/*
	 * See comment in sigqueue32() on handling of 32-bit
	 * sigvals in a 64-bit kernel.
	 */
	dest->aio_sigevent.sigev_value.sival_int =
	    (int)src->aio_sigevent.sigev_value.sival_int;
	dest->aio_sigevent.sigev_notify_function = (void (*)(union sigval))
	    (uintptr_t)src->aio_sigevent.sigev_notify_function;
	dest->aio_sigevent.sigev_notify_attributes = (pthread_attr_t *)
	    (uintptr_t)src->aio_sigevent.sigev_notify_attributes;
	dest->aio_sigevent.__sigev_pad2 = src->aio_sigevent.__sigev_pad2;
	dest->aio_lio_opcode = src->aio_lio_opcode;
	dest->aio_state = src->aio_state;
	dest->aio__pad[0] = src->aio__pad[0];
}
#endif /* _SYSCALL32_IMPL */

/*
 * aio_port_callback() is called just before the event is retrieved from the
 * port. The task of this callback function is to finish the work of the
 * transaction for the application, it means :
 * - copyout transaction data to the application
 *	(this thread is running in the right process context)
 * - keep trace of the transaction (update of counters).
 * - free allocated buffers
 * The aiocb pointer is the object element of the port_kevent_t structure.
 *
 * flag :
 *	PORT_CALLBACK_DEFAULT : do copyout and free resources
 *	PORT_CALLBACK_CLOSE   : don't do copyout, free resources
 */

/*ARGSUSED*/
int
aio_port_callback(void *arg, int *events, pid_t pid, int flag, void *evp)
{
	aio_t		*aiop = curproc->p_aio;
	aio_req_t	*reqp = arg;
	struct	iovec	*iov;
	struct	buf	*bp;
	void		*resultp;

	if (pid != curproc->p_pid) {
		/* wrong proc !!, can not deliver data here ... */
		return (EACCES);
	}

	mutex_enter(&aiop->aio_portq_mutex);
	reqp->aio_req_portkev = NULL;
	aio_req_remove_portq(aiop, reqp); /* remove request from portq */
	mutex_exit(&aiop->aio_portq_mutex);
	aphysio_unlock(reqp);		/* unlock used pages */
	mutex_enter(&aiop->aio_mutex);
	if (reqp->aio_req_flags & AIO_COPYOUTDONE) {
		aio_req_free_port(aiop, reqp);	/* back to free list */
		mutex_exit(&aiop->aio_mutex);
		return (0);
	}

	iov = reqp->aio_req_uio.uio_iov;
	bp = &reqp->aio_req_buf;
	resultp = (void *)reqp->aio_req_resultp;
	if (flag == PORT_CALLBACK_DEFAULT)
		aio_copyout_result_port(iov, bp, resultp);
	aio_req_free_port(aiop, reqp);	/* request struct back to free list */
	mutex_exit(&aiop->aio_mutex);
	return (0);
}