if.c

/*-
 * Copyright (c) 1980, 1986, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)if.c	8.5 (Berkeley) 1/9/95
 * $FreeBSD$
 */

#include <stddef.h>
#include <porting/netport.h>
#include <porting/sync_stub.h>
#include <osv/mutex.h>

#include <sys/param.h>
#include <sys/types.h>
#include <sys/sbuf.h>
#include <sys/mbuf.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/protosw.h>
#include <sys/refcount.h>
#include <sys/sockio.h>
#include <sys/syslog.h>
#include <sys/sysctl.h>
#include <sys/domain.h>
#include <sys/priv.h>


#include <net/if.h>
#include <net/if_arp.h>
#include <net/if_clone.h>
#include <net/if_dl.h>
#include <net/if_types.h>
#include <net/if_var.h>
#include <net/radix.h>
#include <net/route.h>
#include <net/vnet.h>

#if defined(INET) || defined(INET6)
/*XXX*/
#include <netinet/in.h>
#include <netinet/in_var.h>
#ifdef INET6
#include <netinet6/in6_var.h>
#include <netinet6/in6_ifattach.h>
#endif
#endif
#ifdef INET
#include <netinet/if_ether.h>
#endif

#ifdef COMPAT_FREEBSD32
#include <sys/mount.h>
#include <compat/freebsd32/freebsd32.h>
#endif

struct ifindex_entry {
	struct  ifnet *ife_ifnet;
};

#if 0
SYSCTL_NODE(_net, PF_LINK, link, CTLFLAG_RW, 0, "Link layers");
SYSCTL_NODE(_net_link, 0, generic, CTLFLAG_RW, 0, "Generic link-management");

TUNABLE_INT("net.link.ifqmaxlen", &ifqmaxlen);
SYSCTL_INT(_net_link, OID_AUTO, ifqmaxlen, CTLFLAG_RDTUN,
    &ifqmaxlen, 0, "max send queue size");
#endif

/* Log link state change events */
static int log_link_state_change = 1;

#if 0
SYSCTL_INT(_net_link, OID_AUTO, log_link_state_change, CTLFLAG_RW,
	&log_link_state_change, 0,
	"log interface link state change events");
#endif

/* Interface description */
static unsigned int ifdescr_maxlen = 1024;
#if 0
SYSCTL_UINT(_net, OID_AUTO, ifdescr_maxlen, CTLFLAG_RW,
	&ifdescr_maxlen, 0,
	"administrative maximum length for interface description");

MALLOC_DEFINE(M_IFDESCR, "ifdescr", "ifnet descriptions");
#endif

#if 0
/* global sx for non-critical path ifdescr */
static struct sx ifdescr_sx;
SX_SYSINIT(ifdescr_sx, &ifdescr_sx, "ifnet descr");
#endif

void	(*bridge_linkstate_p)(struct ifnet *ifp);
void	(*ng_ether_link_state_p)(struct ifnet *ifp, int state);
void	(*lagg_linkstate_p)(struct ifnet *ifp, int state);

struct mbuf *(*tbr_dequeue_ptr)(struct ifqueue *, int) = NULL;

/*
 * XXX: Style; these should be sorted alphabetically, and unprototyped
 * static functions should be prototyped. Currently they are sorted by
 * declaration order.
 */
static void	if_attachdomain1(struct ifnet *);
static int	ifconf(u_long, caddr_t);
static void	if_freemulti(struct ifmultiaddr *);
static void	if_grow(void);
static void	if_route(struct ifnet *, int flag, int fam);
static int	if_setflag(struct ifnet *, int, int, int *, int);
static int	if_transmit(struct ifnet *ifp, struct mbuf *m);
static void	if_unroute(struct ifnet *, int flag, int fam);
static void	link_rtrequest(int, struct rtentry *, struct rt_addrinfo *);
static int	if_rtdel(struct radix_node *, void *);
static int	ifhwioctl(u_long, struct ifnet *, caddr_t, struct thread *);
static int	if_delmulti_locked(struct ifnet *, struct ifmultiaddr *, int);
static void	do_link_state_change(void *, int);
static int	if_getgroup(struct ifgroupreq *, struct ifnet *);
static int	if_getgroupmembers(struct ifgroupreq *);
static void	if_delgroups(struct ifnet *);
static void	if_attach_internal(struct ifnet *, int);
static void	if_detach_internal(struct ifnet *, int);

#ifdef INET6
/*
 * XXX: declare here to avoid to include many inet6 related files..
 * should be more generalized?
 */
extern void	nd6_setmtu(struct ifnet *);
#endif

VNET_DEFINE(int, if_index);
int	ifqmaxlen = IFQ_MAXLEN;
VNET_DEFINE(struct ifnethead, ifnet);	/* depend on static init XXX */
VNET_DEFINE(struct ifgrouphead, ifg_head);

static VNET_DEFINE(int, if_indexlim) = 8;

/* Table of ifnet by index. */
VNET_DEFINE(struct ifindex_entry *, ifindex_table);

#define	V_if_indexlim		VNET(if_indexlim)
#define	V_ifindex_table		VNET(ifindex_table)

/*
 * The global network interface list (V_ifnet) and related state (such as
 * if_index, if_indexlim, and ifindex_table) are protected by an sxlock and
 * an rwlock.  Either may be acquired shared to stablize the list, but both
 * must be acquired writable to modify the list.  This model allows us to
 * both stablize the interface list during interrupt thread processing, but
 * also to stablize it over long-running ioctls, without introducing priority
 * inversions and deadlocks.
 */
// struct rwlock ifnet_rwlock;
// struct sx ifnet_sxlock;
struct cmutex ifnet_rwmutex;
struct cmutex ifnet_sxmutex;

/*
 * The allocation of network interfaces is a rather non-atomic affair; we
 * need to select an index before we are ready to expose the interface for
 * use, so will use this pointer value to indicate reservation.
 */
#define	IFNET_HOLD	(void *)(uintptr_t)(-1)

static	if_com_alloc_t *if_com_alloc[256];
static	if_com_free_t *if_com_free[256];

#if 0
MALLOC_DEFINE(M_IFNET, "ifnet", "interface internals");
MALLOC_DEFINE(M_IFADDR, "ifaddr", "interface address");
MALLOC_DEFINE(M_IFMADDR, "ether_multi", "link-level multicast address");
#endif

struct ifnet *
ifnet_byindex_locked(u_short idx)
{

	if (idx > V_if_index)
		return (NULL);
	if (V_ifindex_table[idx].ife_ifnet == IFNET_HOLD)
		return (NULL);
	return (V_ifindex_table[idx].ife_ifnet);
}

struct ifnet *
ifnet_byindex(u_short idx)
{
	struct ifnet *ifp;

	IFNET_RLOCK_NOSLEEP();
	ifp = ifnet_byindex_locked(idx);
	IFNET_RUNLOCK_NOSLEEP();
	return (ifp);
}

struct ifnet *
ifnet_byindex_ref(u_short idx)
{
	struct ifnet *ifp;

	IFNET_RLOCK_NOSLEEP();
	ifp = ifnet_byindex_locked(idx);
	if (ifp == NULL || (ifp->if_flags & IFF_DYING)) {
		IFNET_RUNLOCK_NOSLEEP();
		return (NULL);
	}
	if_ref(ifp);
	IFNET_RUNLOCK_NOSLEEP();
	return (ifp);
}

/*
 * Allocate an ifindex array entry; return 0 on success or an error on
 * failure.
 */
static int
ifindex_alloc_locked(u_short *idxp)
{
	u_short idx;

	IFNET_WLOCK_ASSERT();

retry:
	/*
	 * Try to find an empty slot below V_if_index.  If we fail, take the
	 * next slot.
	 */
	for (idx = 1; idx <= V_if_index; idx++) {
		if (V_ifindex_table[idx].ife_ifnet == NULL)
			break;
	}

	/* Catch if_index overflow. */
	if (idx < 1)
		return (ENOSPC);
	if (idx >= V_if_indexlim) {
		if_grow();
		goto retry;
	}
	if (idx > V_if_index)
		V_if_index = idx;
	*idxp = idx;
	return (0);
}

static void
ifindex_free_locked(u_short idx)
{

	IFNET_WLOCK_ASSERT();

	V_ifindex_table[idx].ife_ifnet = NULL;
	while (V_if_index > 0 &&
	    V_ifindex_table[V_if_index].ife_ifnet == NULL)
		V_if_index--;
}

static void
ifindex_free(u_short idx)
{

	IFNET_WLOCK();
	ifindex_free_locked(idx);
	IFNET_WUNLOCK();
}

static void
ifnet_setbyindex_locked(u_short idx, struct ifnet *ifp)
{

	IFNET_WLOCK_ASSERT();

	V_ifindex_table[idx].ife_ifnet = ifp;
}

static void
ifnet_setbyindex(u_short idx, struct ifnet *ifp)
{

	IFNET_WLOCK();
	ifnet_setbyindex_locked(idx, ifp);
	IFNET_WUNLOCK();
}

struct ifaddr *
ifaddr_byindex(u_short idx)
{
	struct ifaddr *ifa;

	IFNET_RLOCK_NOSLEEP();
	ifa = ifnet_byindex_locked(idx)->if_addr;
	if (ifa != NULL)
		ifa_ref(ifa);
	IFNET_RUNLOCK_NOSLEEP();
	return (ifa);
}

/*
 * Network interface utility routines.
 *
 * Routines with ifa_ifwith* names take sockaddr *'s as
 * parameters.
 */

void vnet_if_init(const void *__unused)
{

	TAILQ_INIT(&V_ifnet);
	TAILQ_INIT(&V_ifg_head);
	IFNET_WLOCK();
	if_grow();				/* create initial table */
	IFNET_WUNLOCK();
	vnet_if_clone_init();
}

#if 0
VNET_SYSINIT(vnet_if_init, SI_SUB_INIT_IF, SI_ORDER_SECOND, vnet_if_init,
    NULL);
#endif

/* ARGSUSED*/
void if_init(void *__unused)
{
	IFNET_LOCK_INIT();
	if_clone_init();
}

#if 0
SYSINIT(interfaces, SI_SUB_INIT_IF, SI_ORDER_FIRST, if_init, NULL);
#endif

static void
if_grow(void)
{
	int oldlim;
	u_int n;
	struct ifindex_entry *e;

	IFNET_WLOCK_ASSERT();
	oldlim = V_if_indexlim;
	IFNET_WUNLOCK();
	n = (oldlim << 1) * sizeof(*e);
	e = malloc(n);
	bzero(e, n);
	IFNET_WLOCK();
	if (V_if_indexlim != oldlim) {
		free(e);
		return;
	}
	if (V_ifindex_table != NULL) {
		memcpy((caddr_t)e, (caddr_t)V_ifindex_table, n/2);
		free((caddr_t)V_ifindex_table);
	}
	V_if_indexlim <<= 1;
	V_ifindex_table = e;
}

/*
 * Allocate a struct ifnet and an index for an interface.  A layer 2
 * common structure will also be allocated if an allocation routine is
 * registered for the passed type.
 */
struct ifnet *
if_alloc(u_char type)
{
	struct ifnet *ifp;
	u_short idx;

	ifp = malloc(sizeof(struct ifnet));
	bzero(ifp, sizeof(struct ifnet));
	IFNET_WLOCK();
	if (ifindex_alloc_locked(&idx) != 0) {
		IFNET_WUNLOCK();
		free(ifp);
		return (NULL);
	}
	ifnet_setbyindex_locked(idx, IFNET_HOLD);
	IFNET_WUNLOCK();
	ifp->if_index = idx;
	ifp->if_type = type;
	ifp->if_alloctype = type;
	if (if_com_alloc[type] != NULL) {
		ifp->if_l2com = if_com_alloc[type](type, ifp);
		if (ifp->if_l2com == NULL) {
			free(ifp);
			ifindex_free(idx);
			return (NULL);
		}
	}

	IF_ADDR_LOCK_INIT(ifp);
	/* OSv: Avoid using taksqueues */
	/* TASK_INIT(&ifp->if_linktask, 0, do_link_state_change, ifp); */
	ifp->if_linktask = (void*)0;
	ifp->if_afdata_initialized = 0;
	IF_AFDATA_LOCK_INIT(ifp);
	TAILQ_INIT(&ifp->if_addrhead);
	TAILQ_INIT(&ifp->if_prefixhead);
	TAILQ_INIT(&ifp->if_multiaddrs);
	TAILQ_INIT(&ifp->if_groups);
#ifdef MAC
	mac_ifnet_init(ifp);
#endif
	ifq_init(&ifp->if_snd, ifp);

	refcount_init(&ifp->if_refcount, 1);	/* Index reference. */
	ifnet_setbyindex(ifp->if_index, ifp);
	return (ifp);
}

/*
 * Do the actual work of freeing a struct ifnet, and layer 2 common
 * structure.  This call is made when the last reference to an
 * interface is released.
 */
static void
if_free_internal(struct ifnet *ifp)
{

	KASSERT((ifp->if_flags & IFF_DYING),
	    ("if_free_internal: interface not dying"));

	if (if_com_free[ifp->if_alloctype] != NULL)
		if_com_free[ifp->if_alloctype](ifp->if_l2com,
		    ifp->if_alloctype);

	if (ifp->if_description != NULL)
		free(ifp->if_description);
	IF_AFDATA_DESTROY(ifp);
	IF_ADDR_LOCK_DESTROY(ifp);
	ifq_delete(&ifp->if_snd);
	free(ifp);
}

/*
 * This version should only be called by intefaces that switch their type
 * after calling if_alloc().  if_free_type() will go away again now that we
 * have if_alloctype to cache the original allocation type.  For now, assert
 * that they match, since we require that in practice.
 */
void
if_free_type(struct ifnet *ifp, u_char type)
{

	KASSERT(ifp->if_alloctype == type,
	    ("if_free_type: type (%d) != alloctype (%d)", type,
	    ifp->if_alloctype));

	ifp->if_flags |= IFF_DYING;			/* XXX: Locking */

	IFNET_WLOCK();
	KASSERT(ifp == ifnet_byindex_locked(ifp->if_index),
	    ("%s: freeing unallocated ifnet", ifp->if_xname));

	ifindex_free_locked(ifp->if_index);
	IFNET_WUNLOCK();

	if (!refcount_release(&ifp->if_refcount))
		return;
	if_free_internal(ifp);
}

/*
 * This is the normal version of if_free(), used by device drivers to free a
 * detached network interface.  The contents of if_free_type() will move into
 * here when if_free_type() goes away.
 */
void
if_free(struct ifnet *ifp)
{

	if_free_type(ifp, ifp->if_alloctype);
}

/*
 * Interfaces to keep an ifnet type-stable despite the possibility of the
 * driver calling if_free().  If there are additional references, we defer
 * freeing the underlying data structure.
 */
void
if_ref(struct ifnet *ifp)
{

	/* We don't assert the ifnet list lock here, but arguably should. */
	refcount_acquire(&ifp->if_refcount);
}

void
if_rele(struct ifnet *ifp)
{

	if (!refcount_release(&ifp->if_refcount))
		return;
	if_free_internal(ifp);
}

void
ifq_init(struct ifqueue *ifq, struct ifnet *ifp)
{
	
	mtx_init(&ifq->ifq_mtx, ifp->if_xname, "if send queue", MTX_DEF);

	if (ifq->ifq_maxlen == 0) 
		ifq->ifq_maxlen = ifqmaxlen;
}

void
ifq_delete(struct ifqueue *ifq)
{
	mtx_destroy(&ifq->ifq_mtx);
}

/*
 * Perform generic interface initalization tasks and attach the interface
 * to the list of "active" interfaces.  If vmove flag is set on entry
 * to if_attach_internal(), perform only a limited subset of initialization
 * tasks, given that we are moving from one vnet to another an ifnet which
 * has already been fully initialized.
 *
 * XXX:
 *  - The decision to return void and thus require this function to
 *    succeed is questionable.
 *  - We should probably do more sanity checking.  For instance we don't
 *    do anything to insure if_xname is unique or non-empty.
 */
void
if_attach(struct ifnet *ifp)
{

	if_attach_internal(ifp, 0);
}

static void
if_attach_internal(struct ifnet *ifp, int vmove)
{
	unsigned socksize, ifasize;
	int namelen, masklen;
	struct sockaddr_dl *sdl;
	struct ifaddr *ifa;

	if (ifp->if_index == 0 || ifp != ifnet_byindex(ifp->if_index))
		panic ("%s: BUG: if_attach called without if_alloc'd input()\n",
		    ifp->if_xname);

	if_addgroup(ifp, IFG_ALL);

	ifp->if_data.ifi_datalen = sizeof(struct if_data);

	KASSERT((ifp->if_transmit == NULL && ifp->if_qflush == NULL) ||
	    (ifp->if_transmit != NULL && ifp->if_qflush != NULL),
	    ("transmit and qflush must both either be set or both be NULL"));
	if (ifp->if_transmit == NULL) {
		ifp->if_transmit = if_transmit;
		ifp->if_qflush = if_qflush;
	}
	
	if (!vmove) {
#ifdef MAC
		mac_ifnet_create(ifp);
#endif

		/*
		 * Create a Link Level name for this device.
		 */
		namelen = strlen(ifp->if_xname);
		/*
		 * Always save enough space for any possiable name so we
		 * can do a rename in place later.
		 */
		masklen = offsetof(struct sockaddr_dl, sdl_data[0]) + IFNAMSIZ;
		socksize = masklen + ifp->if_addrlen;
		if (socksize < sizeof(*sdl))
			socksize = sizeof(*sdl);
		socksize = roundup2(socksize, sizeof(long));
		ifasize = sizeof(*ifa) + 2 * socksize;
		ifa = malloc(ifasize);
		bzero(ifa, ifasize);
		ifa_init(ifa);
		sdl = (struct sockaddr_dl *)(ifa + 1);
		sdl->sdl_len = socksize;
		sdl->sdl_family = AF_LINK;
		bcopy(ifp->if_xname, sdl->sdl_data, namelen);
		sdl->sdl_nlen = namelen;
		sdl->sdl_index = ifp->if_index;
		sdl->sdl_type = ifp->if_type;
		ifp->if_addr = ifa;
		ifa->ifa_ifp = ifp;
		ifa->ifa_rtrequest = link_rtrequest;
		ifa->ifa_addr = (struct sockaddr *)sdl;
		sdl = (struct sockaddr_dl *)(socksize + (caddr_t)sdl);
		ifa->ifa_netmask = (struct sockaddr *)sdl;
		sdl->sdl_len = masklen;
		while (namelen != 0)
			sdl->sdl_data[--namelen] = 0xff;
		TAILQ_INSERT_HEAD(&ifp->if_addrhead, ifa, ifa_link);
		/* Reliably crash if used uninitialized. */
		ifp->if_broadcastaddr = NULL;
	}

	IFNET_WLOCK();
	TAILQ_INSERT_TAIL(&V_ifnet, ifp, if_link);
	IFNET_WUNLOCK();

	if (domain_init_status >= 2)
		if_attachdomain1(ifp);

	EVENTHANDLER_INVOKE(ifnet_arrival_event, ifp);

#if 0
	if (IS_DEFAULT_VNET(curvnet))
		devctl_notify("IFNET", ifp->if_xname, "ATTACH", NULL);
#endif

	/* Announce the interface. */
	rt_ifannouncemsg(ifp, IFAN_ARRIVAL);
}

void if_attachdomain(void *dummy)
{
	struct ifnet *ifp;
	int s;

	s = splnet();
	TAILQ_FOREACH(ifp, &V_ifnet, if_link)
		if_attachdomain1(ifp);
	splx(s);
}

#if 0
SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_SECOND,
    if_attachdomain, NULL);
#endif

static void
if_attachdomain1(struct ifnet *ifp)
{
	struct domain *dp;
	int s;

	s = splnet();

	/*
	 * Since dp->dom_ifattach calls malloc() with M_WAITOK, we
	 * cannot lock ifp->if_afdata initialization, entirely.
	 */
	if (IF_AFDATA_TRYLOCK(ifp) == 0) {
		splx(s);
		return;
	}
	if (ifp->if_afdata_initialized >= domain_init_status) {
		IF_AFDATA_UNLOCK(ifp);
		splx(s);
		printf("if_attachdomain called more than once on %s\n",
		    ifp->if_xname);
		return;
	}
	ifp->if_afdata_initialized = domain_init_status;
	IF_AFDATA_UNLOCK(ifp);

	/* address family dependent data region */
	bzero(ifp->if_afdata, sizeof(ifp->if_afdata));
	for (dp = domains; dp; dp = dp->dom_next) {
		if (dp->dom_ifattach)
			ifp->if_afdata[dp->dom_family] =
			    (*dp->dom_ifattach)(ifp);
	}

	splx(s);
}

/*
 * Remove any unicast or broadcast network addresses from an interface.
 */
void
if_purgeaddrs(struct ifnet *ifp)
{
	struct ifaddr *ifa, *next;

	TAILQ_FOREACH_SAFE(ifa, &ifp->if_addrhead, ifa_link, next) {
		if (ifa->ifa_addr->sa_family == AF_LINK)
			continue;
#ifdef INET
		/* XXX: Ugly!! ad hoc just for INET */
		if (ifa->ifa_addr->sa_family == AF_INET) {
			struct ifaliasreq ifr;

			bzero(&ifr, sizeof(ifr));
			ifr.ifra_addr = *ifa->ifa_addr;
			if (ifa->ifa_dstaddr)
				ifr.ifra_broadaddr = *ifa->ifa_dstaddr;
			if (in_control(NULL, SIOCDIFADDR, (caddr_t)&ifr, ifp,
			    NULL) == 0)
				continue;
		}
#endif /* INET */
#ifdef INET6
		if (ifa->ifa_addr->sa_family == AF_INET6) {
			in6_purgeaddr(ifa);
			/* ifp_addrhead is already updated */
			continue;
		}
#endif /* INET6 */
		TAILQ_REMOVE(&ifp->if_addrhead, ifa, ifa_link);
		ifa_free(ifa);
	}
}

/*
 * Remove any multicast network addresses from an interface when an ifnet
 * is going away.
 */
static void
if_purgemaddrs(struct ifnet *ifp)
{
	struct ifmultiaddr *ifma;
	struct ifmultiaddr *next;

	IF_ADDR_WLOCK(ifp);
	TAILQ_FOREACH_SAFE(ifma, &ifp->if_multiaddrs, ifma_link, next)
		if_delmulti_locked(ifp, ifma, 1);
	IF_ADDR_WUNLOCK(ifp);
}

/*
 * Detach an interface, removing it from the list of "active" interfaces.
 * If vmove flag is set on entry to if_detach_internal(), perform only a
 * limited subset of cleanup tasks, given that we are moving an ifnet from
 * one vnet to another, where it must be fully operational.
 *
 * XXXRW: There are some significant questions about event ordering, and
 * how to prevent things from starting to use the interface during detach.
 */
void
if_detach(struct ifnet *ifp)
{

	if_detach_internal(ifp, 0);
}

static void
if_detach_internal(struct ifnet *ifp, int vmove)
{
	struct ifaddr *ifa;
	struct radix_node_head	*rnh;
	int i, j;
	struct domain *dp;
 	struct ifnet *iter;
 	int found = 0;

	IFNET_WLOCK();
	TAILQ_FOREACH(iter, &V_ifnet, if_link)
		if (iter == ifp) {
			TAILQ_REMOVE(&V_ifnet, ifp, if_link);
			found = 1;
			break;
		}
	IFNET_WUNLOCK();
	if (!found) {
		if (vmove)
			panic("%s: ifp=%p not on the ifnet tailq %p",
			    __func__, ifp, &V_ifnet);
		else
			return; /* XXX this should panic as well? */
	}

	/*
	 * Remove/wait for pending events.
	 */

	/* OSv: No taskqueue, no drain */
	/* taskqueue_drain(taskqueue_swi, &ifp->if_linktask); */

	/*
	 * Remove routes and flush queues.
	 */
	if_down(ifp);

	if_purgeaddrs(ifp);

#ifdef INET
	in_ifdetach(ifp);
#endif

#ifdef INET6
	/*
	 * Remove all IPv6 kernel structs related to ifp.  This should be done
	 * before removing routing entries below, since IPv6 interface direct
	 * routes are expected to be removed by the IPv6-specific kernel API.
	 * Otherwise, the kernel will detect some inconsistency and bark it.
	 */
	in6_ifdetach(ifp);
#endif
	if_purgemaddrs(ifp);

	if (!vmove) {
		/*
		 * Prevent further calls into the device driver via ifnet.
		 */
		if_dead(ifp);

		/*
		 * Remove link ifaddr pointer and maybe decrement if_index.
		 * Clean up all addresses.
		 */
		ifp->if_addr = NULL;

		/* We can now free link ifaddr. */
		if (!TAILQ_EMPTY(&ifp->if_addrhead)) {
			ifa = TAILQ_FIRST(&ifp->if_addrhead);
			TAILQ_REMOVE(&ifp->if_addrhead, ifa, ifa_link);
			ifa_free(ifa);
		}
	}

	/*
	 * Delete all remaining routes using this interface
	 * Unfortuneatly the only way to do this is to slog through
	 * the entire routing table looking for routes which point
	 * to this interface...oh well...
	 */
	for (i = 1; i <= AF_MAX; i++) {
		for (j = 0; j < rt_numfibs; j++) {
			rnh = rt_tables_get_rnh(j, i);
			if (rnh == NULL)
				continue;
			RADIX_NODE_HEAD_LOCK(rnh);
			(void) rnh->rnh_walktree(rnh, if_rtdel, ifp);
			RADIX_NODE_HEAD_UNLOCK(rnh);
		}
	}

	/* Announce that the interface is gone. */
	rt_ifannouncemsg(ifp, IFAN_DEPARTURE);
	EVENTHANDLER_INVOKE(ifnet_departure_event, ifp);

#if 0
	if (IS_DEFAULT_VNET(curvnet))
		devctl_notify("IFNET", ifp->if_xname, "DETACH", NULL);
#endif

	if_delgroups(ifp);

	/*
	 * We cannot hold the lock over dom_ifdetach calls as they might
	 * sleep, for example trying to drain a callout, thus open up the
	 * theoretical race with re-attaching.
	 */
	IF_AFDATA_LOCK(ifp);
	i = ifp->if_afdata_initialized;
	ifp->if_afdata_initialized = 0;
	IF_AFDATA_UNLOCK(ifp);
	for (dp = domains; i > 0 && dp; dp = dp->dom_next) {
		if (dp->dom_ifdetach && ifp->if_afdata[dp->dom_family])
			(*dp->dom_ifdetach)(ifp,
			    ifp->if_afdata[dp->dom_family]);
	}
}

/*
 * Add a group to an interface
 */
int
if_addgroup(struct ifnet *ifp, const char *groupname)
{
	struct ifg_list		*ifgl;
	struct ifg_group	*ifg = NULL;
	struct ifg_member	*ifgm;

	if (groupname[0] && groupname[strlen(groupname) - 1] >= '0' &&
	    groupname[strlen(groupname) - 1] <= '9')
		return (EINVAL);

	IFNET_WLOCK();
	TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
		if (!strcmp(ifgl->ifgl_group->ifg_group, groupname)) {
			IFNET_WUNLOCK();
			return (EEXIST);
		}

	if ((ifgl = (struct ifg_list *)malloc(sizeof(struct ifg_list))) == NULL) {
	    	IFNET_WUNLOCK();
		return (ENOMEM);
	}

	if ((ifgm = (struct ifg_member *)malloc(sizeof(struct ifg_member))) == NULL) {
		free(ifgl);
		IFNET_WUNLOCK();
		return (ENOMEM);
	}

	TAILQ_FOREACH(ifg, &V_ifg_head, ifg_next)
		if (!strcmp(ifg->ifg_group, groupname))
			break;

	if (ifg == NULL) {
		if ((ifg = (struct ifg_group *)malloc(sizeof(struct ifg_group))) == NULL) {
			free(ifgl);
			free(ifgm);
			IFNET_WUNLOCK();
			return (ENOMEM);
		}
		strlcpy(ifg->ifg_group, groupname, sizeof(ifg->ifg_group));
		ifg->ifg_refcnt = 0;
		TAILQ_INIT(&ifg->ifg_members);
		EVENTHANDLER_INVOKE(group_attach_event, ifg);
		TAILQ_INSERT_TAIL(&V_ifg_head, ifg, ifg_next);
	}

	ifg->ifg_refcnt++;
	ifgl->ifgl_group = ifg;
	ifgm->ifgm_ifp = ifp;

	IF_ADDR_WLOCK(ifp);
	TAILQ_INSERT_TAIL(&ifg->ifg_members, ifgm, ifgm_next);
	TAILQ_INSERT_TAIL(&ifp->if_groups, ifgl, ifgl_next);
	IF_ADDR_WUNLOCK(ifp);

	IFNET_WUNLOCK();

	EVENTHANDLER_INVOKE(group_change_event, groupname);

	return (0);
}

/*
 * Remove a group from an interface
 */
int
if_delgroup(struct ifnet *ifp, const char *groupname)
{
	struct ifg_list		*ifgl;
	struct ifg_member	*ifgm;

	IFNET_WLOCK();
	TAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
		if (!strcmp(ifgl->ifgl_group->ifg_group, groupname))
			break;
	if (ifgl == NULL) {
		IFNET_WUNLOCK();
		return (ENOENT);
	}

	IF_ADDR_WLOCK(ifp);
	TAILQ_REMOVE(&ifp->if_groups, ifgl, ifgl_next);
	IF_ADDR_WUNLOCK(ifp);

	TAILQ_FOREACH(ifgm, &ifgl->ifgl_group->ifg_members, ifgm_next)
		if (ifgm->ifgm_ifp == ifp)
			break;

	if (ifgm != NULL) {
		TAILQ_REMOVE(&ifgl->ifgl_group->ifg_members, ifgm, ifgm_next);
		free(ifgm);
	}

	if (--ifgl->ifgl_group->ifg_refcnt == 0) {
		TAILQ_REMOVE(&V_ifg_head, ifgl->ifgl_group, ifg_next);
		EVENTHANDLER_INVOKE(group_detach_event, ifgl->ifgl_group);
		free(ifgl->ifgl_group);
	}
	IFNET_WUNLOCK();

	free(ifgl);

	EVENTHANDLER_INVOKE(group_change_event, groupname);

	return (0);
}

/*
 * Remove an interface from all groups
 */
static void
if_delgroups(struct ifnet *ifp)