xfrm_policy.c 80.9 KB
Newer Older
1
/*
Linus Torvalds's avatar
Linus Torvalds committed
2 3 4 5 6 7 8 9 10 11 12
 * xfrm_policy.c
 *
 * Changes:
 *	Mitsuru KANDA @USAGI
 * 	Kazunori MIYAZAWA @USAGI
 * 	Kunihiro Ishiguro <kunihiro@ipinfusion.com>
 * 		IPv6 support
 * 	Kazunori MIYAZAWA @USAGI
 * 	YOSHIFUJI Hideaki
 * 		Split up af-specific portion
 *	Derek Atkins <derek@ihtfp.com>		Add the post_input processor
13
 *
Linus Torvalds's avatar
Linus Torvalds committed
14 15
 */

16
#include <linux/err.h>
Linus Torvalds's avatar
Linus Torvalds committed
17 18 19 20 21 22 23
#include <linux/slab.h>
#include <linux/kmod.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
#include <linux/notifier.h>
#include <linux/netdevice.h>
24
#include <linux/netfilter.h>
Linus Torvalds's avatar
Linus Torvalds committed
25
#include <linux/module.h>
26
#include <linux/cache.h>
Paul Moore's avatar
Paul Moore committed
27
#include <linux/audit.h>
28
#include <net/dst.h>
29
#include <net/flow.h>
Linus Torvalds's avatar
Linus Torvalds committed
30 31
#include <net/xfrm.h>
#include <net/ip.h>
32 33 34
#ifdef CONFIG_XFRM_STATISTICS
#include <net/snmp.h>
#endif
Linus Torvalds's avatar
Linus Torvalds committed
35

36 37
#include "xfrm_hash.h"

38 39 40 41
#define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
#define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
#define XFRM_MAX_QUEUE_LEN	100

42 43 44 45 46
struct xfrm_flo {
	struct dst_entry *dst_orig;
	u8 flags;
};

47
static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
48
static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
49
						__read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
50

51
static struct kmem_cache *xfrm_dst_cache __read_mostly;
52
static __read_mostly seqcount_t xfrm_policy_hash_generation;
Linus Torvalds's avatar
Linus Torvalds committed
53

54
static void xfrm_init_pmtu(struct dst_entry *dst);
55
static int stale_bundle(struct dst_entry *dst);
56
static int xfrm_bundle_ok(struct xfrm_dst *xdst);
57
static void xfrm_policy_queue_process(unsigned long arg);
Linus Torvalds's avatar
Linus Torvalds committed
58

59
static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
60 61 62
static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
						int dir);

63 64 65 66 67
static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
{
	return atomic_inc_not_zero(&policy->refcnt);
}

68
static inline bool
69
__xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
70
{
71 72
	const struct flowi4 *fl4 = &fl->u.ip4;

73 74
	return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
		addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
75 76 77 78
		!((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
		!((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
		(fl4->flowi4_proto == sel->proto || !sel->proto) &&
		(fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
79 80
}

81
static inline bool
82
__xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
83
{
84 85 86 87 88 89 90 91
	const struct flowi6 *fl6 = &fl->u.ip6;

	return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
		addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
		!((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
		!((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
		(fl6->flowi6_proto == sel->proto || !sel->proto) &&
		(fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
92 93
}

94 95
bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
			 unsigned short family)
96 97 98 99 100 101 102
{
	switch (family) {
	case AF_INET:
		return __xfrm4_selector_match(sel, fl);
	case AF_INET6:
		return __xfrm6_selector_match(sel, fl);
	}
103
	return false;
104 105
}

106
static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
Eric Dumazet's avatar
Eric Dumazet committed
107
{
108
	const struct xfrm_policy_afinfo *afinfo;
Eric Dumazet's avatar
Eric Dumazet committed
109

110
	if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
Eric Dumazet's avatar
Eric Dumazet committed
111 112 113 114 115 116 117 118
		return NULL;
	rcu_read_lock();
	afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
	if (unlikely(!afinfo))
		rcu_read_unlock();
	return afinfo;
}

David Ahern's avatar
David Ahern committed
119 120
static inline struct dst_entry *__xfrm_dst_lookup(struct net *net,
						  int tos, int oif,
121 122
						  const xfrm_address_t *saddr,
						  const xfrm_address_t *daddr,
123 124
						  int family)
{
125
	const struct xfrm_policy_afinfo *afinfo;
126 127 128 129 130 131
	struct dst_entry *dst;

	afinfo = xfrm_policy_get_afinfo(family);
	if (unlikely(afinfo == NULL))
		return ERR_PTR(-EAFNOSUPPORT);

David Ahern's avatar
David Ahern committed
132
	dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr);
133

134
	rcu_read_unlock();
135 136 137 138

	return dst;
}

David Ahern's avatar
David Ahern committed
139 140
static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
						int tos, int oif,
141 142
						xfrm_address_t *prev_saddr,
						xfrm_address_t *prev_daddr,
143
						int family)
Linus Torvalds's avatar
Linus Torvalds committed
144
{
145
	struct net *net = xs_net(x);
146 147 148 149
	xfrm_address_t *saddr = &x->props.saddr;
	xfrm_address_t *daddr = &x->id.daddr;
	struct dst_entry *dst;

150
	if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
151
		saddr = x->coaddr;
152 153 154 155
		daddr = prev_daddr;
	}
	if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
		saddr = prev_saddr;
156
		daddr = x->coaddr;
157
	}
Linus Torvalds's avatar
Linus Torvalds committed
158

David Ahern's avatar
David Ahern committed
159
	dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family);
160 161 162 163 164 165 166

	if (!IS_ERR(dst)) {
		if (prev_saddr != saddr)
			memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
		if (prev_daddr != daddr)
			memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
	}
Linus Torvalds's avatar
Linus Torvalds committed
167

168
	return dst;
Linus Torvalds's avatar
Linus Torvalds committed
169 170 171 172 173 174 175
}

static inline unsigned long make_jiffies(long secs)
{
	if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
		return MAX_SCHEDULE_TIMEOUT-1;
	else
176
		return secs*HZ;
Linus Torvalds's avatar
Linus Torvalds committed
177 178 179 180
}

static void xfrm_policy_timer(unsigned long data)
{
181
	struct xfrm_policy *xp = (struct xfrm_policy *)data;
182
	unsigned long now = get_seconds();
Linus Torvalds's avatar
Linus Torvalds committed
183 184 185 186 187 188
	long next = LONG_MAX;
	int warn = 0;
	int dir;

	read_lock(&xp->lock);

189
	if (unlikely(xp->walk.dead))
Linus Torvalds's avatar
Linus Torvalds committed
190 191
		goto out;

192
	dir = xfrm_policy_id2dir(xp->index);
Linus Torvalds's avatar
Linus Torvalds committed
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231

	if (xp->lft.hard_add_expires_seconds) {
		long tmo = xp->lft.hard_add_expires_seconds +
			xp->curlft.add_time - now;
		if (tmo <= 0)
			goto expired;
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.hard_use_expires_seconds) {
		long tmo = xp->lft.hard_use_expires_seconds +
			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
		if (tmo <= 0)
			goto expired;
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.soft_add_expires_seconds) {
		long tmo = xp->lft.soft_add_expires_seconds +
			xp->curlft.add_time - now;
		if (tmo <= 0) {
			warn = 1;
			tmo = XFRM_KM_TIMEOUT;
		}
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.soft_use_expires_seconds) {
		long tmo = xp->lft.soft_use_expires_seconds +
			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
		if (tmo <= 0) {
			warn = 1;
			tmo = XFRM_KM_TIMEOUT;
		}
		if (tmo < next)
			next = tmo;
	}

	if (warn)
232
		km_policy_expired(xp, dir, 0, 0);
Linus Torvalds's avatar
Linus Torvalds committed
233 234 235 236 237 238 239 240 241 242 243
	if (next != LONG_MAX &&
	    !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
		xfrm_pol_hold(xp);

out:
	read_unlock(&xp->lock);
	xfrm_pol_put(xp);
	return;

expired:
	read_unlock(&xp->lock);
244
	if (!xfrm_policy_delete(xp, dir))
245
		km_policy_expired(xp, dir, 1, 0);
Linus Torvalds's avatar
Linus Torvalds committed
246 247 248
	xfrm_pol_put(xp);
}

249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo)
{
	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);

	if (unlikely(pol->walk.dead))
		flo = NULL;
	else
		xfrm_pol_hold(pol);

	return flo;
}

static int xfrm_policy_flo_check(struct flow_cache_object *flo)
{
	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);

	return !pol->walk.dead;
}

static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
{
	xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
}

static const struct flow_cache_ops xfrm_policy_fc_ops = {
	.get = xfrm_policy_flo_get,
	.check = xfrm_policy_flo_check,
	.delete = xfrm_policy_flo_delete,
};
Linus Torvalds's avatar
Linus Torvalds committed
278 279 280 281 282

/* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
 * SPD calls.
 */

283
struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
Linus Torvalds's avatar
Linus Torvalds committed
284 285 286
{
	struct xfrm_policy *policy;

287
	policy = kzalloc(sizeof(struct xfrm_policy), gfp);
Linus Torvalds's avatar
Linus Torvalds committed
288 289

	if (policy) {
290
		write_pnet(&policy->xp_net, net);
291
		INIT_LIST_HEAD(&policy->walk.all);
292 293
		INIT_HLIST_NODE(&policy->bydst);
		INIT_HLIST_NODE(&policy->byidx);
Linus Torvalds's avatar
Linus Torvalds committed
294
		rwlock_init(&policy->lock);
295
		atomic_set(&policy->refcnt, 1);
296
		skb_queue_head_init(&policy->polq.hold_queue);
297 298
		setup_timer(&policy->timer, xfrm_policy_timer,
				(unsigned long)policy);
299 300
		setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process,
			    (unsigned long)policy);
301
		policy->flo.ops = &xfrm_policy_fc_ops;
Linus Torvalds's avatar
Linus Torvalds committed
302 303 304 305 306
	}
	return policy;
}
EXPORT_SYMBOL(xfrm_policy_alloc);

307 308 309 310 311 312 313 314
static void xfrm_policy_destroy_rcu(struct rcu_head *head)
{
	struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);

	security_xfrm_policy_free(policy->security);
	kfree(policy);
}

Linus Torvalds's avatar
Linus Torvalds committed
315 316
/* Destroy xfrm_policy: descendant resources must be released to this moment. */

317
void xfrm_policy_destroy(struct xfrm_policy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
318
{
319
	BUG_ON(!policy->walk.dead);
Linus Torvalds's avatar
Linus Torvalds committed
320

321
	if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
Linus Torvalds's avatar
Linus Torvalds committed
322 323
		BUG();

324
	call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
Linus Torvalds's avatar
Linus Torvalds committed
325
}
326
EXPORT_SYMBOL(xfrm_policy_destroy);
Linus Torvalds's avatar
Linus Torvalds committed
327

Alexander Alemayhu's avatar
Alexander Alemayhu committed
328
/* Rule must be locked. Release descendant resources, announce
Linus Torvalds's avatar
Linus Torvalds committed
329 330 331 332 333
 * entry dead. The rule must be unlinked from lists to the moment.
 */

static void xfrm_policy_kill(struct xfrm_policy *policy)
{
334
	policy->walk.dead = 1;
Linus Torvalds's avatar
Linus Torvalds committed
335

336
	atomic_inc(&policy->genid);
Linus Torvalds's avatar
Linus Torvalds committed
337

338 339
	if (del_timer(&policy->polq.hold_timer))
		xfrm_pol_put(policy);
340
	skb_queue_purge(&policy->polq.hold_queue);
341

342 343 344 345
	if (del_timer(&policy->timer))
		xfrm_pol_put(policy);

	xfrm_pol_put(policy);
Linus Torvalds's avatar
Linus Torvalds committed
346 347
}

348 349
static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;

350
static inline unsigned int idx_hash(struct net *net, u32 index)
351
{
352
	return __idx_hash(index, net->xfrm.policy_idx_hmask);
353 354
}

355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
/* calculate policy hash thresholds */
static void __get_hash_thresh(struct net *net,
			      unsigned short family, int dir,
			      u8 *dbits, u8 *sbits)
{
	switch (family) {
	case AF_INET:
		*dbits = net->xfrm.policy_bydst[dir].dbits4;
		*sbits = net->xfrm.policy_bydst[dir].sbits4;
		break;

	case AF_INET6:
		*dbits = net->xfrm.policy_bydst[dir].dbits6;
		*sbits = net->xfrm.policy_bydst[dir].sbits6;
		break;

	default:
		*dbits = 0;
		*sbits = 0;
	}
}

377 378 379
static struct hlist_head *policy_hash_bysel(struct net *net,
					    const struct xfrm_selector *sel,
					    unsigned short family, int dir)
380
{
381
	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
382 383 384 385 386 387
	unsigned int hash;
	u8 dbits;
	u8 sbits;

	__get_hash_thresh(net, family, dir, &dbits, &sbits);
	hash = __sel_hash(sel, family, hmask, dbits, sbits);
388

389 390 391 392 393
	if (hash == hmask + 1)
		return &net->xfrm.policy_inexact[dir];

	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
394 395
}

396 397 398 399
static struct hlist_head *policy_hash_direct(struct net *net,
					     const xfrm_address_t *daddr,
					     const xfrm_address_t *saddr,
					     unsigned short family, int dir)
400
{
401
	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
402 403 404 405 406 407
	unsigned int hash;
	u8 dbits;
	u8 sbits;

	__get_hash_thresh(net, family, dir, &dbits, &sbits);
	hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
408

409 410
	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
411 412
}

413 414
static void xfrm_dst_hash_transfer(struct net *net,
				   struct hlist_head *list,
415
				   struct hlist_head *ndsttable,
416 417
				   unsigned int nhashmask,
				   int dir)
418
{
419
	struct hlist_node *tmp, *entry0 = NULL;
420
	struct xfrm_policy *pol;
421
	unsigned int h0 = 0;
422 423
	u8 dbits;
	u8 sbits;
424

425
redo:
426
	hlist_for_each_entry_safe(pol, tmp, list, bydst) {
427 428
		unsigned int h;

429
		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
430
		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
431
				pol->family, nhashmask, dbits, sbits);
432
		if (!entry0) {
433 434
			hlist_del_rcu(&pol->bydst);
			hlist_add_head_rcu(&pol->bydst, ndsttable + h);
435 436 437 438
			h0 = h;
		} else {
			if (h != h0)
				continue;
439 440
			hlist_del_rcu(&pol->bydst);
			hlist_add_behind_rcu(&pol->bydst, entry0);
441
		}
442
		entry0 = &pol->bydst;
443 444 445 446
	}
	if (!hlist_empty(list)) {
		entry0 = NULL;
		goto redo;
447 448 449 450 451 452 453
	}
}

static void xfrm_idx_hash_transfer(struct hlist_head *list,
				   struct hlist_head *nidxtable,
				   unsigned int nhashmask)
{
454
	struct hlist_node *tmp;
455 456
	struct xfrm_policy *pol;

457
	hlist_for_each_entry_safe(pol, tmp, list, byidx) {
458 459 460 461 462 463 464 465 466 467 468 469
		unsigned int h;

		h = __idx_hash(pol->index, nhashmask);
		hlist_add_head(&pol->byidx, nidxtable+h);
	}
}

static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
{
	return ((old_hmask + 1) << 1) - 1;
}

470
static void xfrm_bydst_resize(struct net *net, int dir)
471
{
472
	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
473 474
	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
475
	struct hlist_head *ndst = xfrm_hash_alloc(nsize);
476
	struct hlist_head *odst;
477 478 479 480 481
	int i;

	if (!ndst)
		return;

482
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
483 484 485 486
	write_seqcount_begin(&xfrm_policy_hash_generation);

	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
				lockdep_is_held(&net->xfrm.xfrm_policy_lock));
487

488 489 490
	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
				lockdep_is_held(&net->xfrm.xfrm_policy_lock));

491
	for (i = hmask; i >= 0; i--)
492
		xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
493

494
	rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
495
	net->xfrm.policy_bydst[dir].hmask = nhashmask;
496

497
	write_seqcount_end(&xfrm_policy_hash_generation);
498
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
499

500 501
	synchronize_rcu();

502
	xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
503 504
}

505
static void xfrm_byidx_resize(struct net *net, int total)
506
{
507
	unsigned int hmask = net->xfrm.policy_idx_hmask;
508 509
	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
510
	struct hlist_head *oidx = net->xfrm.policy_byidx;
511
	struct hlist_head *nidx = xfrm_hash_alloc(nsize);
512 513 514 515 516
	int i;

	if (!nidx)
		return;

517
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
518 519 520 521

	for (i = hmask; i >= 0; i--)
		xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);

522 523
	net->xfrm.policy_byidx = nidx;
	net->xfrm.policy_idx_hmask = nhashmask;
524

525
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
526

527
	xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
528 529
}

530
static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
531
{
532 533
	unsigned int cnt = net->xfrm.policy_count[dir];
	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
534 535 536 537 538 539 540 541 542 543 544

	if (total)
		*total += cnt;

	if ((hmask + 1) < xfrm_policy_hashmax &&
	    cnt > hmask)
		return 1;

	return 0;
}

545
static inline int xfrm_byidx_should_resize(struct net *net, int total)
546
{
547
	unsigned int hmask = net->xfrm.policy_idx_hmask;
548 549 550 551 552 553 554 555

	if ((hmask + 1) < xfrm_policy_hashmax &&
	    total > hmask)
		return 1;

	return 0;
}

556
void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
557
{
558 559 560 561 562 563 564
	si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
	si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
	si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
	si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
	si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
	si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
	si->spdhcnt = net->xfrm.policy_idx_hmask;
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
565 566 567
	si->spdhmcnt = xfrm_policy_hashmax;
}
EXPORT_SYMBOL(xfrm_spd_getinfo);
568

Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
569
static DEFINE_MUTEX(hash_resize_mutex);
570
static void xfrm_hash_resize(struct work_struct *work)
571
{
572
	struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
573 574 575 576 577
	int dir, total;

	mutex_lock(&hash_resize_mutex);

	total = 0;
Herbert Xu's avatar
Herbert Xu committed
578
	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
579 580
		if (xfrm_bydst_should_resize(net, dir, &total))
			xfrm_bydst_resize(net, dir);
581
	}
582 583
	if (xfrm_byidx_should_resize(net, total))
		xfrm_byidx_resize(net, total);
584 585 586 587

	mutex_unlock(&hash_resize_mutex);
}

588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614
static void xfrm_hash_rebuild(struct work_struct *work)
{
	struct net *net = container_of(work, struct net,
				       xfrm.policy_hthresh.work);
	unsigned int hmask;
	struct xfrm_policy *pol;
	struct xfrm_policy *policy;
	struct hlist_head *chain;
	struct hlist_head *odst;
	struct hlist_node *newpos;
	int i;
	int dir;
	unsigned seq;
	u8 lbits4, rbits4, lbits6, rbits6;

	mutex_lock(&hash_resize_mutex);

	/* read selector prefixlen thresholds */
	do {
		seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);

		lbits4 = net->xfrm.policy_hthresh.lbits4;
		rbits4 = net->xfrm.policy_hthresh.rbits4;
		lbits6 = net->xfrm.policy_hthresh.lbits6;
		rbits6 = net->xfrm.policy_hthresh.rbits6;
	} while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));

615
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
616 617

	/* reset the bydst and inexact table in all directions */
Herbert Xu's avatar
Herbert Xu committed
618
	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640
		INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
		hmask = net->xfrm.policy_bydst[dir].hmask;
		odst = net->xfrm.policy_bydst[dir].table;
		for (i = hmask; i >= 0; i--)
			INIT_HLIST_HEAD(odst + i);
		if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
			/* dir out => dst = remote, src = local */
			net->xfrm.policy_bydst[dir].dbits4 = rbits4;
			net->xfrm.policy_bydst[dir].sbits4 = lbits4;
			net->xfrm.policy_bydst[dir].dbits6 = rbits6;
			net->xfrm.policy_bydst[dir].sbits6 = lbits6;
		} else {
			/* dir in/fwd => dst = local, src = remote */
			net->xfrm.policy_bydst[dir].dbits4 = lbits4;
			net->xfrm.policy_bydst[dir].sbits4 = rbits4;
			net->xfrm.policy_bydst[dir].dbits6 = lbits6;
			net->xfrm.policy_bydst[dir].sbits6 = rbits6;
		}
	}

	/* re-insert all policies by order of creation */
	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
641 642 643 644
		if (xfrm_policy_id2dir(policy->index) >= XFRM_POLICY_MAX) {
			/* skip socket policies */
			continue;
		}
645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660
		newpos = NULL;
		chain = policy_hash_bysel(net, &policy->selector,
					  policy->family,
					  xfrm_policy_id2dir(policy->index));
		hlist_for_each_entry(pol, chain, bydst) {
			if (policy->priority >= pol->priority)
				newpos = &pol->bydst;
			else
				break;
		}
		if (newpos)
			hlist_add_behind(&policy->bydst, newpos);
		else
			hlist_add_head(&policy->bydst, chain);
	}

661
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
662 663 664 665 666 667 668 669 670 671

	mutex_unlock(&hash_resize_mutex);
}

void xfrm_policy_hash_rebuild(struct net *net)
{
	schedule_work(&net->xfrm.policy_hthresh.work);
}
EXPORT_SYMBOL(xfrm_policy_hash_rebuild);

Linus Torvalds's avatar
Linus Torvalds committed
672 673
/* Generate new index... KAME seems to generate them ordered by cost
 * of an absolute inpredictability of ordering of rules. This will not pass. */
674
static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
Linus Torvalds's avatar
Linus Torvalds committed
675 676 677 678
{
	static u32 idx_generator;

	for (;;) {
679 680 681 682 683
		struct hlist_head *list;
		struct xfrm_policy *p;
		u32 idx;
		int found;

684 685 686 687 688 689 690 691
		if (!index) {
			idx = (idx_generator | dir);
			idx_generator += 8;
		} else {
			idx = index;
			index = 0;
		}

Linus Torvalds's avatar
Linus Torvalds committed
692 693
		if (idx == 0)
			idx = 8;
694
		list = net->xfrm.policy_byidx + idx_hash(net, idx);
695
		found = 0;
696
		hlist_for_each_entry(p, list, byidx) {
697 698
			if (p->index == idx) {
				found = 1;
Linus Torvalds's avatar
Linus Torvalds committed
699
				break;
700
			}
Linus Torvalds's avatar
Linus Torvalds committed
701
		}
702
		if (!found)
Linus Torvalds's avatar
Linus Torvalds committed
703 704 705 706
			return idx;
	}
}

707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
{
	u32 *p1 = (u32 *) s1;
	u32 *p2 = (u32 *) s2;
	int len = sizeof(struct xfrm_selector) / sizeof(u32);
	int i;

	for (i = 0; i < len; i++) {
		if (p1[i] != p2[i])
			return 1;
	}

	return 0;
}

722 723 724 725 726 727
static void xfrm_policy_requeue(struct xfrm_policy *old,
				struct xfrm_policy *new)
{
	struct xfrm_policy_queue *pq = &old->polq;
	struct sk_buff_head list;

728 729 730
	if (skb_queue_empty(&pq->hold_queue))
		return;

731 732 733 734
	__skb_queue_head_init(&list);

	spin_lock_bh(&pq->hold_queue.lock);
	skb_queue_splice_init(&pq->hold_queue, &list);
735 736
	if (del_timer(&pq->hold_timer))
		xfrm_pol_put(old);
737 738 739 740 741 742 743
	spin_unlock_bh(&pq->hold_queue.lock);

	pq = &new->polq;

	spin_lock_bh(&pq->hold_queue.lock);
	skb_queue_splice(&list, &pq->hold_queue);
	pq->timeout = XFRM_QUEUE_TMO_MIN;
744 745
	if (!mod_timer(&pq->hold_timer, jiffies))
		xfrm_pol_hold(new);
746 747 748
	spin_unlock_bh(&pq->hold_queue.lock);
}

749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
static bool xfrm_policy_mark_match(struct xfrm_policy *policy,
				   struct xfrm_policy *pol)
{
	u32 mark = policy->mark.v & policy->mark.m;

	if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)
		return true;

	if ((mark & pol->mark.m) == pol->mark.v &&
	    policy->priority == pol->priority)
		return true;

	return false;
}

Linus Torvalds's avatar
Linus Torvalds committed
764 765
int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
{
766
	struct net *net = xp_net(policy);
767 768 769
	struct xfrm_policy *pol;
	struct xfrm_policy *delpol;
	struct hlist_head *chain;
770
	struct hlist_node *newpos;
Linus Torvalds's avatar
Linus Torvalds committed
771

772
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
773
	chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
774 775
	delpol = NULL;
	newpos = NULL;
776
	hlist_for_each_entry(pol, chain, bydst) {
Herbert Xu's avatar
Herbert Xu committed
777
		if (pol->type == policy->type &&
778
		    !selector_cmp(&pol->selector, &policy->selector) &&
779
		    xfrm_policy_mark_match(policy, pol) &&
Herbert Xu's avatar
Herbert Xu committed
780 781
		    xfrm_sec_ctx_match(pol->security, policy->security) &&
		    !WARN_ON(delpol)) {
Linus Torvalds's avatar
Linus Torvalds committed
782
			if (excl) {
783
				spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
Linus Torvalds's avatar
Linus Torvalds committed
784 785 786 787 788 789
				return -EEXIST;
			}
			delpol = pol;
			if (policy->priority > pol->priority)
				continue;
		} else if (policy->priority >= pol->priority) {
Herbert Xu's avatar
Herbert Xu committed
790
			newpos = &pol->bydst;
Linus Torvalds's avatar
Linus Torvalds committed
791 792 793 794 795 796
			continue;
		}
		if (delpol)
			break;
	}
	if (newpos)
797
		hlist_add_behind(&policy->bydst, newpos);
798 799
	else
		hlist_add_head(&policy->bydst, chain);
800
	__xfrm_policy_link(policy, dir);
801
	atomic_inc(&net->xfrm.flow_cache_genid);
fan.du's avatar
fan.du committed
802 803 804 805 806 807 808

	/* After previous checking, family can either be AF_INET or AF_INET6 */
	if (policy->family == AF_INET)
		rt_genid_bump_ipv4(net);
	else
		rt_genid_bump_ipv6(net);

809 810
	if (delpol) {
		xfrm_policy_requeue(delpol, policy);
811
		__xfrm_policy_unlink(delpol, dir);
812
	}
813
	policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
814
	hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
815
	policy->curlft.add_time = get_seconds();
Linus Torvalds's avatar
Linus Torvalds committed
816 817 818
	policy->curlft.use_time = 0;
	if (!mod_timer(&policy->timer, jiffies + HZ))
		xfrm_pol_hold(policy);
819
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
Linus Torvalds's avatar
Linus Torvalds committed
820

821
	if (delpol)
Linus Torvalds's avatar
Linus Torvalds committed
822
		xfrm_policy_kill(delpol);
823 824
	else if (xfrm_bydst_should_resize(net, dir, NULL))
		schedule_work(&net->xfrm.policy_hash_work);
825

Linus Torvalds's avatar
Linus Torvalds committed
826 827 828 829
	return 0;
}
EXPORT_SYMBOL(xfrm_policy_insert);

830 831
struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u8 type,
					  int dir, struct xfrm_selector *sel,
832 833
					  struct xfrm_sec_ctx *ctx, int delete,
					  int *err)
Linus Torvalds's avatar
Linus Torvalds committed
834
{
835 836
	struct xfrm_policy *pol, *ret;
	struct hlist_head *chain;
Linus Torvalds's avatar
Linus Torvalds committed
837

838
	*err = 0;
839
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
840
	chain = policy_hash_bysel(net, sel, sel->family, dir);
841
	ret = NULL;
842
	hlist_for_each_entry(pol, chain, bydst) {
843
		if (pol->type == type &&
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
844
		    (mark & pol->mark.m) == pol->mark.v &&
845 846
		    !selector_cmp(sel, &pol->selector) &&
		    xfrm_sec_ctx_match(ctx, pol->security)) {
Linus Torvalds's avatar
Linus Torvalds committed
847
			xfrm_pol_hold(pol);
848
			if (delete) {
849 850
				*err = security_xfrm_policy_delete(
								pol->security);
851
				if (*err) {
852
					spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
853 854
					return pol;
				}
855
				__xfrm_policy_unlink(pol, dir);
856 857
			}
			ret = pol;
Linus Torvalds's avatar
Linus Torvalds committed
858 859 860
			break;
		}
	}
861
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
Linus Torvalds's avatar
Linus Torvalds committed
862

863
	if (ret && delete)
864 865
		xfrm_policy_kill(ret);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
866
}
867
EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
Linus Torvalds's avatar
Linus Torvalds committed
868

869 870
struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u8 type,
				     int dir, u32 id, int delete, int *err)
Linus Torvalds's avatar
Linus Torvalds committed
871
{
872 873
	struct xfrm_policy *pol, *ret;
	struct hlist_head *chain;
Linus Torvalds's avatar
Linus Torvalds committed
874

875 876 877 878
	*err = -ENOENT;
	if (xfrm_policy_id2dir(id) != dir)
		return NULL;

879
	*err = 0;
880
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
881
	chain = net->xfrm.policy_byidx + idx_hash(net, id);
882
	ret = NULL;
883
	hlist_for_each_entry(pol, chain, byidx) {
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
884 885
		if (pol->type == type && pol->index == id &&
		    (mark & pol->mark.m) == pol->mark.v) {
Linus Torvalds's avatar
Linus Torvalds committed
886
			xfrm_pol_hold(pol);
887
			if (delete) {
888 889
				*err = security_xfrm_policy_delete(
								pol->security);
890
				if (*err) {
891
					spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
892 893
					return pol;
				}
894
				__xfrm_policy_unlink(pol, dir);
895 896
			}
			ret = pol;
Linus Torvalds's avatar
Linus Torvalds committed
897 898 899
			break;
		}
	}
900
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
Linus Torvalds's avatar
Linus Torvalds committed
901

902
	if (ret && delete)
903 904
		xfrm_policy_kill(ret);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
905 906 907
}
EXPORT_SYMBOL(xfrm_policy_byid);

908 909
#ifdef CONFIG_SECURITY_NETWORK_XFRM
static inline int
910
xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
Linus Torvalds's avatar
Linus Torvalds committed
911
{
912 913 914 915 916 917
	int dir, err = 0;

	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
		struct xfrm_policy *pol;
		int i;

918
		hlist_for_each_entry(pol,
919
				     &net->xfrm.policy_inexact[dir], bydst) {
920 921
			if (pol->type != type)
				continue;
922
			err = security_xfrm_policy_delete(pol->security);