xfrm_policy.c 80.9 KB
Newer Older
1
/*
Linus Torvalds's avatar
Linus Torvalds committed
2
3
4
5
6
7
8
9
10
11
12
 * xfrm_policy.c
 *
 * Changes:
 *	Mitsuru KANDA @USAGI
 * 	Kazunori MIYAZAWA @USAGI
 * 	Kunihiro Ishiguro <kunihiro@ipinfusion.com>
 * 		IPv6 support
 * 	Kazunori MIYAZAWA @USAGI
 * 	YOSHIFUJI Hideaki
 * 		Split up af-specific portion
 *	Derek Atkins <derek@ihtfp.com>		Add the post_input processor
13
 *
Linus Torvalds's avatar
Linus Torvalds committed
14
15
 */

16
#include <linux/err.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
18
19
20
21
22
23
#include <linux/slab.h>
#include <linux/kmod.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/workqueue.h>
#include <linux/notifier.h>
#include <linux/netdevice.h>
24
#include <linux/netfilter.h>
Linus Torvalds's avatar
Linus Torvalds committed
25
#include <linux/module.h>
26
#include <linux/cache.h>
Paul Moore's avatar
Paul Moore committed
27
#include <linux/audit.h>
28
#include <net/dst.h>
29
#include <net/flow.h>
Linus Torvalds's avatar
Linus Torvalds committed
30
31
#include <net/xfrm.h>
#include <net/ip.h>
32
33
34
#ifdef CONFIG_XFRM_STATISTICS
#include <net/snmp.h>
#endif
Linus Torvalds's avatar
Linus Torvalds committed
35

36
37
#include "xfrm_hash.h"

38
39
40
41
#define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
#define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
#define XFRM_MAX_QUEUE_LEN	100

42
43
44
45
46
struct xfrm_flo {
	struct dst_entry *dst_orig;
	u8 flags;
};

47
static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
48
static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
49
						__read_mostly;
Linus Torvalds's avatar
Linus Torvalds committed
50

51
static struct kmem_cache *xfrm_dst_cache __read_mostly;
52
static __read_mostly seqcount_t xfrm_policy_hash_generation;
Linus Torvalds's avatar
Linus Torvalds committed
53

54
static void xfrm_init_pmtu(struct dst_entry *dst);
55
static int stale_bundle(struct dst_entry *dst);
56
static int xfrm_bundle_ok(struct xfrm_dst *xdst);
57
static void xfrm_policy_queue_process(unsigned long arg);
Linus Torvalds's avatar
Linus Torvalds committed
58

59
static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
60
61
62
static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
						int dir);

63
64
65
66
67
static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
{
	return atomic_inc_not_zero(&policy->refcnt);
}

68
static inline bool
69
__xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
70
{
71
72
	const struct flowi4 *fl4 = &fl->u.ip4;

73
74
	return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
		addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
75
76
77
78
		!((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
		!((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
		(fl4->flowi4_proto == sel->proto || !sel->proto) &&
		(fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
79
80
}

81
static inline bool
82
__xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
83
{
84
85
86
87
88
89
90
91
	const struct flowi6 *fl6 = &fl->u.ip6;

	return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
		addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
		!((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
		!((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
		(fl6->flowi6_proto == sel->proto || !sel->proto) &&
		(fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
92
93
}

94
95
bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
			 unsigned short family)
96
97
98
99
100
101
102
{
	switch (family) {
	case AF_INET:
		return __xfrm4_selector_match(sel, fl);
	case AF_INET6:
		return __xfrm6_selector_match(sel, fl);
	}
103
	return false;
104
105
}

106
static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
Eric Dumazet's avatar
Eric Dumazet committed
107
{
108
	const struct xfrm_policy_afinfo *afinfo;
Eric Dumazet's avatar
Eric Dumazet committed
109

110
	if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
Eric Dumazet's avatar
Eric Dumazet committed
111
112
113
114
115
116
117
118
		return NULL;
	rcu_read_lock();
	afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
	if (unlikely(!afinfo))
		rcu_read_unlock();
	return afinfo;
}

David Ahern's avatar
David Ahern committed
119
120
static inline struct dst_entry *__xfrm_dst_lookup(struct net *net,
						  int tos, int oif,
121
122
						  const xfrm_address_t *saddr,
						  const xfrm_address_t *daddr,
123
124
						  int family)
{
125
	const struct xfrm_policy_afinfo *afinfo;
126
127
128
129
130
131
	struct dst_entry *dst;

	afinfo = xfrm_policy_get_afinfo(family);
	if (unlikely(afinfo == NULL))
		return ERR_PTR(-EAFNOSUPPORT);

David Ahern's avatar
David Ahern committed
132
	dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr);
133

134
	rcu_read_unlock();
135
136
137
138

	return dst;
}

David Ahern's avatar
David Ahern committed
139
140
static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
						int tos, int oif,
141
142
						xfrm_address_t *prev_saddr,
						xfrm_address_t *prev_daddr,
143
						int family)
Linus Torvalds's avatar
Linus Torvalds committed
144
{
145
	struct net *net = xs_net(x);
146
147
148
149
	xfrm_address_t *saddr = &x->props.saddr;
	xfrm_address_t *daddr = &x->id.daddr;
	struct dst_entry *dst;

150
	if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
151
		saddr = x->coaddr;
152
153
154
155
		daddr = prev_daddr;
	}
	if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
		saddr = prev_saddr;
156
		daddr = x->coaddr;
157
	}
Linus Torvalds's avatar
Linus Torvalds committed
158

David Ahern's avatar
David Ahern committed
159
	dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family);
160
161
162
163
164
165
166

	if (!IS_ERR(dst)) {
		if (prev_saddr != saddr)
			memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
		if (prev_daddr != daddr)
			memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
	}
Linus Torvalds's avatar
Linus Torvalds committed
167

168
	return dst;
Linus Torvalds's avatar
Linus Torvalds committed
169
170
171
172
173
174
175
}

static inline unsigned long make_jiffies(long secs)
{
	if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
		return MAX_SCHEDULE_TIMEOUT-1;
	else
176
		return secs*HZ;
Linus Torvalds's avatar
Linus Torvalds committed
177
178
179
180
}

static void xfrm_policy_timer(unsigned long data)
{
181
	struct xfrm_policy *xp = (struct xfrm_policy *)data;
182
	unsigned long now = get_seconds();
Linus Torvalds's avatar
Linus Torvalds committed
183
184
185
186
187
188
	long next = LONG_MAX;
	int warn = 0;
	int dir;

	read_lock(&xp->lock);

189
	if (unlikely(xp->walk.dead))
Linus Torvalds's avatar
Linus Torvalds committed
190
191
		goto out;

192
	dir = xfrm_policy_id2dir(xp->index);
Linus Torvalds's avatar
Linus Torvalds committed
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231

	if (xp->lft.hard_add_expires_seconds) {
		long tmo = xp->lft.hard_add_expires_seconds +
			xp->curlft.add_time - now;
		if (tmo <= 0)
			goto expired;
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.hard_use_expires_seconds) {
		long tmo = xp->lft.hard_use_expires_seconds +
			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
		if (tmo <= 0)
			goto expired;
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.soft_add_expires_seconds) {
		long tmo = xp->lft.soft_add_expires_seconds +
			xp->curlft.add_time - now;
		if (tmo <= 0) {
			warn = 1;
			tmo = XFRM_KM_TIMEOUT;
		}
		if (tmo < next)
			next = tmo;
	}
	if (xp->lft.soft_use_expires_seconds) {
		long tmo = xp->lft.soft_use_expires_seconds +
			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
		if (tmo <= 0) {
			warn = 1;
			tmo = XFRM_KM_TIMEOUT;
		}
		if (tmo < next)
			next = tmo;
	}

	if (warn)
232
		km_policy_expired(xp, dir, 0, 0);
Linus Torvalds's avatar
Linus Torvalds committed
233
234
235
236
237
238
239
240
241
242
243
	if (next != LONG_MAX &&
	    !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
		xfrm_pol_hold(xp);

out:
	read_unlock(&xp->lock);
	xfrm_pol_put(xp);
	return;

expired:
	read_unlock(&xp->lock);
244
	if (!xfrm_policy_delete(xp, dir))
245
		km_policy_expired(xp, dir, 1, 0);
Linus Torvalds's avatar
Linus Torvalds committed
246
247
248
	xfrm_pol_put(xp);
}

249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo)
{
	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);

	if (unlikely(pol->walk.dead))
		flo = NULL;
	else
		xfrm_pol_hold(pol);

	return flo;
}

static int xfrm_policy_flo_check(struct flow_cache_object *flo)
{
	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);

	return !pol->walk.dead;
}

static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
{
	xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
}

static const struct flow_cache_ops xfrm_policy_fc_ops = {
	.get = xfrm_policy_flo_get,
	.check = xfrm_policy_flo_check,
	.delete = xfrm_policy_flo_delete,
};
Linus Torvalds's avatar
Linus Torvalds committed
278
279
280
281
282

/* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
 * SPD calls.
 */

283
struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
Linus Torvalds's avatar
Linus Torvalds committed
284
285
286
{
	struct xfrm_policy *policy;

287
	policy = kzalloc(sizeof(struct xfrm_policy), gfp);
Linus Torvalds's avatar
Linus Torvalds committed
288
289

	if (policy) {
290
		write_pnet(&policy->xp_net, net);
291
		INIT_LIST_HEAD(&policy->walk.all);
292
293
		INIT_HLIST_NODE(&policy->bydst);
		INIT_HLIST_NODE(&policy->byidx);
Linus Torvalds's avatar
Linus Torvalds committed
294
		rwlock_init(&policy->lock);
295
		atomic_set(&policy->refcnt, 1);
296
		skb_queue_head_init(&policy->polq.hold_queue);
297
298
		setup_timer(&policy->timer, xfrm_policy_timer,
				(unsigned long)policy);
299
300
		setup_timer(&policy->polq.hold_timer, xfrm_policy_queue_process,
			    (unsigned long)policy);
301
		policy->flo.ops = &xfrm_policy_fc_ops;
Linus Torvalds's avatar
Linus Torvalds committed
302
303
304
305
306
	}
	return policy;
}
EXPORT_SYMBOL(xfrm_policy_alloc);

307
308
309
310
311
312
313
314
static void xfrm_policy_destroy_rcu(struct rcu_head *head)
{
	struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);

	security_xfrm_policy_free(policy->security);
	kfree(policy);
}

Linus Torvalds's avatar
Linus Torvalds committed
315
316
/* Destroy xfrm_policy: descendant resources must be released to this moment. */

317
void xfrm_policy_destroy(struct xfrm_policy *policy)
Linus Torvalds's avatar
Linus Torvalds committed
318
{
319
	BUG_ON(!policy->walk.dead);
Linus Torvalds's avatar
Linus Torvalds committed
320

321
	if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
Linus Torvalds's avatar
Linus Torvalds committed
322
323
		BUG();

324
	call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
Linus Torvalds's avatar
Linus Torvalds committed
325
}
326
EXPORT_SYMBOL(xfrm_policy_destroy);
Linus Torvalds's avatar
Linus Torvalds committed
327

Alexander Alemayhu's avatar
Alexander Alemayhu committed
328
/* Rule must be locked. Release descendant resources, announce
Linus Torvalds's avatar
Linus Torvalds committed
329
330
331
332
333
 * entry dead. The rule must be unlinked from lists to the moment.
 */

static void xfrm_policy_kill(struct xfrm_policy *policy)
{
334
	policy->walk.dead = 1;
Linus Torvalds's avatar
Linus Torvalds committed
335

336
	atomic_inc(&policy->genid);
Linus Torvalds's avatar
Linus Torvalds committed
337

338
339
	if (del_timer(&policy->polq.hold_timer))
		xfrm_pol_put(policy);
340
	skb_queue_purge(&policy->polq.hold_queue);
341

342
343
344
345
	if (del_timer(&policy->timer))
		xfrm_pol_put(policy);

	xfrm_pol_put(policy);
Linus Torvalds's avatar
Linus Torvalds committed
346
347
}

348
349
static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;

350
static inline unsigned int idx_hash(struct net *net, u32 index)
351
{
352
	return __idx_hash(index, net->xfrm.policy_idx_hmask);
353
354
}

355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
/* calculate policy hash thresholds */
static void __get_hash_thresh(struct net *net,
			      unsigned short family, int dir,
			      u8 *dbits, u8 *sbits)
{
	switch (family) {
	case AF_INET:
		*dbits = net->xfrm.policy_bydst[dir].dbits4;
		*sbits = net->xfrm.policy_bydst[dir].sbits4;
		break;

	case AF_INET6:
		*dbits = net->xfrm.policy_bydst[dir].dbits6;
		*sbits = net->xfrm.policy_bydst[dir].sbits6;
		break;

	default:
		*dbits = 0;
		*sbits = 0;
	}
}

377
378
379
static struct hlist_head *policy_hash_bysel(struct net *net,
					    const struct xfrm_selector *sel,
					    unsigned short family, int dir)
380
{
381
	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
382
383
384
385
386
387
	unsigned int hash;
	u8 dbits;
	u8 sbits;

	__get_hash_thresh(net, family, dir, &dbits, &sbits);
	hash = __sel_hash(sel, family, hmask, dbits, sbits);
388

389
390
391
392
393
	if (hash == hmask + 1)
		return &net->xfrm.policy_inexact[dir];

	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
394
395
}

396
397
398
399
static struct hlist_head *policy_hash_direct(struct net *net,
					     const xfrm_address_t *daddr,
					     const xfrm_address_t *saddr,
					     unsigned short family, int dir)
400
{
401
	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
402
403
404
405
406
407
	unsigned int hash;
	u8 dbits;
	u8 sbits;

	__get_hash_thresh(net, family, dir, &dbits, &sbits);
	hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
408

409
410
	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
411
412
}

413
414
static void xfrm_dst_hash_transfer(struct net *net,
				   struct hlist_head *list,
415
				   struct hlist_head *ndsttable,
416
417
				   unsigned int nhashmask,
				   int dir)
418
{
419
	struct hlist_node *tmp, *entry0 = NULL;
420
	struct xfrm_policy *pol;
421
	unsigned int h0 = 0;
422
423
	u8 dbits;
	u8 sbits;
424

425
redo:
426
	hlist_for_each_entry_safe(pol, tmp, list, bydst) {
427
428
		unsigned int h;

429
		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
430
		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
431
				pol->family, nhashmask, dbits, sbits);
432
		if (!entry0) {
433
434
			hlist_del_rcu(&pol->bydst);
			hlist_add_head_rcu(&pol->bydst, ndsttable + h);
435
436
437
438
			h0 = h;
		} else {
			if (h != h0)
				continue;
439
440
			hlist_del_rcu(&pol->bydst);
			hlist_add_behind_rcu(&pol->bydst, entry0);
441
		}
442
		entry0 = &pol->bydst;
443
444
445
446
	}
	if (!hlist_empty(list)) {
		entry0 = NULL;
		goto redo;
447
448
449
450
451
452
453
	}
}

static void xfrm_idx_hash_transfer(struct hlist_head *list,
				   struct hlist_head *nidxtable,
				   unsigned int nhashmask)
{
454
	struct hlist_node *tmp;
455
456
	struct xfrm_policy *pol;

457
	hlist_for_each_entry_safe(pol, tmp, list, byidx) {
458
459
460
461
462
463
464
465
466
467
468
469
		unsigned int h;

		h = __idx_hash(pol->index, nhashmask);
		hlist_add_head(&pol->byidx, nidxtable+h);
	}
}

static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
{
	return ((old_hmask + 1) << 1) - 1;
}

470
static void xfrm_bydst_resize(struct net *net, int dir)
471
{
472
	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
473
474
	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
475
	struct hlist_head *ndst = xfrm_hash_alloc(nsize);
476
	struct hlist_head *odst;
477
478
479
480
481
	int i;

	if (!ndst)
		return;

482
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
483
484
485
486
	write_seqcount_begin(&xfrm_policy_hash_generation);

	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
				lockdep_is_held(&net->xfrm.xfrm_policy_lock));
487

488
489
490
	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
				lockdep_is_held(&net->xfrm.xfrm_policy_lock));

491
	for (i = hmask; i >= 0; i--)
492
		xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
493

494
	rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
495
	net->xfrm.policy_bydst[dir].hmask = nhashmask;
496

497
	write_seqcount_end(&xfrm_policy_hash_generation);
498
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
499

500
501
	synchronize_rcu();

502
	xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
503
504
}

505
static void xfrm_byidx_resize(struct net *net, int total)
506
{
507
	unsigned int hmask = net->xfrm.policy_idx_hmask;
508
509
	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
510
	struct hlist_head *oidx = net->xfrm.policy_byidx;
511
	struct hlist_head *nidx = xfrm_hash_alloc(nsize);
512
513
514
515
516
	int i;

	if (!nidx)
		return;

517
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
518
519
520
521

	for (i = hmask; i >= 0; i--)
		xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);

522
523
	net->xfrm.policy_byidx = nidx;
	net->xfrm.policy_idx_hmask = nhashmask;
524

525
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
526

527
	xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
528
529
}

530
static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
531
{
532
533
	unsigned int cnt = net->xfrm.policy_count[dir];
	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
534
535
536
537
538
539
540
541
542
543
544

	if (total)
		*total += cnt;

	if ((hmask + 1) < xfrm_policy_hashmax &&
	    cnt > hmask)
		return 1;

	return 0;
}

545
static inline int xfrm_byidx_should_resize(struct net *net, int total)
546
{
547
	unsigned int hmask = net->xfrm.policy_idx_hmask;
548
549
550
551
552
553
554
555

	if ((hmask + 1) < xfrm_policy_hashmax &&
	    total > hmask)
		return 1;

	return 0;
}

556
void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
557
{
558
559
560
561
562
563
564
	si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
	si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
	si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
	si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
	si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
	si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
	si->spdhcnt = net->xfrm.policy_idx_hmask;
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
565
566
567
	si->spdhmcnt = xfrm_policy_hashmax;
}
EXPORT_SYMBOL(xfrm_spd_getinfo);
568

Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
569
static DEFINE_MUTEX(hash_resize_mutex);
570
static void xfrm_hash_resize(struct work_struct *work)
571
{
572
	struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
573
574
575
576
577
	int dir, total;

	mutex_lock(&hash_resize_mutex);

	total = 0;
Herbert Xu's avatar
Herbert Xu committed
578
	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
579
580
		if (xfrm_bydst_should_resize(net, dir, &total))
			xfrm_bydst_resize(net, dir);
581
	}
582
583
	if (xfrm_byidx_should_resize(net, total))
		xfrm_byidx_resize(net, total);
584
585
586
587

	mutex_unlock(&hash_resize_mutex);
}

588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
static void xfrm_hash_rebuild(struct work_struct *work)
{
	struct net *net = container_of(work, struct net,
				       xfrm.policy_hthresh.work);
	unsigned int hmask;
	struct xfrm_policy *pol;
	struct xfrm_policy *policy;
	struct hlist_head *chain;
	struct hlist_head *odst;
	struct hlist_node *newpos;
	int i;
	int dir;
	unsigned seq;
	u8 lbits4, rbits4, lbits6, rbits6;

	mutex_lock(&hash_resize_mutex);

	/* read selector prefixlen thresholds */
	do {
		seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);

		lbits4 = net->xfrm.policy_hthresh.lbits4;
		rbits4 = net->xfrm.policy_hthresh.rbits4;
		lbits6 = net->xfrm.policy_hthresh.lbits6;
		rbits6 = net->xfrm.policy_hthresh.rbits6;
	} while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));

615
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
616
617

	/* reset the bydst and inexact table in all directions */
Herbert Xu's avatar
Herbert Xu committed
618
	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
		INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
		hmask = net->xfrm.policy_bydst[dir].hmask;
		odst = net->xfrm.policy_bydst[dir].table;
		for (i = hmask; i >= 0; i--)
			INIT_HLIST_HEAD(odst + i);
		if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
			/* dir out => dst = remote, src = local */
			net->xfrm.policy_bydst[dir].dbits4 = rbits4;
			net->xfrm.policy_bydst[dir].sbits4 = lbits4;
			net->xfrm.policy_bydst[dir].dbits6 = rbits6;
			net->xfrm.policy_bydst[dir].sbits6 = lbits6;
		} else {
			/* dir in/fwd => dst = local, src = remote */
			net->xfrm.policy_bydst[dir].dbits4 = lbits4;
			net->xfrm.policy_bydst[dir].sbits4 = rbits4;
			net->xfrm.policy_bydst[dir].dbits6 = lbits6;
			net->xfrm.policy_bydst[dir].sbits6 = rbits6;
		}
	}

	/* re-insert all policies by order of creation */
	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
641
642
643
644
		if (xfrm_policy_id2dir(policy->index) >= XFRM_POLICY_MAX) {
			/* skip socket policies */
			continue;
		}
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
		newpos = NULL;
		chain = policy_hash_bysel(net, &policy->selector,
					  policy->family,
					  xfrm_policy_id2dir(policy->index));
		hlist_for_each_entry(pol, chain, bydst) {
			if (policy->priority >= pol->priority)
				newpos = &pol->bydst;
			else
				break;
		}
		if (newpos)
			hlist_add_behind(&policy->bydst, newpos);
		else
			hlist_add_head(&policy->bydst, chain);
	}

661
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
662
663
664
665
666
667
668
669
670
671

	mutex_unlock(&hash_resize_mutex);
}

void xfrm_policy_hash_rebuild(struct net *net)
{
	schedule_work(&net->xfrm.policy_hthresh.work);
}
EXPORT_SYMBOL(xfrm_policy_hash_rebuild);

Linus Torvalds's avatar
Linus Torvalds committed
672
673
/* Generate new index... KAME seems to generate them ordered by cost
 * of an absolute inpredictability of ordering of rules. This will not pass. */
674
static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
Linus Torvalds's avatar
Linus Torvalds committed
675
676
677
678
{
	static u32 idx_generator;

	for (;;) {
679
680
681
682
683
		struct hlist_head *list;
		struct xfrm_policy *p;
		u32 idx;
		int found;

684
685
686
687
688
689
690
691
		if (!index) {
			idx = (idx_generator | dir);
			idx_generator += 8;
		} else {
			idx = index;
			index = 0;
		}

Linus Torvalds's avatar
Linus Torvalds committed
692
693
		if (idx == 0)
			idx = 8;
694
		list = net->xfrm.policy_byidx + idx_hash(net, idx);
695
		found = 0;
696
		hlist_for_each_entry(p, list, byidx) {
697
698
			if (p->index == idx) {
				found = 1;
Linus Torvalds's avatar
Linus Torvalds committed
699
				break;
700
			}
Linus Torvalds's avatar
Linus Torvalds committed
701
		}
702
		if (!found)
Linus Torvalds's avatar
Linus Torvalds committed
703
704
705
706
			return idx;
	}
}

707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
{
	u32 *p1 = (u32 *) s1;
	u32 *p2 = (u32 *) s2;
	int len = sizeof(struct xfrm_selector) / sizeof(u32);
	int i;

	for (i = 0; i < len; i++) {
		if (p1[i] != p2[i])
			return 1;
	}

	return 0;
}

722
723
724
725
726
727
static void xfrm_policy_requeue(struct xfrm_policy *old,
				struct xfrm_policy *new)
{
	struct xfrm_policy_queue *pq = &old->polq;
	struct sk_buff_head list;

728
729
730
	if (skb_queue_empty(&pq->hold_queue))
		return;

731
732
733
734
	__skb_queue_head_init(&list);

	spin_lock_bh(&pq->hold_queue.lock);
	skb_queue_splice_init(&pq->hold_queue, &list);
735
736
	if (del_timer(&pq->hold_timer))
		xfrm_pol_put(old);
737
738
739
740
741
742
743
	spin_unlock_bh(&pq->hold_queue.lock);

	pq = &new->polq;

	spin_lock_bh(&pq->hold_queue.lock);
	skb_queue_splice(&list, &pq->hold_queue);
	pq->timeout = XFRM_QUEUE_TMO_MIN;
744
745
	if (!mod_timer(&pq->hold_timer, jiffies))
		xfrm_pol_hold(new);
746
747
748
	spin_unlock_bh(&pq->hold_queue.lock);
}

749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
static bool xfrm_policy_mark_match(struct xfrm_policy *policy,
				   struct xfrm_policy *pol)
{
	u32 mark = policy->mark.v & policy->mark.m;

	if (policy->mark.v == pol->mark.v && policy->mark.m == pol->mark.m)
		return true;

	if ((mark & pol->mark.m) == pol->mark.v &&
	    policy->priority == pol->priority)
		return true;

	return false;
}

Linus Torvalds's avatar
Linus Torvalds committed
764
765
int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
{
766
	struct net *net = xp_net(policy);
767
768
769
	struct xfrm_policy *pol;
	struct xfrm_policy *delpol;
	struct hlist_head *chain;
770
	struct hlist_node *newpos;
Linus Torvalds's avatar
Linus Torvalds committed
771

772
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
773
	chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
774
775
	delpol = NULL;
	newpos = NULL;
776
	hlist_for_each_entry(pol, chain, bydst) {
Herbert Xu's avatar
Herbert Xu committed
777
		if (pol->type == policy->type &&
778
		    !selector_cmp(&pol->selector, &policy->selector) &&
779
		    xfrm_policy_mark_match(policy, pol) &&
Herbert Xu's avatar
Herbert Xu committed
780
781
		    xfrm_sec_ctx_match(pol->security, policy->security) &&
		    !WARN_ON(delpol)) {
Linus Torvalds's avatar
Linus Torvalds committed
782
			if (excl) {
783
				spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
Linus Torvalds's avatar
Linus Torvalds committed
784
785
786
787
788
789
				return -EEXIST;
			}
			delpol = pol;
			if (policy->priority > pol->priority)
				continue;
		} else if (policy->priority >= pol->priority) {
Herbert Xu's avatar
Herbert Xu committed
790
			newpos = &pol->bydst;
Linus Torvalds's avatar
Linus Torvalds committed
791
792
793
794
795
796
			continue;
		}
		if (delpol)
			break;
	}
	if (newpos)
797
		hlist_add_behind(&policy->bydst, newpos);
798
799
	else
		hlist_add_head(&policy->bydst, chain);
800
	__xfrm_policy_link(policy, dir);
801
	atomic_inc(&net->xfrm.flow_cache_genid);
fan.du's avatar
fan.du committed
802
803
804
805
806
807
808

	/* After previous checking, family can either be AF_INET or AF_INET6 */
	if (policy->family == AF_INET)
		rt_genid_bump_ipv4(net);
	else
		rt_genid_bump_ipv6(net);

809
810
	if (delpol) {
		xfrm_policy_requeue(delpol, policy);
811
		__xfrm_policy_unlink(delpol, dir);
812
	}
813
	policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
814
	hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
815
	policy->curlft.add_time = get_seconds();
Linus Torvalds's avatar
Linus Torvalds committed
816
817
818
	policy->curlft.use_time = 0;
	if (!mod_timer(&policy->timer, jiffies + HZ))
		xfrm_pol_hold(policy);
819
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
Linus Torvalds's avatar
Linus Torvalds committed
820

821
	if (delpol)
Linus Torvalds's avatar
Linus Torvalds committed
822
		xfrm_policy_kill(delpol);
823
824
	else if (xfrm_bydst_should_resize(net, dir, NULL))
		schedule_work(&net->xfrm.policy_hash_work);
825

Linus Torvalds's avatar
Linus Torvalds committed
826
827
828
829
	return 0;
}
EXPORT_SYMBOL(xfrm_policy_insert);

830
831
struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u8 type,
					  int dir, struct xfrm_selector *sel,
832
833
					  struct xfrm_sec_ctx *ctx, int delete,
					  int *err)
Linus Torvalds's avatar
Linus Torvalds committed
834
{
835
836
	struct xfrm_policy *pol, *ret;
	struct hlist_head *chain;
Linus Torvalds's avatar
Linus Torvalds committed
837

838
	*err = 0;
839
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
840
	chain = policy_hash_bysel(net, sel, sel->family, dir);
841
	ret = NULL;
842
	hlist_for_each_entry(pol, chain, bydst) {
843
		if (pol->type == type &&
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
844
		    (mark & pol->mark.m) == pol->mark.v &&
845
846
		    !selector_cmp(sel, &pol->selector) &&
		    xfrm_sec_ctx_match(ctx, pol->security)) {
Linus Torvalds's avatar
Linus Torvalds committed
847
			xfrm_pol_hold(pol);
848
			if (delete) {
849
850
				*err = security_xfrm_policy_delete(
								pol->security);
851
				if (*err) {
852
					spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
853
854
					return pol;
				}
855
				__xfrm_policy_unlink(pol, dir);
856
857
			}
			ret = pol;
Linus Torvalds's avatar
Linus Torvalds committed
858
859
860
			break;
		}
	}
861
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
Linus Torvalds's avatar
Linus Torvalds committed
862

863
	if (ret && delete)
864
865
		xfrm_policy_kill(ret);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
866
}
867
EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
Linus Torvalds's avatar
Linus Torvalds committed
868

869
870
struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u8 type,
				     int dir, u32 id, int delete, int *err)
Linus Torvalds's avatar
Linus Torvalds committed
871
{
872
873
	struct xfrm_policy *pol, *ret;
	struct hlist_head *chain;
Linus Torvalds's avatar
Linus Torvalds committed
874

875
876
877
878
	*err = -ENOENT;
	if (xfrm_policy_id2dir(id) != dir)
		return NULL;

879
	*err = 0;
880
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
881
	chain = net->xfrm.policy_byidx + idx_hash(net, id);
882
	ret = NULL;
883
	hlist_for_each_entry(pol, chain, byidx) {
Jamal Hadi Salim's avatar
Jamal Hadi Salim committed
884
885
		if (pol->type == type && pol->index == id &&
		    (mark & pol->mark.m) == pol->mark.v) {
Linus Torvalds's avatar
Linus Torvalds committed
886
			xfrm_pol_hold(pol);
887
			if (delete) {
888
889
				*err = security_xfrm_policy_delete(
								pol->security);
890
				if (*err) {
891
					spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
892
893
					return pol;
				}
894
				__xfrm_policy_unlink(pol, dir);
895
896
			}
			ret = pol;
Linus Torvalds's avatar
Linus Torvalds committed
897
898
899
			break;
		}
	}
900
	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
Linus Torvalds's avatar
Linus Torvalds committed
901

902
	if (ret && delete)
903
904
		xfrm_policy_kill(ret);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
905
906
907
}
EXPORT_SYMBOL(xfrm_policy_byid);

908
909
#ifdef CONFIG_SECURITY_NETWORK_XFRM
static inline int
910
xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
Linus Torvalds's avatar
Linus Torvalds committed
911
{
912
913
914
915
916
917
	int dir, err = 0;

	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
		struct xfrm_policy *pol;
		int i;

918
		hlist_for_each_entry(pol,
919
				     &net->xfrm.policy_inexact[dir], bydst) {
920
921
			if (pol->type != type)
				continue;
922
			err = security_xfrm_policy_delete(pol->security);
923
			if (err) {
924
				xfrm_audit_policy_delete(pol, 0, task_valid);
925
926
				return err;
			}
927
		}
928
		for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
929
			hlist_for_each_entry(pol,
930
					     net->xfrm.policy_bydst[dir].table + i,
931
932
933
					     bydst) {
				if (pol->type != type)
					continue;
934
935
				err = security_xfrm_policy_delete(
								pol->security);
936
				if (err) {
Joy Latten's avatar
Joy Latten committed
937
					xfrm_audit_policy_delete(pol, 0,
938
								 task_valid);
939
940
941
942
943
944
945
946
947
					return err;
				}
			}
		}
	}
	return err;
}
#else
static inline int
948
xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
949
950
951
952
953
{
	return 0;
}
#endif

954
int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
955
{
956
	int dir, err = 0, cnt = 0;
Linus Torvalds's avatar
Linus Torvalds committed
957

958
	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
959

960
	err = xfrm_policy_flush_secctx_check(net, type, task_valid);
961
962
963
	if (err)
		goto out;

Linus Torvalds's avatar
Linus Torvalds committed
964
	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
965
		struct xfrm_policy *pol;
966
		int i;
967
968

	again1:
969
		hlist_for_each_entry(pol,
970
				     &net->xfrm.policy_inexact[dir], bydst) {
971
972
			if (pol->type != type)
				continue;
973
			__xfrm_policy_unlink(pol, dir);
974
			spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
975
			cnt++;
Linus Torvalds's avatar
Linus Torvalds committed
976

977
			xfrm_audit_policy_delete(pol, 1, task_valid);
Joy Latten's avatar
Joy Latten committed
978

979
			xfrm_policy_kill(pol);
Linus Torvalds's avatar
Linus Torvalds committed
980

981
			spin_lock_bh(&net->xfrm.xfrm_policy_lock);
982
983
984
			goto again1;
		}

985
		for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
986
	again2:
987
			hlist_for_each_entry(pol,
988
					     net->xfrm.policy_bydst[dir].table + i,
989
990
991
					     bydst) {
				if (pol->type != type)
					continue;
992
				__xfrm_policy_unlink(pol, dir);
993
				spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
994
				cnt++;
995

996
				xfrm_audit_policy_delete(pol, 1, task_valid);
997
998
				xfrm_policy_kill(pol);

999
				spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1000
				goto again2;