summaryrefslogtreecommitdiff
path: root/net/xfrm
diff options
context:
space:
mode:
Diffstat (limited to 'net/xfrm')
-rw-r--r--net/xfrm/espintcp.c4
-rw-r--r--net/xfrm/xfrm_algo.c7
-rw-r--r--net/xfrm/xfrm_device.c64
-rw-r--r--net/xfrm/xfrm_interface_core.c49
-rw-r--r--net/xfrm/xfrm_ipcomp.c432
-rw-r--r--net/xfrm/xfrm_iptfs.c6
-rw-r--r--net/xfrm/xfrm_nat_keepalive.c30
-rw-r--r--net/xfrm/xfrm_output.c49
-rw-r--r--net/xfrm/xfrm_policy.c19
-rw-r--r--net/xfrm/xfrm_state.c112
-rw-r--r--net/xfrm/xfrm_user.c71
11 files changed, 479 insertions, 364 deletions
diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c
index fe82e2d07300..fc7a603b04f1 100644
--- a/net/xfrm/espintcp.c
+++ b/net/xfrm/espintcp.c
@@ -171,8 +171,10 @@ int espintcp_queue_out(struct sock *sk, struct sk_buff *skb)
struct espintcp_ctx *ctx = espintcp_getctx(sk);
if (skb_queue_len(&ctx->out_queue) >=
- READ_ONCE(net_hotdata.max_backlog))
+ READ_ONCE(net_hotdata.max_backlog)) {
+ kfree_skb(skb);
return -ENOBUFS;
+ }
__skb_queue_tail(&ctx->out_queue, skb);
diff --git a/net/xfrm/xfrm_algo.c b/net/xfrm/xfrm_algo.c
index e6da7e8495c9..749011e031c0 100644
--- a/net/xfrm/xfrm_algo.c
+++ b/net/xfrm/xfrm_algo.c
@@ -5,13 +5,13 @@
* Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
*/
+#include <crypto/acompress.h>
#include <crypto/aead.h>
#include <crypto/hash.h>
#include <crypto/skcipher.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pfkeyv2.h>
-#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <net/xfrm.h>
#if IS_ENABLED(CONFIG_INET_ESP) || IS_ENABLED(CONFIG_INET6_ESP)
@@ -669,7 +669,7 @@ static const struct xfrm_algo_list xfrm_ealg_list = {
};
static const struct xfrm_algo_list xfrm_calg_list = {
- .find = crypto_has_comp,
+ .find = crypto_has_acomp,
.algs = calg_list,
.entries = ARRAY_SIZE(calg_list),
};
@@ -828,8 +828,7 @@ void xfrm_probe_algs(void)
}
for (i = 0; i < calg_entries(); i++) {
- status = crypto_has_comp(calg_list[i].name, 0,
- CRYPTO_ALG_ASYNC);
+ status = crypto_has_acomp(calg_list[i].name, 0, 0);
if (calg_list[i].available != status)
calg_list[i].available = status;
}
diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c
index d1fa94e52cea..81fd486b5e56 100644
--- a/net/xfrm/xfrm_device.c
+++ b/net/xfrm/xfrm_device.c
@@ -145,10 +145,6 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur
return NULL;
}
- /* This skb was already validated on the upper/virtual dev */
- if ((x->xso.dev != dev) && (x->xso.real_dev == dev))
- return skb;
-
local_irq_save(flags);
sd = this_cpu_ptr(&softnet_data);
err = !skb_queue_empty(&sd->xfrm_backlog);
@@ -159,8 +155,7 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur
return skb;
}
- if (skb_is_gso(skb) && (unlikely(x->xso.dev != dev) ||
- unlikely(xmit_xfrm_check_overflow(skb)))) {
+ if (skb_is_gso(skb) && unlikely(xmit_xfrm_check_overflow(skb))) {
struct sk_buff *segs;
/* Packet got rerouted, fixup features and segment it. */
@@ -244,11 +239,6 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
xfrm_address_t *daddr;
bool is_packet_offload;
- if (!x->type_offload) {
- NL_SET_ERR_MSG(extack, "Type doesn't support offload");
- return -EINVAL;
- }
-
if (xuo->flags &
~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND | XFRM_OFFLOAD_PACKET)) {
NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
@@ -261,6 +251,11 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
return -EINVAL;
}
+ if (xuo->flags & XFRM_OFFLOAD_INBOUND && x->if_id) {
+ NL_SET_ERR_MSG(extack, "XFRM if_id is not supported in RX path");
+ return -EINVAL;
+ }
+
is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET;
/* We don't yet support TFC padding. */
@@ -310,9 +305,15 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
return -EINVAL;
}
+ xfrm_set_type_offload(x);
+ if (!x->type_offload) {
+ NL_SET_ERR_MSG(extack, "Type doesn't support offload");
+ dev_put(dev);
+ return -EINVAL;
+ }
+
xso->dev = dev;
netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);
- xso->real_dev = dev;
if (xuo->flags & XFRM_OFFLOAD_INBOUND)
xso->dir = XFRM_DEV_OFFLOAD_IN;
@@ -324,14 +325,14 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
else
xso->type = XFRM_DEV_OFFLOAD_CRYPTO;
- err = dev->xfrmdev_ops->xdo_dev_state_add(x, extack);
+ err = dev->xfrmdev_ops->xdo_dev_state_add(dev, x, extack);
if (err) {
xso->dev = NULL;
xso->dir = 0;
- xso->real_dev = NULL;
netdev_put(dev, &xso->dev_tracker);
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
+ xfrm_unset_type_offload(x);
/* User explicitly requested packet offload mode and configured
* policy in addition to the XFRM state. So be civil to users,
* and return an error instead of taking fallback path.
@@ -375,7 +376,6 @@ int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
xdo->dev = dev;
netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC);
- xdo->real_dev = dev;
xdo->type = XFRM_DEV_OFFLOAD_PACKET;
switch (dir) {
case XFRM_POLICY_IN:
@@ -397,7 +397,6 @@ int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
err = dev->xfrmdev_ops->xdo_dev_policy_add(xp, extack);
if (err) {
xdo->dev = NULL;
- xdo->real_dev = NULL;
xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
xdo->dir = 0;
netdev_put(dev, &xdo->dev_tracker);
@@ -415,14 +414,12 @@ bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
struct dst_entry *dst = skb_dst(skb);
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
struct net_device *dev = x->xso.dev;
+ bool check_tunnel_size;
- if (!x->type_offload ||
- (x->xso.type == XFRM_DEV_OFFLOAD_UNSPECIFIED && x->encap))
+ if (x->xso.type == XFRM_DEV_OFFLOAD_UNSPECIFIED)
return false;
- if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET ||
- ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
- !xdst->child->xfrm)) {
+ if ((dev == xfrm_dst_path(dst)->dev) && !xdst->child->xfrm) {
mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
if (skb->len <= mtu)
goto ok;
@@ -434,8 +431,29 @@ bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
return false;
ok:
- if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
- return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
+ check_tunnel_size = x->xso.type == XFRM_DEV_OFFLOAD_PACKET &&
+ x->props.mode == XFRM_MODE_TUNNEL;
+ switch (x->props.family) {
+ case AF_INET:
+ /* Check for IPv4 options */
+ if (ip_hdr(skb)->ihl != 5)
+ return false;
+ if (check_tunnel_size && xfrm4_tunnel_check_size(skb))
+ return false;
+ break;
+ case AF_INET6:
+ /* Check for IPv6 extensions */
+ if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
+ return false;
+ if (check_tunnel_size && xfrm6_tunnel_check_size(skb))
+ return false;
+ break;
+ default:
+ break;
+ }
+
+ if (dev->xfrmdev_ops->xdo_dev_offload_ok)
+ return dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
return true;
}
diff --git a/net/xfrm/xfrm_interface_core.c b/net/xfrm/xfrm_interface_core.c
index c397eb99d867..cb1e12740c87 100644
--- a/net/xfrm/xfrm_interface_core.c
+++ b/net/xfrm/xfrm_interface_core.c
@@ -242,10 +242,9 @@ static void xfrmi_dev_free(struct net_device *dev)
gro_cells_destroy(&xi->gro_cells);
}
-static int xfrmi_create(struct net_device *dev)
+static int xfrmi_create(struct net *net, struct net_device *dev)
{
struct xfrm_if *xi = netdev_priv(dev);
- struct net *net = dev_net(dev);
struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
int err;
@@ -814,15 +813,17 @@ static void xfrmi_netlink_parms(struct nlattr *data[],
parms->collect_md = true;
}
-static int xfrmi_newlink(struct net *src_net, struct net_device *dev,
- struct nlattr *tb[], struct nlattr *data[],
- struct netlink_ext_ack *extack)
+static int xfrmi_newlink(struct net_device *dev,
+ struct rtnl_newlink_params *params,
+ struct netlink_ext_ack *extack)
{
- struct net *net = dev_net(dev);
+ struct nlattr **data = params->data;
struct xfrm_if_parms p = {};
struct xfrm_if *xi;
+ struct net *net;
int err;
+ net = params->link_net ? : dev_net(dev);
xfrmi_netlink_parms(data, &p);
if (p.collect_md) {
struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
@@ -851,7 +852,7 @@ static int xfrmi_newlink(struct net *src_net, struct net_device *dev,
xi->net = net;
xi->dev = dev;
- err = xfrmi_create(dev);
+ err = xfrmi_create(net, dev);
return err;
}
@@ -951,32 +952,28 @@ static struct rtnl_link_ops xfrmi_link_ops __read_mostly = {
.get_link_net = xfrmi_get_link_net,
};
-static void __net_exit xfrmi_exit_batch_rtnl(struct list_head *net_exit_list,
- struct list_head *dev_to_kill)
+static void __net_exit xfrmi_exit_rtnl(struct net *net,
+ struct list_head *dev_to_kill)
{
- struct net *net;
+ struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
+ struct xfrm_if __rcu **xip;
+ struct xfrm_if *xi;
+ int i;
- ASSERT_RTNL();
- list_for_each_entry(net, net_exit_list, exit_list) {
- struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
- struct xfrm_if __rcu **xip;
- struct xfrm_if *xi;
- int i;
-
- for (i = 0; i < XFRMI_HASH_SIZE; i++) {
- for (xip = &xfrmn->xfrmi[i];
- (xi = rtnl_dereference(*xip)) != NULL;
- xip = &xi->next)
- unregister_netdevice_queue(xi->dev, dev_to_kill);
- }
- xi = rtnl_dereference(xfrmn->collect_md_xfrmi);
- if (xi)
+ for (i = 0; i < XFRMI_HASH_SIZE; i++) {
+ for (xip = &xfrmn->xfrmi[i];
+ (xi = rtnl_net_dereference(net, *xip)) != NULL;
+ xip = &xi->next)
unregister_netdevice_queue(xi->dev, dev_to_kill);
}
+
+ xi = rtnl_net_dereference(net, xfrmn->collect_md_xfrmi);
+ if (xi)
+ unregister_netdevice_queue(xi->dev, dev_to_kill);
}
static struct pernet_operations xfrmi_net_ops = {
- .exit_batch_rtnl = xfrmi_exit_batch_rtnl,
+ .exit_rtnl = xfrmi_exit_rtnl,
.id = &xfrmi_net_id,
.size = sizeof(struct xfrmi_net),
};
diff --git a/net/xfrm/xfrm_ipcomp.c b/net/xfrm/xfrm_ipcomp.c
index 9c0fa0e1786a..907c3ccb440d 100644
--- a/net/xfrm/xfrm_ipcomp.c
+++ b/net/xfrm/xfrm_ipcomp.c
@@ -3,7 +3,7 @@
* IP Payload Compression Protocol (IPComp) - RFC3173.
*
* Copyright (c) 2003 James Morris <jmorris@intercode.com.au>
- * Copyright (c) 2003-2008 Herbert Xu <herbert@gondor.apana.org.au>
+ * Copyright (c) 2003-2025 Herbert Xu <herbert@gondor.apana.org.au>
*
* Todo:
* - Tunable compression parameters.
@@ -11,303 +11,301 @@
* - Adaptive compression.
*/
-#include <linux/crypto.h>
+#include <crypto/acompress.h>
#include <linux/err.h>
-#include <linux/list.h>
#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/percpu.h>
+#include <linux/skbuff_ref.h>
#include <linux/slab.h>
-#include <linux/smp.h>
-#include <linux/vmalloc.h>
-#include <net/ip.h>
#include <net/ipcomp.h>
#include <net/xfrm.h>
-struct ipcomp_tfms {
- struct list_head list;
- struct crypto_comp * __percpu *tfms;
- int users;
+#define IPCOMP_SCRATCH_SIZE 65400
+
+struct ipcomp_skb_cb {
+ struct xfrm_skb_cb xfrm;
+ struct acomp_req *req;
};
-static DEFINE_MUTEX(ipcomp_resource_mutex);
-static void * __percpu *ipcomp_scratches;
-static int ipcomp_scratch_users;
-static LIST_HEAD(ipcomp_tfms_list);
+struct ipcomp_data {
+ u16 threshold;
+ struct crypto_acomp *tfm;
+};
-static int ipcomp_decompress(struct xfrm_state *x, struct sk_buff *skb)
+struct ipcomp_req_extra {
+ struct xfrm_state *x;
+ struct scatterlist sg[];
+};
+
+static inline struct ipcomp_skb_cb *ipcomp_cb(struct sk_buff *skb)
{
- struct ipcomp_data *ipcd = x->data;
- const int plen = skb->len;
- int dlen = IPCOMP_SCRATCH_SIZE;
- const u8 *start = skb->data;
- u8 *scratch = *this_cpu_ptr(ipcomp_scratches);
- struct crypto_comp *tfm = *this_cpu_ptr(ipcd->tfms);
- int err = crypto_comp_decompress(tfm, start, plen, scratch, &dlen);
- int len;
+ struct ipcomp_skb_cb *cb = (void *)skb->cb;
- if (err)
- return err;
+ BUILD_BUG_ON(sizeof(*cb) > sizeof(skb->cb));
+ return cb;
+}
- if (dlen < (plen + sizeof(struct ip_comp_hdr)))
- return -EINVAL;
+static int ipcomp_post_acomp(struct sk_buff *skb, int err, int hlen)
+{
+ struct acomp_req *req = ipcomp_cb(skb)->req;
+ struct ipcomp_req_extra *extra;
+ struct scatterlist *dsg;
+ int len, dlen;
- len = dlen - plen;
- if (len > skb_tailroom(skb))
- len = skb_tailroom(skb);
+ if (unlikely(err))
+ goto out_free_req;
- __skb_put(skb, len);
+ extra = acomp_request_extra(req);
+ dsg = extra->sg;
+ dlen = req->dlen;
- len += plen;
- skb_copy_to_linear_data(skb, scratch, len);
+ pskb_trim_unique(skb, 0);
+ __skb_put(skb, hlen);
- while ((scratch += len, dlen -= len) > 0) {
+ /* Only update truesize on input. */
+ if (!hlen)
+ skb->truesize += dlen;
+ skb->data_len = dlen;
+ skb->len += dlen;
+
+ do {
skb_frag_t *frag;
struct page *page;
- if (WARN_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS))
- return -EMSGSIZE;
-
frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
- page = alloc_page(GFP_ATOMIC);
-
- if (!page)
- return -ENOMEM;
+ page = sg_page(dsg);
+ dsg = sg_next(dsg);
len = PAGE_SIZE;
if (dlen < len)
len = dlen;
skb_frag_fill_page_desc(frag, page, 0, len);
- memcpy(skb_frag_address(frag), scratch, len);
-
- skb->truesize += len;
- skb->data_len += len;
- skb->len += len;
skb_shinfo(skb)->nr_frags++;
- }
+ } while ((dlen -= len));
- return 0;
+ for (; dsg; dsg = sg_next(dsg))
+ __free_page(sg_page(dsg));
+
+out_free_req:
+ acomp_request_free(req);
+ return err;
}
-int ipcomp_input(struct xfrm_state *x, struct sk_buff *skb)
+static int ipcomp_input_done2(struct sk_buff *skb, int err)
{
- int nexthdr;
- int err = -ENOMEM;
- struct ip_comp_hdr *ipch;
-
- if (skb_linearize_cow(skb))
- goto out;
-
- skb->ip_summed = CHECKSUM_NONE;
+ struct ip_comp_hdr *ipch = ip_comp_hdr(skb);
+ const int plen = skb->len;
- /* Remove ipcomp header and decompress original payload */
- ipch = (void *)skb->data;
- nexthdr = ipch->nexthdr;
+ skb_reset_transport_header(skb);
- skb->transport_header = skb->network_header + sizeof(*ipch);
- __skb_pull(skb, sizeof(*ipch));
- err = ipcomp_decompress(x, skb);
- if (err)
- goto out;
+ return ipcomp_post_acomp(skb, err, 0) ?:
+ skb->len < (plen + sizeof(ip_comp_hdr)) ? -EINVAL :
+ ipch->nexthdr;
+}
- err = nexthdr;
+static void ipcomp_input_done(void *data, int err)
+{
+ struct sk_buff *skb = data;
-out:
- return err;
+ xfrm_input_resume(skb, ipcomp_input_done2(skb, err));
}
-EXPORT_SYMBOL_GPL(ipcomp_input);
-static int ipcomp_compress(struct xfrm_state *x, struct sk_buff *skb)
+static struct acomp_req *ipcomp_setup_req(struct xfrm_state *x,
+ struct sk_buff *skb, int minhead,
+ int dlen)
{
+ const int dnfrags = min(MAX_SKB_FRAGS, 16);
struct ipcomp_data *ipcd = x->data;
+ struct ipcomp_req_extra *extra;
+ struct scatterlist *sg, *dsg;
const int plen = skb->len;
- int dlen = IPCOMP_SCRATCH_SIZE;
- u8 *start = skb->data;
- struct crypto_comp *tfm;
- u8 *scratch;
+ struct crypto_acomp *tfm;
+ struct acomp_req *req;
+ int nfrags;
+ int total;
int err;
+ int i;
- local_bh_disable();
- scratch = *this_cpu_ptr(ipcomp_scratches);
- tfm = *this_cpu_ptr(ipcd->tfms);
- err = crypto_comp_compress(tfm, start, plen, scratch, &dlen);
- if (err)
- goto out;
-
- if ((dlen + sizeof(struct ip_comp_hdr)) >= plen) {
- err = -EMSGSIZE;
- goto out;
- }
+ ipcomp_cb(skb)->req = NULL;
- memcpy(start + sizeof(struct ip_comp_hdr), scratch, dlen);
- local_bh_enable();
+ do {
+ struct sk_buff *trailer;
- pskb_trim(skb, dlen + sizeof(struct ip_comp_hdr));
- return 0;
+ if (skb->len > PAGE_SIZE) {
+ if (skb_linearize_cow(skb))
+ return ERR_PTR(-ENOMEM);
+ nfrags = 1;
+ break;
+ }
-out:
- local_bh_enable();
- return err;
-}
+ if (!skb_cloned(skb) && skb_headlen(skb) >= minhead) {
+ if (!skb_is_nonlinear(skb)) {
+ nfrags = 1;
+ break;
+ } else if (!skb_has_frag_list(skb)) {
+ nfrags = skb_shinfo(skb)->nr_frags;
+ nfrags++;
+ break;
+ }
+ }
-int ipcomp_output(struct xfrm_state *x, struct sk_buff *skb)
-{
- int err;
- struct ip_comp_hdr *ipch;
- struct ipcomp_data *ipcd = x->data;
+ nfrags = skb_cow_data(skb, skb_headlen(skb) < minhead ?
+ minhead - skb_headlen(skb) : 0,
+ &trailer);
+ if (nfrags < 0)
+ return ERR_PTR(nfrags);
+ } while (0);
+
+ tfm = ipcd->tfm;
+ req = acomp_request_alloc_extra(
+ tfm, sizeof(*extra) + sizeof(*sg) * (nfrags + dnfrags),
+ GFP_ATOMIC);
+ ipcomp_cb(skb)->req = req;
+ if (!req)
+ return ERR_PTR(-ENOMEM);
+
+ extra = acomp_request_extra(req);
+ extra->x = x;
+
+ dsg = extra->sg;
+ sg = dsg + dnfrags;
+ sg_init_table(sg, nfrags);
+ err = skb_to_sgvec(skb, sg, 0, plen);
+ if (unlikely(err < 0))
+ return ERR_PTR(err);
+
+ sg_init_table(dsg, dnfrags);
+ total = 0;
+ for (i = 0; i < dnfrags && total < dlen; i++) {
+ struct page *page;
- if (skb->len < ipcd->threshold) {
- /* Don't bother compressing */
- goto out_ok;
+ page = alloc_page(GFP_ATOMIC);
+ if (!page)
+ break;
+ sg_set_page(dsg + i, page, PAGE_SIZE, 0);
+ total += PAGE_SIZE;
}
+ if (!i)
+ return ERR_PTR(-ENOMEM);
+ sg_mark_end(dsg + i - 1);
+ dlen = min(dlen, total);
- if (skb_linearize_cow(skb))
- goto out_ok;
-
- err = ipcomp_compress(x, skb);
-
- if (err) {
- goto out_ok;
- }
+ acomp_request_set_params(req, sg, dsg, plen, dlen);
- /* Install ipcomp header, convert into ipcomp datagram. */
- ipch = ip_comp_hdr(skb);
- ipch->nexthdr = *skb_mac_header(skb);
- ipch->flags = 0;
- ipch->cpi = htons((u16 )ntohl(x->id.spi));
- *skb_mac_header(skb) = IPPROTO_COMP;
-out_ok:
- skb_push(skb, -skb_network_offset(skb));
- return 0;
+ return req;
}
-EXPORT_SYMBOL_GPL(ipcomp_output);
-static void ipcomp_free_scratches(void)
+static int ipcomp_decompress(struct xfrm_state *x, struct sk_buff *skb)
{
- int i;
- void * __percpu *scratches;
-
- if (--ipcomp_scratch_users)
- return;
+ struct acomp_req *req;
+ int err;
- scratches = ipcomp_scratches;
- if (!scratches)
- return;
+ req = ipcomp_setup_req(x, skb, 0, IPCOMP_SCRATCH_SIZE);
+ err = PTR_ERR(req);
+ if (IS_ERR(req))
+ goto out;
- for_each_possible_cpu(i)
- vfree(*per_cpu_ptr(scratches, i));
+ acomp_request_set_callback(req, 0, ipcomp_input_done, skb);
+ err = crypto_acomp_decompress(req);
+ if (err == -EINPROGRESS)
+ return err;
- free_percpu(scratches);
- ipcomp_scratches = NULL;
+out:
+ return ipcomp_input_done2(skb, err);
}
-static void * __percpu *ipcomp_alloc_scratches(void)
+int ipcomp_input(struct xfrm_state *x, struct sk_buff *skb)
{
- void * __percpu *scratches;
- int i;
-
- if (ipcomp_scratch_users++)
- return ipcomp_scratches;
-
- scratches = alloc_percpu(void *);
- if (!scratches)
- return NULL;
+ struct ip_comp_hdr *ipch __maybe_unused;
- ipcomp_scratches = scratches;
+ if (!pskb_may_pull(skb, sizeof(*ipch)))
+ return -EINVAL;
- for_each_possible_cpu(i) {
- void *scratch;
+ skb->ip_summed = CHECKSUM_NONE;
- scratch = vmalloc_node(IPCOMP_SCRATCH_SIZE, cpu_to_node(i));
- if (!scratch)
- return NULL;
- *per_cpu_ptr(scratches, i) = scratch;
- }
+ /* Remove ipcomp header and decompress original payload */
+ __skb_pull(skb, sizeof(*ipch));
- return scratches;
+ return ipcomp_decompress(x, skb);
}
+EXPORT_SYMBOL_GPL(ipcomp_input);
-static void ipcomp_free_tfms(struct crypto_comp * __percpu *tfms)
+static int ipcomp_output_push(struct sk_buff *skb)
{
- struct ipcomp_tfms *pos;
- int cpu;
-
- list_for_each_entry(pos, &ipcomp_tfms_list, list) {
- if (pos->tfms == tfms)
- break;
- }
-
- WARN_ON(list_entry_is_head(pos, &ipcomp_tfms_list, list));
-
- if (--pos->users)
- return;
+ skb_push(skb, -skb_network_offset(skb));
+ return 0;
+}
- list_del(&pos->list);
- kfree(pos);
+static int ipcomp_output_done2(struct xfrm_state *x, struct sk_buff *skb,
+ int err)
+{
+ struct ip_comp_hdr *ipch;
- if (!tfms)
- return;
+ err = ipcomp_post_acomp(skb, err, sizeof(*ipch));
+ if (err)
+ goto out_ok;
- for_each_possible_cpu(cpu) {
- struct crypto_comp *tfm = *per_cpu_ptr(tfms, cpu);
- crypto_free_comp(tfm);
- }
- free_percpu(tfms);
+ /* Install ipcomp header, convert into ipcomp datagram. */
+ ipch = ip_comp_hdr(skb);
+ ipch->nexthdr = *skb_mac_header(skb);
+ ipch->flags = 0;
+ ipch->cpi = htons((u16 )ntohl(x->id.spi));
+ *skb_mac_header(skb) = IPPROTO_COMP;
+out_ok:
+ return ipcomp_output_push(skb);
}
-static struct crypto_comp * __percpu *ipcomp_alloc_tfms(const char *alg_name)
+static void ipcomp_output_done(void *data, int err)
{
- struct ipcomp_tfms *pos;
- struct crypto_comp * __percpu *tfms;
- int cpu;
+ struct ipcomp_req_extra *extra;
+ struct sk_buff *skb = data;
+ struct acomp_req *req;
+ req = ipcomp_cb(skb)->req;
+ extra = acomp_request_extra(req);
- list_for_each_entry(pos, &ipcomp_tfms_list, list) {
- struct crypto_comp *tfm;
+ xfrm_output_resume(skb_to_full_sk(skb), skb,
+ ipcomp_output_done2(extra->x, skb, err));
+}
- /* This can be any valid CPU ID so we don't need locking. */
- tfm = this_cpu_read(*pos->tfms);
+static int ipcomp_compress(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ip_comp_hdr *ipch __maybe_unused;
+ struct acomp_req *req;
+ int err;
- if (!strcmp(crypto_comp_name(tfm), alg_name)) {
- pos->users++;
- return pos->tfms;
- }
- }
+ req = ipcomp_setup_req(x, skb, sizeof(*ipch),
+ skb->len - sizeof(*ipch));
+ err = PTR_ERR(req);
+ if (IS_ERR(req))
+ goto out;
- pos = kmalloc(sizeof(*pos), GFP_KERNEL);
- if (!pos)
- return NULL;
+ acomp_request_set_callback(req, 0, ipcomp_output_done, skb);
+ err = crypto_acomp_compress(req);
+ if (err == -EINPROGRESS)
+ return err;
- pos->users = 1;
- INIT_LIST_HEAD(&pos->list);
- list_add(&pos->list, &ipcomp_tfms_list);
+out:
+ return ipcomp_output_done2(x, skb, err);
+}
- pos->tfms = tfms = alloc_percpu(struct crypto_comp *);
- if (!tfms)
- goto error;
+int ipcomp_output(struct xfrm_state *x, struct sk_buff *skb)
+{
+ struct ipcomp_data *ipcd = x->data;
- for_each_possible_cpu(cpu) {
- struct crypto_comp *tfm = crypto_alloc_comp(alg_name, 0,
- CRYPTO_ALG_ASYNC);
- if (IS_ERR(tfm))
- goto error;
- *per_cpu_ptr(tfms, cpu) = tfm;
+ if (skb->len < ipcd->threshold) {
+ /* Don't bother compressing */
+ return ipcomp_output_push(skb);
}
- return tfms;
-
-error:
- ipcomp_free_tfms(tfms);
- return NULL;
+ return ipcomp_compress(x, skb);
}
+EXPORT_SYMBOL_GPL(ipcomp_output);
static void ipcomp_free_data(struct ipcomp_data *ipcd)
{
- if (ipcd->tfms)
- ipcomp_free_tfms(ipcd->tfms);
- ipcomp_free_scratches();
+ crypto_free_acomp(ipcd->tfm);
}
void ipcomp_destroy(struct xfrm_state *x)
@@ -316,9 +314,7 @@ void ipcomp_destroy(struct xfrm_state *x)
if (!ipcd)
return;
xfrm_state_delete_tunnel(x);
- mutex_lock(&ipcomp_resource_mutex);
ipcomp_free_data(ipcd);
- mutex_unlock(&ipcomp_resource_mutex);
kfree(ipcd);
}
EXPORT_SYMBOL_GPL(ipcomp_destroy);
@@ -345,14 +341,9 @@ int ipcomp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
if (!ipcd)
goto out;
- mutex_lock(&ipcomp_resource_mutex);
- if (!ipcomp_alloc_scratches())
- goto error;
-
- ipcd->tfms = ipcomp_alloc_tfms(x->calg->alg_name);
- if (!ipcd->tfms)
+ ipcd->tfm = crypto_alloc_acomp(x->calg->alg_name, 0, 0);
+ if (IS_ERR(ipcd->tfm))
goto error;
- mutex_unlock(&ipcomp_resource_mutex);
calg_desc = xfrm_calg_get_byname(x->calg->alg_name, 0);
BUG_ON(!calg_desc);
@@ -364,7 +355,6 @@ out:
error:
ipcomp_free_data(ipcd);
- mutex_unlock(&ipcomp_resource_mutex);
kfree(ipcd);
goto out;
}
diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c
index 755f1eea8bfa..3b6d7284fc70 100644
--- a/net/xfrm/xfrm_iptfs.c
+++ b/net/xfrm/xfrm_iptfs.c
@@ -2625,12 +2625,10 @@ static void __iptfs_init_state(struct xfrm_state *x,
struct xfrm_iptfs_data *xtfs)
{
__skb_queue_head_init(&xtfs->queue);
- hrtimer_init(&xtfs->iptfs_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
- xtfs->iptfs_timer.function = iptfs_delay_timer;
+ hrtimer_setup(&xtfs->iptfs_timer, iptfs_delay_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
spin_lock_init(&xtfs->drop_lock);
- hrtimer_init(&xtfs->drop_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
- xtfs->drop_timer.function = iptfs_drop_timer;
+ hrtimer_setup(&xtfs->drop_timer, iptfs_drop_timer, CLOCK_MONOTONIC, IPTFS_HRTIMER_MODE);
/* Modify type (esp) adjustment values */
diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c
index 82f0a301683f..ebf95d48e86c 100644
--- a/net/xfrm/xfrm_nat_keepalive.c
+++ b/net/xfrm/xfrm_nat_keepalive.c
@@ -9,9 +9,13 @@
#include <net/ip6_checksum.h>
#include <net/xfrm.h>
-static DEFINE_PER_CPU(struct sock *, nat_keepalive_sk_ipv4);
+static DEFINE_PER_CPU(struct sock_bh_locked, nat_keepalive_sk_ipv4) = {
+ .bh_lock = INIT_LOCAL_LOCK(bh_lock),
+};
#if IS_ENABLED(CONFIG_IPV6)
-static DEFINE_PER_CPU(struct sock *, nat_keepalive_sk_ipv6);
+static DEFINE_PER_CPU(struct sock_bh_locked, nat_keepalive_sk_ipv6) = {
+ .bh_lock = INIT_LOCAL_LOCK(bh_lock),
+};
#endif
struct nat_keepalive {
@@ -56,10 +60,12 @@ static int nat_keepalive_send_ipv4(struct sk_buff *skb,
skb_dst_set(skb, &rt->dst);
- sk = *this_cpu_ptr(&nat_keepalive_sk_ipv4);
+ local_lock_nested_bh(&nat_keepalive_sk_ipv4.bh_lock);
+ sk = this_cpu_read(nat_keepalive_sk_ipv4.sock);
sock_net_set(sk, net);
err = ip_build_and_send_pkt(skb, sk, fl4.saddr, fl4.daddr, NULL, tos);
sock_net_set(sk, &init_net);
+ local_unlock_nested_bh(&nat_keepalive_sk_ipv4.bh_lock);
return err;
}
@@ -89,15 +95,19 @@ static int nat_keepalive_send_ipv6(struct sk_buff *skb,
fl6.fl6_sport = ka->encap_sport;
fl6.fl6_dport = ka->encap_dport;
- sk = *this_cpu_ptr(&nat_keepalive_sk_ipv6);
+ local_lock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock);
+ sk = this_cpu_read(nat_keepalive_sk_ipv6.sock);
sock_net_set(sk, net);
dst = ipv6_stub->ipv6_dst_lookup_flow(net, sk, &fl6, NULL);
- if (IS_ERR(dst))
+ if (IS_ERR(dst)) {
+ local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock);
return PTR_ERR(dst);
+ }
skb_dst_set(skb, dst);
err = ipv6_stub->ip6_xmit(sk, skb, &fl6, skb->mark, NULL, 0, 0);
sock_net_set(sk, &init_net);
+ local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock);
return err;
}
#endif
@@ -202,7 +212,7 @@ static void nat_keepalive_work(struct work_struct *work)
(ctx.next_run - ctx.now) * HZ);
}
-static int nat_keepalive_sk_init(struct sock * __percpu *socks,
+static int nat_keepalive_sk_init(struct sock_bh_locked __percpu *socks,
unsigned short family)
{
struct sock *sk;
@@ -214,22 +224,22 @@ static int nat_keepalive_sk_init(struct sock * __percpu *socks,
if (err < 0)
goto err;
- *per_cpu_ptr(socks, i) = sk;
+ per_cpu_ptr(socks, i)->sock = sk;
}
return 0;
err:
for_each_possible_cpu(i)
- inet_ctl_sock_destroy(*per_cpu_ptr(socks, i));
+ inet_ctl_sock_destroy(per_cpu_ptr(socks, i)->sock);
return err;
}
-static void nat_keepalive_sk_fini(struct sock * __percpu *socks)
+static void nat_keepalive_sk_fini(struct sock_bh_locked __percpu *socks)
{
int i;
for_each_possible_cpu(i)
- inet_ctl_sock_destroy(*per_cpu_ptr(socks, i));
+ inet_ctl_sock_destroy(per_cpu_ptr(socks, i)->sock);
}
void xfrm_nat_keepalive_state_updated(struct xfrm_state *x)
diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c
index f7abd42c077d..9077730ff7d0 100644
--- a/net/xfrm/xfrm_output.c
+++ b/net/xfrm/xfrm_output.c
@@ -612,6 +612,40 @@ out:
}
EXPORT_SYMBOL_GPL(xfrm_output_resume);
+static int xfrm_dev_direct_output(struct sock *sk, struct xfrm_state *x,
+ struct sk_buff *skb)
+{
+ struct dst_entry *dst = skb_dst(skb);
+ struct net *net = xs_net(x);
+ int err;
+
+ dst = skb_dst_pop(skb);
+ if (!dst) {
+ XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
+ kfree_skb(skb);
+ return -EHOSTUNREACH;
+ }
+ skb_dst_set(skb, dst);
+ nf_reset_ct(skb);
+
+ err = skb_dst(skb)->ops->local_out(net, sk, skb);
+ if (unlikely(err != 1)) {
+ kfree_skb(skb);
+ return err;
+ }
+
+ /* In transport mode, network destination is
+ * directly reachable, while in tunnel mode,
+ * inner packet network may not be. In packet
+ * offload type, HW is responsible for hard
+ * header packet mangling so directly xmit skb
+ * to netdevice.
+ */
+ skb->dev = x->xso.dev;
+ __skb_push(skb, skb->dev->hard_header_len);
+ return dev_queue_xmit(skb);
+}
+
static int xfrm_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
{
return xfrm_output_resume(sk, skb, 1);
@@ -735,6 +769,13 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb)
return -EHOSTUNREACH;
}
+ /* Exclusive direct xmit for tunnel mode, as
+ * some filtering or matching rules may apply
+ * in transport mode.
+ */
+ if (x->props.mode == XFRM_MODE_TUNNEL)
+ return xfrm_dev_direct_output(sk, x, skb);
+
return xfrm_output_resume(sk, skb, 0);
}
@@ -758,7 +799,7 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb)
skb->encapsulation = 1;
if (skb_is_gso(skb)) {
- if (skb->inner_protocol)
+ if (skb->inner_protocol && x->props.mode == XFRM_MODE_TUNNEL)
return xfrm_output_gso(net, sk, skb);
skb_shinfo(skb)->gso_type |= SKB_GSO_ESP;
@@ -786,7 +827,7 @@ out:
}
EXPORT_SYMBOL_GPL(xfrm_output);
-static int xfrm4_tunnel_check_size(struct sk_buff *skb)
+int xfrm4_tunnel_check_size(struct sk_buff *skb)
{
int mtu, ret = 0;
@@ -812,6 +853,7 @@ static int xfrm4_tunnel_check_size(struct sk_buff *skb)
out:
return ret;
}
+EXPORT_SYMBOL_GPL(xfrm4_tunnel_check_size);
static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)
{
@@ -834,7 +876,7 @@ static int xfrm4_extract_output(struct xfrm_state *x, struct sk_buff *skb)
}
#if IS_ENABLED(CONFIG_IPV6)
-static int xfrm6_tunnel_check_size(struct sk_buff *skb)
+int xfrm6_tunnel_check_size(struct sk_buff *skb)
{
int mtu, ret = 0;
struct dst_entry *dst = skb_dst(skb);
@@ -864,6 +906,7 @@ static int xfrm6_tunnel_check_size(struct sk_buff *skb)
out:
return ret;
}
+EXPORT_SYMBOL_GPL(xfrm6_tunnel_check_size);
#endif
static int xfrm6_extract_output(struct xfrm_state *x, struct sk_buff *skb)
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 6551e588fe52..d4134a18c658 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -462,7 +462,7 @@ void xfrm_policy_destroy(struct xfrm_policy *policy)
{
BUG_ON(!policy->walk.dead);
- if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
+ if (timer_delete(&policy->timer) || timer_delete(&policy->polq.hold_timer))
BUG();
xfrm_dev_policy_free(policy);
@@ -487,11 +487,11 @@ static void xfrm_policy_kill(struct xfrm_policy *policy)
atomic_inc(&policy->genid);
- if (del_timer(&policy->polq.hold_timer))
+ if (timer_delete(&policy->polq.hold_timer))
xfrm_pol_put(policy);
skb_queue_purge(&policy->polq.hold_queue);
- if (del_timer(&policy->timer))
+ if (timer_delete(&policy->timer))
xfrm_pol_put(policy);
/* XXX: Flush state cache */
@@ -1469,7 +1469,7 @@ static void xfrm_policy_requeue(struct xfrm_policy *old,
spin_lock_bh(&pq->hold_queue.lock);
skb_queue_splice_init(&pq->hold_queue, &list);
- if (del_timer(&pq->hold_timer))
+ if (timer_delete(&pq->hold_timer))
xfrm_pol_put(old);
spin_unlock_bh(&pq->hold_queue.lock);
@@ -1581,6 +1581,9 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
struct xfrm_policy *delpol;
struct hlist_head *chain;
+ /* Sanitize mark before store */
+ policy->mark.v &= policy->mark.m;
+
spin_lock_bh(&net->xfrm.xfrm_policy_lock);
chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
if (chain)
@@ -3004,7 +3007,7 @@ static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *s
sched_next = jiffies + pq->timeout;
- if (del_timer(&pq->hold_timer)) {
+ if (timer_delete(&pq->hold_timer)) {
if (time_before(pq->hold_timer.expires, sched_next))
sched_next = pq->hold_timer.expires;
xfrm_pol_put(pol);
@@ -3294,7 +3297,7 @@ no_transform:
ok:
xfrm_pols_put(pols, drop_pols);
- if (dst && dst->xfrm &&
+ if (dst->xfrm &&
(dst->xfrm->props.mode == XFRM_MODE_TUNNEL ||
dst->xfrm->props.mode == XFRM_MODE_IPTFS))
dst->flags |= DST_XFRM_TUNNEL;
@@ -4630,7 +4633,7 @@ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
struct xfrm_migrate *m, int num_migrate,
struct xfrm_kmaddress *k, struct net *net,
struct xfrm_encap_tmpl *encap, u32 if_id,
- struct netlink_ext_ack *extack)
+ struct netlink_ext_ack *extack, struct xfrm_user_offload *xuo)
{
int i, err, nx_cur = 0, nx_new = 0;
struct xfrm_policy *pol = NULL;
@@ -4663,7 +4666,7 @@ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
if ((x = xfrm_migrate_state_find(mp, net, if_id))) {
x_cur[nx_cur] = x;
nx_cur++;
- xc = xfrm_state_migrate(x, mp, encap);
+ xc = xfrm_state_migrate(x, mp, encap, net, xuo, extack);
if (xc) {
x_new[nx_new] = xc;
nx_new++;
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index ad2202fa82f3..203b585c2ae2 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -424,18 +424,18 @@ void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
}
EXPORT_SYMBOL(xfrm_unregister_type_offload);
-static const struct xfrm_type_offload *
-xfrm_get_type_offload(u8 proto, unsigned short family, bool try_load)
+void xfrm_set_type_offload(struct xfrm_state *x)
{
const struct xfrm_type_offload *type = NULL;
struct xfrm_state_afinfo *afinfo;
+ bool try_load = true;
retry:
- afinfo = xfrm_state_get_afinfo(family);
+ afinfo = xfrm_state_get_afinfo(x->props.family);
if (unlikely(afinfo == NULL))
- return NULL;
+ goto out;
- switch (proto) {
+ switch (x->id.proto) {
case IPPROTO_ESP:
type = afinfo->type_offload_esp;
break;
@@ -449,18 +449,16 @@ retry:
rcu_read_unlock();
if (!type && try_load) {
- request_module("xfrm-offload-%d-%d", family, proto);
+ request_module("xfrm-offload-%d-%d", x->props.family,
+ x->id.proto);
try_load = false;
goto retry;
}
- return type;
-}
-
-static void xfrm_put_type_offload(const struct xfrm_type_offload *type)
-{
- module_put(type->owner);
+out:
+ x->type_offload = type;
}
+EXPORT_SYMBOL(xfrm_set_type_offload);
static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = {
[XFRM_MODE_BEET] = {
@@ -600,17 +598,15 @@ static void ___xfrm_state_destroy(struct xfrm_state *x)
if (x->mode_cbs && x->mode_cbs->destroy_state)
x->mode_cbs->destroy_state(x);
hrtimer_cancel(&x->mtimer);
- del_timer_sync(&x->rtimer);
- kfree(x->aead);
- kfree(x->aalg);
- kfree(x->ealg);
+ timer_delete_sync(&x->rtimer);
+ kfree_sensitive(x->aead);
+ kfree_sensitive(x->aalg);
+ kfree_sensitive(x->ealg);
kfree(x->calg);
kfree(x->encap);
kfree(x->coaddr);
kfree(x->replay_esn);
kfree(x->preplay_esn);
- if (x->type_offload)
- xfrm_put_type_offload(x->type_offload);
if (x->type) {
x->type->destructor(x);
xfrm_put_type(x->type);
@@ -746,8 +742,8 @@ struct xfrm_state *xfrm_state_alloc(struct net *net)
INIT_HLIST_NODE(&x->bysrc);
INIT_HLIST_NODE(&x->byspi);
INIT_HLIST_NODE(&x->byseq);
- hrtimer_init(&x->mtimer, CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);
- x->mtimer.function = xfrm_timer_handler;
+ hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME,
+ HRTIMER_MODE_ABS_SOFT);
timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
x->curlft.add_time = ktime_get_real_seconds();
x->lft.soft_byte_limit = XFRM_INF;
@@ -771,7 +767,7 @@ void xfrm_dev_state_delete(struct xfrm_state *x)
struct net_device *dev = READ_ONCE(xso->dev);
if (dev) {
- dev->xfrmdev_ops->xdo_dev_state_delete(x);
+ dev->xfrmdev_ops->xdo_dev_state_delete(dev, x);
spin_lock_bh(&xfrm_state_dev_gc_lock);
hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list);
spin_unlock_bh(&xfrm_state_dev_gc_lock);
@@ -784,6 +780,8 @@ void xfrm_dev_state_free(struct xfrm_state *x)
struct xfrm_dev_offload *xso = &x->xso;
struct net_device *dev = READ_ONCE(xso->dev);
+ xfrm_unset_type_offload(x);
+
if (dev && dev->xfrmdev_ops) {
spin_lock_bh(&xfrm_state_dev_gc_lock);
if (!hlist_unhashed(&x->dev_gclist))
@@ -791,7 +789,7 @@ void xfrm_dev_state_free(struct xfrm_state *x)
spin_unlock_bh(&xfrm_state_dev_gc_lock);
if (dev->xfrmdev_ops->xdo_dev_state_free)
- dev->xfrmdev_ops->xdo_dev_state_free(x);
+ dev->xfrmdev_ops->xdo_dev_state_free(dev, x);
WRITE_ONCE(xso->dev, NULL);
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
netdev_put(dev, &xso->dev_tracker);
@@ -840,9 +838,6 @@ int __xfrm_state_delete(struct xfrm_state *x)
xfrm_nat_keepalive_state_updated(x);
spin_unlock(&net->xfrm.xfrm_state_lock);
- if (x->encap_sk)
- sock_put(rcu_dereference_raw(x->encap_sk));
-
xfrm_dev_state_delete(x);
/* All xfrm_state objects are created by xfrm_state_alloc.
@@ -1553,19 +1548,19 @@ found:
if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
struct xfrm_dev_offload *xdo = &pol->xdo;
struct xfrm_dev_offload *xso = &x->xso;
+ struct net_device *dev = xdo->dev;
xso->type = XFRM_DEV_OFFLOAD_PACKET;
xso->dir = xdo->dir;
- xso->dev = xdo->dev;
- xso->real_dev = xdo->real_dev;
+ xso->dev = dev;
xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
- netdev_hold(xso->dev, &xso->dev_tracker, GFP_ATOMIC);
- error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x, NULL);
+ netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC);
+ error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x,
+ NULL);
if (error) {
xso->dir = 0;
- netdev_put(xso->dev, &xso->dev_tracker);
+ netdev_put(dev, &xso->dev_tracker);
xso->dev = NULL;
- xso->real_dev = NULL;
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
x->km.state = XFRM_STATE_DEAD;
to_put = x;
@@ -1723,6 +1718,9 @@ static void __xfrm_state_insert(struct xfrm_state *x)
list_add(&x->km.all, &net->xfrm.state_all);
+ /* Sanitize mark before store */
+ x->mark.v &= x->mark.m;
+
h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
x->props.reqid, x->props.family);
XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,
@@ -1960,8 +1958,9 @@ static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *secu
return 0;
}
-static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
- struct xfrm_encap_tmpl *encap)
+static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig,
+ struct xfrm_encap_tmpl *encap,
+ struct xfrm_migrate *m)
{
struct net *net = xs_net(orig);
struct xfrm_state *x = xfrm_state_alloc(net);
@@ -2060,6 +2059,11 @@ static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig,
goto error;
}
+
+ x->props.family = m->new_family;
+ memcpy(&x->id.daddr, &m->new_daddr, sizeof(x->id.daddr));
+ memcpy(&x->props.saddr, &m->new_saddr, sizeof(x->props.saddr));
+
return x;
error:
@@ -2122,21 +2126,23 @@ EXPORT_SYMBOL(xfrm_migrate_state_find);
struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
struct xfrm_migrate *m,
- struct xfrm_encap_tmpl *encap)
+ struct xfrm_encap_tmpl *encap,
+ struct net *net,
+ struct xfrm_user_offload *xuo,
+ struct netlink_ext_ack *extack)
{
struct xfrm_state *xc;
- xc = xfrm_state_clone(x, encap);
+ xc = xfrm_state_clone_and_setup(x, encap, m);
if (!xc)
return NULL;
- xc->props.family = m->new_family;
-
if (xfrm_init_state(xc) < 0)
goto error;
- memcpy(&xc->id.daddr, &m->new_daddr, sizeof(xc->id.daddr));
- memcpy(&xc->props.saddr, &m->new_saddr, sizeof(xc->props.saddr));
+ /* configure the hardware if offload is requested */
+ if (xuo && xfrm_dev_state_add(net, xc, xuo, extack))
+ goto error;
/* add state */
if (xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {
@@ -2315,12 +2321,12 @@ xfrm_state_lookup_byaddr(struct net *net, u32 mark,
struct xfrm_hash_state_ptrs state_ptrs;
struct xfrm_state *x;
- spin_lock_bh(&net->xfrm.xfrm_state_lock);
+ rcu_read_lock();
xfrm_hash_ptrs_get(net, &state_ptrs);
x = __xfrm_state_lookup_byaddr(&state_ptrs, mark, daddr, saddr, proto, family);
- spin_unlock_bh(&net->xfrm.xfrm_state_lock);
+ rcu_read_unlock();
return x;
}
EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
@@ -3122,8 +3128,7 @@ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
}
EXPORT_SYMBOL_GPL(xfrm_state_mtu);
-int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
- struct netlink_ext_ack *extack)
+int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
{
const struct xfrm_mode *inner_mode;
const struct xfrm_mode *outer_mode;
@@ -3178,8 +3183,6 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
goto error;
}
- x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload);
-
err = x->type->init_state(x, extack);
if (err)
goto error;
@@ -3192,12 +3195,6 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
}
x->outer_mode = *outer_mode;
- if (init_replay) {
- err = xfrm_init_replay(x, extack);
- if (err)
- goto error;
- }
-
if (x->nat_keepalive_interval) {
if (x->dir != XFRM_SA_DIR_OUT) {
NL_SET_ERR_MSG(extack, "NAT keepalive is only supported for outbound SAs");
@@ -3229,11 +3226,16 @@ int xfrm_init_state(struct xfrm_state *x)
{
int err;
- err = __xfrm_init_state(x, true, false, NULL);
- if (!err)
- x->km.state = XFRM_STATE_VALID;
+ err = __xfrm_init_state(x, NULL);
+ if (err)
+ return err;
- return err;
+ err = xfrm_init_replay(x, NULL);
+ if (err)
+ return err;
+
+ x->km.state = XFRM_STATE_VALID;
+ return 0;
}
EXPORT_SYMBOL(xfrm_init_state);
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index 08c6d6f0179f..59f258daf830 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -178,6 +178,28 @@ static inline int verify_replay(struct xfrm_usersa_info *p,
"Replay seq and seq_hi should be 0 for output SA");
return -EINVAL;
}
+
+ if (!(p->flags & XFRM_STATE_ESN)) {
+ if (rs->oseq_hi) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay oseq_hi should be 0 in non-ESN mode for output SA");
+ return -EINVAL;
+ }
+ if (rs->oseq == U32_MAX) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay oseq should be less than 0xFFFFFFFF in non-ESN mode for output SA");
+ return -EINVAL;
+ }
+ } else {
+ if (rs->oseq == U32_MAX && rs->oseq_hi == U32_MAX) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay oseq and oseq_hi should be less than 0xFFFFFFFF for output SA");
+ return -EINVAL;
+ }
+ }
if (rs->bmp_len) {
NL_SET_ERR_MSG(extack, "Replay bmp_len should 0 for output SA");
return -EINVAL;
@@ -190,6 +212,28 @@ static inline int verify_replay(struct xfrm_usersa_info *p,
"Replay oseq and oseq_hi should be 0 for input SA");
return -EINVAL;
}
+ if (!(p->flags & XFRM_STATE_ESN)) {
+ if (rs->seq_hi) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay seq_hi should be 0 in non-ESN mode for input SA");
+ return -EINVAL;
+ }
+
+ if (rs->seq == U32_MAX) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay seq should be less than 0xFFFFFFFF in non-ESN mode for input SA");
+ return -EINVAL;
+ }
+ } else {
+ if (rs->seq == U32_MAX && rs->seq_hi == U32_MAX) {
+ NL_SET_ERR_MSG(
+ extack,
+ "Replay seq and seq_hi should be less than 0xFFFFFFFF for input SA");
+ return -EINVAL;
+ }
+ }
}
return 0;
@@ -907,7 +951,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
goto error;
}
- err = __xfrm_init_state(x, false, attrs[XFRMA_OFFLOAD_DEV], extack);
+ err = __xfrm_init_state(x, extack);
if (err)
goto error;
@@ -1161,7 +1205,7 @@ static int copy_to_user_auth(struct xfrm_algo_auth *auth, struct sk_buff *skb)
if (!nla)
return -EMSGSIZE;
algo = nla_data(nla);
- strscpy_pad(algo->alg_name, auth->alg_name, sizeof(algo->alg_name));
+ strscpy_pad(algo->alg_name, auth->alg_name);
if (redact_secret && auth->alg_key_len)
memset(algo->alg_key, 0, (auth->alg_key_len + 7) / 8);
@@ -1174,7 +1218,7 @@ static int copy_to_user_auth(struct xfrm_algo_auth *auth, struct sk_buff *skb)
if (!nla)
return -EMSGSIZE;
ap = nla_data(nla);
- strscpy_pad(ap->alg_name, auth->alg_name, sizeof(ap->alg_name));
+ strscpy_pad(ap->alg_name, auth->alg_name);
ap->alg_key_len = auth->alg_key_len;
ap->alg_trunc_len = auth->alg_trunc_len;
if (redact_secret && auth->alg_key_len)
@@ -1195,7 +1239,7 @@ static int copy_to_user_aead(struct xfrm_algo_aead *aead, struct sk_buff *skb)
return -EMSGSIZE;
ap = nla_data(nla);
- strscpy_pad(ap->alg_name, aead->alg_name, sizeof(ap->alg_name));
+ strscpy_pad(ap->alg_name, aead->alg_name);
ap->alg_key_len = aead->alg_key_len;
ap->alg_icv_len = aead->alg_icv_len;
@@ -1217,7 +1261,7 @@ static int copy_to_user_ealg(struct xfrm_algo *ealg, struct sk_buff *skb)
return -EMSGSIZE;
ap = nla_data(nla);
- strscpy_pad(ap->alg_name, ealg->alg_name, sizeof(ap->alg_name));
+ strscpy_pad(ap->alg_name, ealg->alg_name);
ap->alg_key_len = ealg->alg_key_len;
if (redact_secret && ealg->alg_key_len)
@@ -1238,7 +1282,7 @@ static int copy_to_user_calg(struct xfrm_algo *calg, struct sk_buff *skb)
return -EMSGSIZE;
ap = nla_data(nla);
- strscpy_pad(ap->alg_name, calg->alg_name, sizeof(ap->alg_name));
+ strscpy_pad(ap->alg_name, calg->alg_name);
ap->alg_key_len = 0;
return 0;
@@ -3057,6 +3101,7 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
int n = 0;
struct net *net = sock_net(skb->sk);
struct xfrm_encap_tmpl *encap = NULL;
+ struct xfrm_user_offload *xuo = NULL;
u32 if_id = 0;
if (!attrs[XFRMA_MIGRATE]) {
@@ -3087,11 +3132,19 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
if (attrs[XFRMA_IF_ID])
if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
+ if (attrs[XFRMA_OFFLOAD_DEV]) {
+ xuo = kmemdup(nla_data(attrs[XFRMA_OFFLOAD_DEV]),
+ sizeof(*xuo), GFP_KERNEL);
+ if (!xuo) {
+ err = -ENOMEM;
+ goto error;
+ }
+ }
err = xfrm_migrate(&pi->sel, pi->dir, type, m, n, kmp, net, encap,
- if_id, extack);
-
+ if_id, extack, xuo);
+error:
kfree(encap);
-
+ kfree(xuo);
return err;
}
#else
generated by cgit (git 2.34.1) at 2025-05-31 05:52:31 +0000