Ever needed to add your custom logic into the network stack? Ever hacked the network stack but wasn't certain you're doing it right? Shmulik Ladkani talks about various mechanisms for customizing packet processing logic to the network stack's data path. He covers covering topics such as packet sockets, netfilter hooks, traffic control actions and ebpf. We will discuss their applicable use-cases, advantages and disadvantages. Shmulik Ladkani is a Tech Lead at Ravello Systems. Shmulik started his career at Jungo (acquired by NDS/Cisco) implementing residential gateway software, focusing on embedded Linux, Linux kernel, networking and hardware/software integration. 51966 coffees and billions of forwarded packets later, with millions of homes running his software, Shmulik left his position as Jungo’s lead architect and joined Ravello Systems (acquired by Oracle) as tech lead, developing a virtual data center as a cloud service. He's now focused around virtualization systems, network virtualization and SDN.

2. ●
●
●
●
●

3. ●
○
○
○
●
○

5. ●
●
●
●

7. …
…

11. packet_socket = socket(AF_PACKET, int socket_type, int protocol);

12. ●
○
●
○
●
○
●
○
●
●

13. ●
○ htons(ETH_P_EAPOL), htons(ETH_P_IP)
○
…
…

14. ●
○
○
○
…
…
… …

15. ●
○
●
○
●
○
●
struct sockaddr_ll {
unsigned short sll_family;
unsigned short sll_protocol;
int sll_ifindex;
unsigned short sll_hatype;
unsigned char sll_pkttype;
unsigned char sll_halen;
unsigned char sll_addr[8];
};

16. ●
○ setsockopt(s, SOL_SOCKET, SO_ATTACH_FILTER, …)
●
# sudo tcpdump -d arp
(000) ldh [12]
(001) jeq #0x806 jt 2 jf 3
(002) ret #262144
(003) ret #0
# sudo tcpdump -dd arp
{ 0x28, 0, 0, 0x0000000c },
{ 0x15, 0, 1, 0x00000806 },
{ 0x6, 0, 0, 0x00040000 },
{ 0x6, 0, 0, 0x00000000 },

17. ●
●
●
●
●

19. ●
●
○
○

20. ●
●
●
●
●
●
●
●
○ …

22. ●
●
●
●

23. int nf_register_net_hook(struct net *net,
const struct nf_hook_ops *ops);
int nf_register_net_hooks(struct net *net,
const struct nf_hook_ops *reg,
unsigned int n);
int nf_register_hook(struct nf_hook_ops *reg);
int nf_register_hooks(struct nf_hook_ops *reg,
unsigned int n);

24. static struct nf_hook_ops ipv4_synproxy_ops[] = {
{
.hook = ipv4_synproxy_hook,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_LOCAL_IN,
.priority = NF_IP_PRI_CONNTRACK_CONFIRM - 1,
.priv = NULL,
},
{
.hook = ipv4_synproxy_hook,
.pf = NFPROTO_IPV4,
.hooknum = NF_INET_POST_ROUTING,
.priority = NF_IP_PRI_CONNTRACK_CONFIRM - 1,
.priv = NULL,
},
};

25. static unsigned int ipv4_synproxy_hook(void *priv,
struct sk_buff *skb,
const struct nf_hook_state *nhs)
{
do_stuff();
if (...)
return NF_ACCEPT;
do_more_stuff();
if (...)
return NF_DROP;
return NF_ACCEPT;
}

26. ●
○
●
○
●
○
●
○
●
○

27. ●
●
○
○
●
○
○
○
●
●

28. ●
●
●
○
○
○
●
○
●

29. ●
●
●
○
●
●
●
○
○

30. ●
●
●
●
●
●

32. ●
○
○
○
○
○

34. …
…

35. ●
●

36. # tc filter add dev eth0 parent 1:0 protocol ip prio 1
u32 match ip src 4.3.2.1/32 match ip sport 80 0xffff
flowid 1:3
# tc filter add dev eth0 parent 1:0 protocol ip prio 1
u32 match u16 0x0000 0xffc0 at 2 flowid 1:4
# tc filter add dev eth0 parent 1:0 protocol ip prio 1
handle 1 fw classid 1:10
# tc filter add dev eth0 parent 1:
basic match 'meta(priority eq 6)' classid 1:10
# tc filter add dev eth0 parent 1:0
bpf obj bpf.o sec mycls flowid 1:1

37. ●
●
●
○
●

38. …
…

39. # tc qdisc add dev eth0 ingress
# tc filter add dev eth0 parent ffff: protocol all prio 1
u32 match u32 0 0
action mirred egress redirect dev eth1
# tc filter add dev eth0 parent ffff: protocol ip prio 6
u32 match ip src 10.0.0.9/32 action drop
# tc filter add dev eth0 parent ffff: pref 11 protocol ip
u32 match ip protocol 1 0xff flowid 1:1
u32 match ip src 10.0.0.2 flowid 1:1
action vlan push id 123
# tc filter replace dev eth0 parent ffff:
basic
action bpf obj bpf.o sec my-action

40. ●
●
●
●
○
●
○
●

41. ●
○
○
○
●
○
○
○
● →

42. ●
●
○

43. ●
●
●
●
●
●
●
●
●
●

44. static inline void set_tcp_dest_port(struct __sk_buff *skb, __u16 new_port)
{
__u16 old_port = htons(load_half(skb, TCP_DPORT_OFF));
bpf_skb_store_bytes(skb, TCP_DPORT_OFF, &new_port, sizeof(new_port), 0);
bpf_l4_csum_replace(skb, TCP_CSUM_OFF, old_port, new_port, sizeof(new_port));
}
__attribute__((section("redirect_xmit"), used))
int _redirect_xmit(struct __sk_buff *skb)
{
__u8 proto = load_byte(skb, ETH_HLEN + offsetof(struct iphdr, protocol));
if (proto == IPPROTO_TCP)
set_tcp_dest_port(skb, 5001);
return bpf_redirect(skb->ifindex + 1, 0);
}

45. struct bpf_map_def __attribute__((section("maps"), used)) my_map = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(u32),
.value_size = sizeof(long),
.max_entries = 256,
};
__attribute__((section("socket1"), used))
int bpf_prog1(struct __sk_buff *skb)
{
int index = load_byte(skb, ETH_HLEN + offsetof(struct iphdr, protocol));
long *value;
if (skb->pkt_type != PACKET_OUTGOING)
return 0;
value = bpf_map_lookup_elem(&my_map, &index);
if (value)
__sync_fetch_and_add(value, skb->len);
return 0;
}

46. ●
●
○
○
○
○
●
●
●
○

48. ●
●
●
●

49. ●
●
●
●
●
●
●
●