Mangle for QoS, CAKE

neki · Mon Mar 18, 2024 2:34 pm

Hi, I'm trying to set up QoS with CAKE and it's logical exercise I'm failing at...

I have two queues, one for download and one for upload, then I have two magle rules and I'm marking packets download/upload.

It is working fine for internet traffic and I'm getting very nice numbers from bufferbload test. Unfortunately it's also limiting my LAN traffic and I can not wrap my head around it, when I can use only IN interfaces.

If I try to move things over LAN, the upload queue is active and limiting the transfer speed.

# 2024-03-18 10:44:41 by RouterOS 7.14.1
# model = C53UiG+5HPaxD2HPaxD

/interface bridge
add name=bridge1 vlan-filtering=yes

/interface vlan
add interface=bridge1 name=vlan10 vlan-id=10
add interface=bridge1 name=vlan20 vlan-id=20
add interface=bridge1 name=vlan30 vlan-id=30
add interface=bridge1 name=vlan40 vlan-id=40
add interface=bridge1 name=vlan50 vlan-id=50

/interface pppoe-client
add add-default-route=yes default-route-distance=10 disabled=no interface=\
    ether2 name=pppoe-out1

/interface list
add comment="Internet connections" name=WAN
add comment="Local wireless network" name=WLAN
add comment="Local network" include=WLAN name=LAN

/interface lte apn
set [ find default=yes ] default-route-distance=20 ip-type=ipv4 \
    use-network-apn=no use-peer-dns=no

/ip pool
add name=pool10 ranges=10.10.0.100-10.10.0.200
add name=pool20 ranges=10.20.0.100-10.20.0.200
add name=pool30 ranges=10.30.0.100-10.30.0.200
add name=pool40 ranges=10.40.0.100-10.40.0.200
add name=pool50 ranges=10.50.0.100-10.50.0.200

/ip dhcp-server
add address-pool=pool10 interface=vlan10 name=server10
add address-pool=pool20 interface=vlan20 name=server20
add address-pool=pool30 interface=vlan30 name=server30
add address-pool=pool40 interface=vlan40 name=server40
add address-pool=pool50 interface=vlan50 name=server50

/queue type
add cake-diffserv=diffserv4 cake-flowmode=dual-dsthost cake-nat=yes \
    cake-rtt-scheme=internet kind=cake name=cake-download
add cake-diffserv=diffserv4 cake-flowmode=dual-srchost cake-nat=yes \
    cake-rtt-scheme=internet kind=cake name=cake-upload

/queue tree
add max-limit=5M name=queue1 packet-mark=upload parent=global queue=\
    cake-upload
add max-limit=24M name=queue2 packet-mark=download parent=global queue=\
    cake-download

/interface bridge port
add bridge=bridge1 frame-types=admit-only-vlan-tagged interface=ether1
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether3 pvid=20
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether4 pvid=10
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=wifi1 pvid=30
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=wifi2 pvid=30
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=wifi3 pvid=40
add bridge=bridge1 frame-types=admit-only-untagged-and-priority-tagged \
    interface=wifi4 pvid=50

/ipv6 settings
set disable-ipv6=yes

/interface bridge vlan
add bridge=bridge1 tagged=bridge1,ether1 untagged=ether4 vlan-ids=10
add bridge=bridge1 tagged=bridge1,ether1 untagged=ether3 vlan-ids=20
add bridge=bridge1 tagged=bridge1,ether1 untagged=wifi1,wifi2 vlan-ids=30
add bridge=bridge1 tagged=bridge1,ether1 untagged=wifi3 vlan-ids=40
add bridge=bridge1 tagged=bridge1,ether1 untagged=wifi4 vlan-ids=50

/interface list member
add interface=vlan10 list=LAN
add interface=vlan20 list=LAN
add interface=vlan30 list=LAN
add interface=vlan40 list=LAN
add interface=vlan50 list=LAN
add interface=pppoe-out1 list=WAN
add interface=lte1 list=WAN
add interface=ether2 list=WAN
add interface=wifi1 list=WLAN
add interface=wifi2 list=WLAN
add interface=wifi3 list=WLAN
add interface=wifi4 list=WLAN

/ip address
add address=10.10.0.1/24 interface=vlan10 network=10.10.0.0
add address=10.20.0.1/24 interface=vlan20 network=10.20.0.0
add address=10.30.0.1/24 interface=vlan30 network=10.30.0.0
add address=10.40.0.1/24 interface=vlan40 network=10.40.0.0
add address=10.50.0.1/24 interface=vlan50 network=10.50.0.0
add address=192.168.1.2/30 interface=ether2 network=192.168.1.0

/ip dhcp-server network
add address=10.10.0.0/24 dns-server=10.20.0.10 gateway=10.10.0.1 ntp-server=\
    10.10.0.1
add address=10.20.0.0/24 dns-server=10.20.0.10 gateway=10.20.0.1 ntp-server=\
    10.20.0.1
add address=10.30.0.0/24 dns-server=10.20.0.10 gateway=10.30.0.1 ntp-server=\
    10.30.0.1
add address=10.40.0.0/24 dns-server=10.20.0.10 gateway=10.40.0.1 ntp-server=\
    10.40.0.1
add address=10.50.0.0/24 dns-server=10.20.0.10 gateway=10.50.0.1 ntp-server=\
    10.50.0.1

/ip dns
set servers=10.20.0.10

/ip firewall filter
add action=accept chain=input connection-state=established,related,untracked
add action=drop chain=input connection-state=invalid
add action=accept chain=input protocol=icmp
add action=accept chain=input dst-port=123 protocol=udp
add action=accept chain=input dst-address=127.0.0.1
add action=accept chain=input in-interface=vlan10
add action=accept chain=input in-interface=ether5
add action=drop chain=input
add action=accept chain=forward connection-state=\
    established,related,untracked
add action=drop chain=forward connection-state=invalid
add action=accept chain=forward dst-address=10.20.0.10 dst-port=53 \
    in-interface-list=LAN protocol=tcp
add action=accept chain=forward dst-address=10.20.0.10 dst-port=53 \
    in-interface-list=LAN protocol=udp
add action=accept chain=forward in-interface-list=LAN out-interface-list=WAN
add action=accept chain=forward connection-nat-state=dstnat
add action=drop chain=forward disabled=yes

/ip firewall mangle
add action=mark-packet chain=prerouting in-interface-list=WAN \
    new-packet-mark=download passthrough=yes
add action=mark-packet chain=prerouting in-interface-list=LAN \
    new-packet-mark=upload passthrough=yes

/ip firewall nat
add action=masquerade chain=srcnat out-interface-list=WAN

Before queues:

buff1.png

With queues:

buff2.png

blacksnow · Mon Mar 18, 2024 2:50 pm

I won't comment on your config from a general perspective as there may be other things to organize/cleanup but to answer your question directly. The following rules are not specific enough, you are marking all traffic that is coming in from the WAN interface to any destination (which can only be any other network, in your case this is LAN) as download. This may be fine for your situaiton but I would argue you want to add an "out-interface" filter to ensure you are only makring packets that are trully considered downloading. Similarly you are marking all packets that are originating from your LAN network (in-interface = LAN) to any destination as upload. This would be true in the case you never had LAN -> LAN traffic (often called east - west traffic). This is why you are experiencing the issue you are facing, to fix it, simply do the below.

/ip firewall mangle
add action=mark-packet chain=forward in-interface-list=WAN out-interface-list=LAN \
    new-packet-mark=download passthrough=yes
add action=mark-packet chain=forward in-interface-list=LAN out-interface-list=WAN \
    new-packet-mark=upload passthrough=yes

The key changes are adding the out-interface lists and use forward chain since your traffic is not destined for the router.

anav · Mon Mar 18, 2024 4:49 pm

The question I have is why are you mangling or queueing at all......
You have nothing different in either direction.......... all incoming traffic goes to entire LAN, all outgoing traffic comes from entire LAN.

Okay! Its about bufferbloat.
For me I would have to weigh any advantage of bufferebloat over the loss in throughput due to losing fastrack ???

You only need Mangling for advanced queue tree menu is what I read, so SIMPLE queues should be possible without mangling!

neki · Mon Mar 18, 2024 6:16 pm

I won't comment on your config from a general perspective as there may be other things to organize/cleanup

Please do! Any suggestions are welcome...

I understand where is the issue, but couldn't find way around it, maybe little bit of tunnel vision, because I was focused on OUT interfaces but you can not use OUT interfaces with prerouting.

Anyway I think that I found solution, I can use DST address list even with prerouting.

So I did:

/ip firewall address-list
add address=10.10.0.0/24 list=LAN
add address=10.20.0.0/24 list=LAN
add address=10.30.0.0/24 list=LAN
add address=10.40.0.0/24 list=LAN
add address=10.50.0.0/24 list=LAN

/ip firewall mangle
add action=mark-packet chain=prerouting dst-address-list=!LAN \
    in-interface-list=LAN new-packet-mark=upload passthrough=yes
add action=mark-packet chain=prerouting in-interface-list=WAN \
    new-packet-mark=download passthrough=yes

infabo · Mon Mar 18, 2024 6:18 pm

You don't lose fasttrack when using queue tree...

So I'd suggest attaching the tree to the interfaces and not on root. According to your interface list use the following WAN interfaces (egress) and to your bridge (ingress). Remove the mangle rules for your custom packet marks. Marking packets only works with fasttrack disabled!

/queue tree
add max-limit=5M name=cake-egress-lte1 packet-mark=no-mark parent=lte1 queue=cake-upload
add max-limit=5M name=cake-egress-ether2 packet-mark=no-mark parent=ether2 queue=cake-upload
add max-limit=24M name=cake-ingress packet-mark=no-mark parent=bridge1 queue=cake-download

IMHO you don't need to attach a tree on pppoe-out1 as it goes through ether2 ultimately. Tune your egress queue according to your available bandwidth on lte1/ether2.

neki · Mon Mar 18, 2024 6:43 pm

Okay! Its about bufferbloat.
For me I would have to weigh any advantage of bufferebloat over the loss in throughput due to losing fastrack ???

That's what I'm thinking about, I have slow link 25/5 and fast CPU (hAP ax3) so It's better for me to use queues? Because I don't have any throughput anyway?

BTW, I knew that you will understand me:

Regardless if you need to mangle your gaming ports, then its your gaming skills that is the problem.

You don't lose fasttrack when using queue tree...

So I'd suggest attaching the tree to the interfaces and not on root. According to your interface list use the following WAN interfaces (egress) and to your bridge (ingress). Remove the mangle rules for your custom packet marks. Marking packets only works with fasttrack disabled!
Code: Select all
/queue tree
add max-limit=5M name=queue1 packet-mark=no-mark parent=pppoe-out1 queue=cake-upload
add max-limit=5M name=queue2 packet-mark=no-mark parent=lte1 queue=cake-upload
add max-limit=5M name=queue3 packet-mark=no-mark parent=ether2 queue=cake-upload
add max-limit=24M name=queue4 packet-mark=no-mark parent=bridge1 queue=cake-download
tbh I dont know if that works out correctly. No experience with multiple WAN interfaces and how to handle that properly using queue-tree.

Going to try that... LTE is just backup with FUP limit, normaly not in use, it's mainly for smart home connectivity. And ether2 is there so I can connect to VDSL modem for configuration.

infabo · Mon Mar 18, 2024 6:45 pm

Just edited my answer above. You could probably skip queue tree on interface pppoe-out-1.

neki · Mon Mar 18, 2024 7:08 pm

pppoe is fine, that's what I need, ether2 itself is used only for modem interface.

Your setup is working fine for internet traffic, but now the LAN traffic is limited by download queue (bridge1). That's probably why I ended up with magle rules.

BTW, is it possible to put another queue over that to prioritize VOIP?

infabo · Mon Mar 18, 2024 9:45 pm

Is there any used (hardware) port for LAN not part of bridge1?

neki · Mon Mar 18, 2024 10:58 pm

Yes, ether2 is connected to modem and not part of the bridge. PPPoE can not be added on slave interface.

ether5 is not used and not part of the bridge, it's meant as "disaster recovery" port

ether4 is not used and it is part of the bridge

mke · Tue Mar 19, 2024 4:14 am

A bit off topic, not specific to OPs question, but wouldn't the best way to mangle for cake be to add DSCP marks, since cake supports various diffserv groups out of the box?

Below are some flent rrul tests.

First image is my standard cake setup using queue types with:

cake-diffserv=diffserv4 cake-flowmode=dual-[src/dst]host  cake-nat=yes

Second image is using queue types with:

cake-diffserv=besteffort cake-flowmode=triple-isolate cake-nat=no

As you can see cake does a great job of queuing and prioritising traffic that has DSCP marks (which flent is adding in these examples).

I fall flat trying to set up mangling properly, but I'd love it if someone could provide some mangle rules to mark DSCP according to cakes diffserv buckets.

rrul-internet.png

rrul-besteffort-tripleisolate-nonat.png

hajes29a · Fri Jun 21, 2024 11:27 pm

I tried similar setup on 1Gbe/100Mbe Internet connection. I had traffic shaping, prioritizing, fasttrack off. Otherwise, global queues don't work. I did never find out how to manage QoS/prio with fasttrack. It seems to be impossible so far. There is a solution to get running only interface hw queues + DSCP/TOS, but who knows...queues get less traffic because it is fasttracked.

Sadly, my Chateau LTE 12 router manages only 100 Mbps with advanced firewall

Mangle for QoS, CAKE

Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE

Re: Mangle for QoS, CAKE