Very slow intervlan routing

mikey · Wed Mar 02, 2022 8:36 pm

Good afternoon

Firmware: 7.1rc7
Device: Hex S

I recently noticed I only get 200Mb/s intervlan routing. The cpu is only hitting 40-50%. Is there a way to get the CPU to be utilized more? I also noticed the fast track is not working when doing inter vlan routing.

I have checked the cables and devices using an unmanaged switch and they all hit 1Gbps.

# mar/02/2022 16:33:10 by RouterOS 7.1rc7
# software id = *****
#
# model = RB760iGS
/interface bridge
add ingress-filtering=no name=lan-bridge pvid=999 vlan-filtering=yes
/interface ethernet
set [ find default-name=ether5 ] poe-out=forced-on
/interface vlan
add interface=lan-bridge name=IoT-vlan-interface vlan-id=20
add interface=lan-bridge name=guest-vlan-interface vlan-id=40
add interface=lan-bridge name=main-vlan-interface vlan-id=10
add interface=lan-bridge name=server-vlan-interface vlan-id=30
add interface=lan-bridge name=wifi-vlan-interface vlan-id=50
/interface pppoe-client
add add-default-route=yes disabled=no interface=ether1 name=WAN_Proximus_PPoE \
    use-peer-dns=yes user=****
/interface list
add comment="List with all the my vlan interfaces" name=List_vlan_interfaces
add comment=\
    "WAN list to use in firewall. Makes the changing of WAN much easier" \
    name=WAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=dhcp-pool-main-network ranges=10.0.1.101-10.0.1.254
add name=dhcp-pool-IoT-network ranges=10.0.2.101-10.0.2.254
add name=dhcp-pool-guest-network ranges=10.0.4.101-10.0.4.254
add name=dhcp-pool-wifi-network ranges=10.0.5.101-10.0.5.254
/ip dhcp-server
add address-pool=dhcp-pool-main-network interface=main-vlan-interface \
    lease-time=2d name=dhcp-main-network
add address-pool=dhcp-pool-IoT-network interface=IoT-vlan-interface \
    lease-time=2d name=dhcp-IoT-network
add address-pool=dhcp-pool-guest-network interface=guest-vlan-interface \
    lease-time=1d name=dhcp-guest-network
add address-pool=dhcp-pool-wifi-network interface=wifi-vlan-interface \
    lease-time=2d name=dhcp-wifi-network
/port
set 0 name=serial0
/interface bridge port
add bridge=lan-bridge interface=ether3 pvid=10
add bridge=lan-bridge frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether4 pvid=10
add bridge=lan-bridge frame-types=admit-only-untagged-and-priority-tagged \
    interface=ether2 pvid=10
add bridge=lan-bridge interface=ether5 pvid=10
/ip neighbor discovery-settings
set discover-interface-list=!dynamic
/ip settings
set tcp-syncookies=yes
/interface bridge vlan
add bridge=lan-bridge comment="main vlan " tagged=lan-bridge untagged=\
    ether4,ether2,ether3,ether5 vlan-ids=10
add bridge=lan-bridge comment="IoT vlan" tagged=ether5,lan-bridge vlan-ids=20
add bridge=lan-bridge comment="server vlan" tagged=lan-bridge,ether3 \
    vlan-ids=30
add bridge=lan-bridge comment="guest vlan" tagged=lan-bridge,ether5 vlan-ids=\
    40
add bridge=lan-bridge comment="wifi vlan" tagged=ether5,lan-bridge vlan-ids=\
    50
/interface list member
add interface=IoT-vlan-interface list=List_vlan_interfaces
add interface=guest-vlan-interface list=List_vlan_interfaces
add interface=main-vlan-interface list=List_vlan_interfaces
add interface=server-vlan-interface list=List_vlan_interfaces
add interface=WAN_Proximus_PPoE list=WAN
add interface=wifi-vlan-interface list=List_vlan_interfaces
/ip address
add address=10.0.1.1/24 comment="gateway main vlan" interface=\
    main-vlan-interface network=10.0.1.0
add address=10.0.2.1/24 comment="gateway IoT vlan" interface=\
    IoT-vlan-interface network=10.0.2.0
add address=10.0.3.1/24 comment="gateway server vlan" interface=\
    server-vlan-interface network=10.0.3.0
add address=10.0.4.1/24 comment="gateway guest vlan" interface=\
    guest-vlan-interface network=10.0.4.0
add address=10.0.5.1/24 comment="gateway wifi vlan" interface=\
    wifi-vlan-interface network=10.0.5.0
/ip dhcp-server network
add address=10.0.1.0/24 comment="main network" dns-server=1.1.1.1,1.0.0.1 \
    gateway=10.0.1.1
add address=10.0.2.0/24 comment="IoT network" dns-server=1.1.1.1,1.0.0.1 \
    gateway=10.0.2.1
add address=10.0.4.0/24 comment="Guest network" dns-server=1.1.1.1,1.0.0.1 \
    gateway=10.0.4.1
add address=10.0.5.0/24 comment="Wifi network" dns-server=1.1.1.1,1.0.0.1 \
    gateway=10.0.5.1
/ip dns
set servers=1.1.1.1,8.8.8.8,8.8.4.4
/ip firewall address-list
add address=0.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=172.16.0.0/12 comment=RFC6890 list=not_in_internet
add address=192.168.0.0/16 comment=RFC6890 list=not_in_internet
add address=10.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=169.254.0.0/16 comment=RFC6890 list=not_in_internet
add address=127.0.0.0/8 comment=RFC6890 list=not_in_internet
add address=224.0.0.0/4 comment=Multicast list=not_in_internet
add address=198.18.0.0/15 comment=RFC6890 list=not_in_internet
add address=192.0.0.0/24 comment=RFC6890 list=not_in_internet
add address=192.0.2.0/24 comment=RFC6890 list=not_in_internet
add address=198.51.100.0/24 comment=RFC6890 list=not_in_internet
add address=203.0.113.0/24 comment=RFC6890 list=not_in_internet
add address=100.64.0.0/10 comment=RFC6890 list=not_in_internet
add address=240.0.0.0/4 comment=RFC6890 list=not_in_internet
add address=192.88.99.0/24 comment="6to4 relay Anycast [RFC 3068]" list=\
    not_in_internet
add address=10.0.1.0/24 list=vlan_network_address_list
add address=10.0.2.0/24 list=vlan_network_address_list
add address=10.0.3.0/24 list=vlan_network_address_list
add address=10.0.4.0/24 list=vlan_network_address_list
add address=10.0.5.0/24 list=vlan_network_address_list
/ip firewall filter
add action=accept chain=input comment="allow established related connections" \
    connection-state=established,related in-interface=main-vlan-interface
add action=accept chain=input comment="allow acces from main vlan" \
    in-interface=main-vlan-interface
add action=accept chain=input comment="allow icmp" protocol=icmp
add action=drop chain=input comment="drop all others"
add action=fasttrack-connection chain=forward comment=\
    "FastTrack established, related connections" connection-state=\
    established,related hw-offload=yes
add action=accept chain=forward comment=\
    "accept established, related connections" connection-state=\
    established,related
add action=accept chain=forward comment="accept NAT'ed connections" \
    connection-nat-state=srcnat in-interface-list=WAN
add action=accept chain=forward comment="Main-vlan to Challenger NAS" \
    dst-address=10.0.3.2 src-address=10.0.1.0/24
add action=accept chain=forward comment="Main-vlan to Challenger NAS" \
    dst-address=10.0.3.5 src-address=10.0.1.0/24
add action=drop chain=forward comment="Drop intervlan routing" \
    dst-address-list=vlan_network_address_list src-address-list=\
    vlan_network_address_list
add action=drop chain=forward comment=\
    "drop reaching private addresses via WAN" dst-address-list=\
    not_in_internet out-interface-list=WAN
add action=drop chain=forward comment="Drop invalid connections " \
    connection-state=invalid
add action=accept chain=forward comment="Allow vlans to internet" \
    out-interface-list=WAN src-address-list=vlan_network_address_list
add action=jump chain=forward comment="jump to ICMP filters" jump-target=icmp \
    protocol=icmp
add action=accept chain=icmp comment="echo reply" icmp-options=0:0 protocol=\
    icmp
add action=accept chain=icmp comment="net unreachable" icmp-options=3:0 \
    protocol=icmp
add action=accept chain=icmp comment="host unreachable" icmp-options=3:1 \
    protocol=icmp
add action=accept chain=icmp comment=\
    "host unreachable fragmentation required" icmp-options=3:4 protocol=icmp
add action=accept chain=icmp comment="allow echo request" icmp-options=8:0 \
    protocol=icmp
add action=accept chain=icmp comment="allow time exceed" icmp-options=11:0 \
    protocol=icmp
add action=accept chain=icmp comment="allow parameter bad" icmp-options=12:0 \
    protocol=icmp
add action=drop chain=icmp comment="deny all other types"
add action=drop chain=forward comment="Drop all the rest"
/ip firewall nat
add action=masquerade chain=srcnat out-interface-list=WAN
/ip service
set telnet disabled=yes
set ftp disabled=yes
set ssh disabled=yes
set api disabled=yes
set api-ssl disabled=yes
/system clock
set time-zone-name=Europe/Brussels

Buckeye · Thu Mar 03, 2022 5:18 am

I am trying to learn vlans on ROS too. I just recently got a hEX S. I know Ubiquiti ER-X much better, but I am starting to understand how ROS does things.
You may be hitting a Single core limit, the MT7621A SOC has 2 full cores (hyper threaded to 4).
If I am understanding your config corrrectly, all "bridge ports" (2-5) are set with pvid=10 untagged and ether3 (tagged 30) and ether5 (tagged 20, 40, 50) are hybrid ports.
Can you show a diagram of how things are connected?
Does

/interface bridge port print

show H indicating hardware switching enabled?
How are you testing? iperf3? Copying large file from NAS?
Have you tested throughput on untagged vlan 10 between two ports? I realize this traffic won't be routed, but just want to make sure this traffic isn't going through the CPU.
What is connected to the two hybrid ports, ether3 and ether5? Access points, or vlan-aware switch?
Did you test between vlan 10 and 20 from two different ports, as well as from the same port (ether5) Also between port3 vlan30 and port5 vlan50?

elbob2002 · Thu Mar 03, 2022 9:20 am

You're using a bridge based VLAN which is only really supported on a limited number of newer devices. While it works it does result in the performance you are seeing.

mikey · Thu Mar 03, 2022 9:56 am

You're using a bridge based VLAN which is only really supported on a limited number of newer devices. While it works it does result in the performance you are seeing.

How is the right way to configure it? And why is it only using 50% of the CPU instead of 100%?

mikey · Thu Mar 03, 2022 10:08 am

I am trying to learn vlans on ROS too. I just recently got a hEX S. I know Ubiquiti ER-X much better, but I am starting to understand how ROS does things.
You may be hitting a Single core limit, the MT7621A SOC has 2 full cores (hyper threaded to 4).
If I am understanding your config corrrectly, all "bridge ports" (2-5) are set with pvid=10 untagged and ether3 (tagged 30) and ether5 (tagged 20, 40, 50) are hybrid ports.
Can you show a diagram of how things are connected?
Does
Code: Select all
/interface bridge port print
show H indicating hardware switching enabled?
How are you testing? iperf3? Copying large file from NAS?
Have you tested throughput on untagged vlan 10 between two ports? I realize this traffic won't be routed, but just want to make sure this traffic isn't going through the CPU.
What is connected to the two hybrid ports, ether3 and ether5? Access points, or vlan-aware switch?
Did you test between vlan 10 and 20 from two different ports, as well as from the same port (ether5) Also between port3 vlan30 and port5 vlan50?

Good morning

The single core limit might be it. Any way to make it use the two cores?

Vlan 10: Trusted
Vlan 20: IoT
Vlan 30: Servers
Vlan 40: Guest
Vlan 50: Trusted wifi

Port 1: WAN
Port 2: PC
Port 3: virtual switch inside Hyper-V Hypervisor (2nd pc on vlan 10, Virtual NAS on vlan 30)
Port 4: TV
Port 5: Access point

I tested with iperf3 and with a large file. Hardware switching is enabled (H is visible).
The throughput between devices on the same vlan using the hex S is 1Gbp/s (Tested from PC to Server and from PC to NAS).
Thank you for your help

jookraw · Thu Mar 03, 2022 10:36 am

You're using a bridge based VLAN which is only really supported on a limited number of newer devices. While it works it does result in the performance you are seeing.
How is the right way to configure it? And why is it only using 50% of the CPU instead of 100%?

50% of all cores combined, means 100% one core is being used. you may check by looking:

/tool profile cpu=all

or

/system/resource/cpu/print

The reason of slow speed may be the the non-ARM cpu and the lack of route cache on rOS 7.x.
Mikrotik seems to be moving to ARM SoC's and I don't think that other/legacy devices will become any better.

mikey · Thu Mar 03, 2022 11:05 am

How is the right way to configure it? And why is it only using 50% of the CPU instead of 100%?
50% of all cores combined, means 100% one core is being used. you may check by looking:
Code: Select all
/tool profile cpu=all
or
Code: Select all
/system/resource/cpu/print
The reason of slow speed may be the the non-ARM cpu and the lack of route cache on rOS 7.x.
Mikrotik seems to be moving to ARM SoC's and I don't think that other/legacy devices will become any better.

I have indeed 1 of the 4 threads running at 100%

0  cpu0  29%   29%   0%  
1  cpu1  1%    0%    0%  
2  cpu2  59%   59%   0%  
3  cpu3  100%  100%  0%

So what is the solution? Is there anything I can do on routerOS or do I have to get a managed switch and trunk everything to the cpu instead of using bridge based vlan? Would I then get 1Gbps intervlan routing?

Buckeye · Thu Mar 03, 2022 12:47 pm

If the bottleneck is routing, I don't see how using an external switch will help, since inter-vlan traffic needs to be routed (through the CPU).
Since vlan 10 is trusted. can your hyper-V allow a second interface on the virtual NAS, with an virtual interface in vlan 10? Then the largest consumer of the data (PC's?) will be on the same layer 2 network as the NAS, and no routing will be required by devices on vlan 10, it will just be switched.
You will loose all ability to firewall in the hEX S, but you may be able to firewall at the virtual NAS.
Do you trust your TV to be on the trusted network? It is currently on vlan 10. I would have expected it to be on the IoT network. But perhaps it is streaming from the NAS?
I don't know enough about ROS to be much more help, it's like the blind leading the blind.
It seems that jookray suggested that route caching (is that fasttrack?) may have been available in 6. But the hardware assisted bridging wasn't for the RB760iGS until 7.1r5, so if you downgraded you would probably need to use an external switch or possibly configure vlans with the switch section (which is no longer the recommendation, if I understand correctly).
Or get a faster router.
Unless there is a way to enable fast path/fasttrack, and I don't know if that works on the hEX S or not. I haven't gotten that far yet.

jookraw · Thu Mar 03, 2022 1:04 pm

route cache was removed by the Linux kernel, it is not coming back.

I've looked again the config, it seems that should be enabled. you could try using Jumbo frames internally, if supported.
if you really need near 1g or more, I think that the only way is an RB5009 or RB4011, as they are way more capable.
for e.g. my RB5009 does reach at least 2.5g between vlans, I do not have 2 10g devices to test

mikey · Thu Mar 03, 2022 2:37 pm

If the bottleneck is routing, I don't see how using an external switch will help, since inter-vlan traffic needs to be routed (through the CPU).
Since vlan 10 is trusted. can your hyper-V allow a second interface on the virtual NAS, with an virtual interface in vlan 10? Then the largest consumer of the data (PC's?) will be on the same layer 2 network as the NAS, and no routing will be required by devices on vlan 10, it will just be switched.
You will loose all ability to firewall in the hEX S, but you may be able to firewall at the virtual NAS.
Do you trust your TV to be on the trusted network? It is currently on vlan 10. I would have expected it to be on the IoT network. But perhaps it is streaming from the NAS?
I don't know enough about ROS to be much more help, it's like the blind leading the blind.
It seems that jookray suggested that route caching (is that fasttrack?) may have been available in 6. But the hardware assisted bridging wasn't for the RB760iGS until 7.1r5, so if you downgraded you would probably need to use an external switch or possibly configure vlans with the switch section (which is no longer the recommendation, if I understand correctly).
Or get a faster router.
Unless there is a way to enable fast path/fasttrack, and I don't know if that works on the hEX S or not. I haven't gotten that far yet.

The TV is on vlan 10 for casting. However I haven't really used casting so I might as well set it on the IoT vlan. I will put the NAS with a 2nd interface on the Trusted vlan. I just find this a "dirty" solution.

route cache was removed by the Linux kernel, it is not coming back.

I've looked again the config, it seems that should be enabled. you could try using Jumbo frames internally, if supported.
if you really need near 1g or more, I think that the only way is an RB5009 or RB4011, as they are way more capable.
for e.g. my RB5009 does reach at least 2.5g between vlans, I do not have 2 10g devices to test

1Gbps with fast track would be ideal. If fast track does not work on inter vlan routing I would like to at least get the advertised 380Mb/s. Right now I am at half that speed.
The RB5009 does not have a POE out which I use for my access point. And I do not really want to buy an older RB4011 which is at the same price of the RB5009.

I still have a hard time understanding why the fast track does not work when doing intervlan routing but does work with the wan to lan.

mikey · Thu Mar 03, 2022 3:40 pm

I was reading the changelogs of the routerOS versions above mine and found this line

*) bridge - added fast-path and inter-VLAN routing FastTrack support when vlan-filtering is enabled;

I will give this version a try.

mikey · Thu Mar 03, 2022 4:16 pm

Good news. It is working now

1Gbps/s intervlan routing with hardware offloading and vlan-filtering on my little hEX S.

Buckeye · Fri Mar 04, 2022 12:10 am

Good to hear you got it working.
It took me a bit of time to find the reference you quoted; it is in the Testing release tree release notes https://mikrotik.com/download/changelog ... lease-tree

What's new in 7.2rc2 (2022-Jan-28 11:00):
*) bridge - added fast-path and inter-VLAN routing FastTrack support when vlan-filtering is enabled;

I am using my hEX in a lab situation (trying to find a solution to be able to use hEX S where we use ER-X's) and since I am starting fresh with ROS, I am focusing on v7, and have the latest testing version loaded. In the latest version, I did notice there are quite a few bridge fixes; note the first one "*) bridge - fixed FastPath when using "frame-types=admit-only-untagged-and-priority-tagged" setting;"

What's new in 7.2rc4 (2022-Feb-22 13:37):
*) bridge - fixed FastPath when using "frame-types=admit-only-untagged-and-priority-tagged" setting;
*) bridge - fixed IP address on untagged bridge interface when vlan-filtering is enabled (introduced in v7.2rc2);
*) bridge - fixed PPPoE packet forwarding when using "use-ip-firewall-for-pppoe" setting;
*) bridge - fixed destination NAT when using "use-ip-firewall" setting;
*) bridge - fixed filter and NAT "set-priority" on ARM64 devices;
*) bridge - fixed filter rules when using interface lists;
*) bridge - fixed priority tagged frame forwarding when using "frame-types=admit-only-untagged-and-priority-tagged" setting;

But I wouldn't use the testing version in a production environment. I probably would wait at a minimum until it reached stable, but normally I wait for "software" to mature a bit before moving to production, so I prefer the long term releases. But if you are using vlans on bridge with the vlan-filtering option on a RB760iGS or RB750r3, you will have poor performance unless you are using the hardware assist. And as you discovered, without FastTrack, routing performance won't be stellar either.

minho · Fri Mar 04, 2022 3:29 pm

Good news. It is working now 1Gbps/s intervlan routing with hardware offloading and vlan-filtering on my little hEX S.

could you upload here your current functional configuration, please

Zacharias · Fri Mar 04, 2022 9:21 pm

Inter-Vlan hardware Routing https://help.mikrotik.com/docs/display/ ... LANRouting

Very slow intervlan routing [SOLVED]

Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing [SOLVED]

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing

Re: Very slow intervlan routing