IPv6 MTU Discovery not working properly

mikey · Sun Apr 02, 2023 11:09 am

Good morning

I'm having some trouble with getting the IPv6 MTU Discovery to work. If I do not set the MTU at 1492 in the router advertisement most websites will not load. I'm using a PPPoE connection with an automatic MTU of 1492. I understand that some servers do not accept ICMPv6. But shouldn't my own client automatically switch to an MTU to 1492 when reaching my WAN since my own router knows that the WAN path is at 1492 MTU? Why doesn't it except if I state an MTU of 1492 in the router discovery?

What am I doing wrong?

/interface bridge
add admin-mac=2C:C8:1B:FD:72:F8 auto-mac=no comment=defconf name=bridge pvid=900 vlan-filtering=yes
/interface vlan
add interface=bridge name=Guest_vlan vlan-id=40
add interface=bridge name=IoT_vlan vlan-id=20
add interface=bridge name=Server_vlan vlan-id=30
add interface=bridge name=Trusted_vlan vlan-id=10
/interface pppoe-client
add add-default-route=yes disabled=no interface=ether1 name=pppoe-proximus user=xxxxx@PROXIMUS
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip hotspot profile
set [ find default=yes ] html-directory=hotspot
/ip pool
add name=pool_trusted ranges=10.0.1.100-10.0.1.254
add name=pool_IoT ranges=10.0.2.100-10.0.2.254
add name=pool_guest ranges=10.0.4.100-10.0.4.254
/ip dhcp-server
add address-pool=pool_trusted interface=Trusted_vlan lease-time=1w name=DHCP_trusted
add address-pool=pool_IoT interface=IoT_vlan lease-time=1w name=DHCP_IoT
add address-pool=pool_guest interface=Guest_vlan lease-time=1d name=DHCP_guest
/port
set 0 name=serial0
/interface bridge port
add bridge=bridge frame-types=admit-only-untagged-and-priority-tagged interface=ether3 pvid=10
add bridge=bridge frame-types=admit-only-vlan-tagged interface=ether4 pvid=900
add bridge=bridge interface=ether5 pvid=10
add bridge=bridge interface=sfp1
add bridge=bridge frame-types=admit-only-untagged-and-priority-tagged interface=ether2 pvid=10
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface bridge vlan
add bridge=bridge tagged=bridge,ether4 untagged=ether5,ether3,ether2 vlan-ids=10
add bridge=bridge tagged=bridge,ether4,ether5 vlan-ids=20
add bridge=bridge tagged=bridge,ether4 vlan-ids=30
add bridge=bridge tagged=bridge,ether5,ether4 vlan-ids=40
/interface list member
add interface=pppoe-proximus list=WAN
add interface=Trusted_vlan list=LAN
add interface=IoT_vlan list=LAN
add interface=Guest_vlan list=LAN
add interface=Server_vlan list=LAN
/ip address
add address=10.0.1.1/24 interface=Trusted_vlan network=10.0.1.0
add address=10.0.2.1/24 interface=IoT_vlan network=10.0.2.0
add address=10.33.3.1/24 interface=Server_vlan network=10.33.3.0
add address=10.0.4.1/24 interface=Guest_vlan network=10.0.4.0
/ip cloud
set ddns-enabled=yes
/ip dhcp-server lease
add address=10.0.1.254 client-id=1:3c:7c:3f:27:f7:e0 mac-address=3C:7C:3F:27:F7:E0 server=DHCP_trusted
/ip dhcp-server network
add address=10.0.1.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.0.1.1
add address=10.0.2.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.0.2.1
add address=10.0.4.0/24 dns-server=1.1.1.1,1.0.0.1 gateway=10.0.4.1
/ip dns
set allow-remote-requests=yes servers=1.1.1.1,1.0.0.1
/ip firewall filter
add action=accept chain=input comment="accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="drop invalid" connection-state=invalid
add action=accept chain=input comment="accept ICMP" protocol=icmp
add action=accept chain=input comment="Allow trusted vlan" in-interface=Trusted_vlan
add action=drop chain=input comment="Drop any"
add action=accept chain=forward comment="accept established,related" connection-state=established,related
add action=drop chain=forward comment="drop invalid" connection-state=invalid
add action=accept chain=forward comment="Accept LAN to WAN" in-interface-list=LAN out-interface-list=WAN
add action=drop chain=forward comment="Drop any"
/ip firewall nat
add action=masquerade chain=srcnat comment=masquerade ipsec-policy=out,none out-interface-list=WAN
/ip service
set telnet disabled=yes
set ftp disabled=yes
set ssh disabled=yes
set api disabled=yes
set api-ssl disabled=yes
/ipv6 address
add from-pool=GA_Pool interface=Trusted_vlan
add from-pool=GA_Pool interface=IoT_vlan
add from-pool=GA_Pool interface=Server_vlan
add from-pool=GA_Pool interface=Guest_vlan
/ipv6 dhcp-client
add interface=pppoe-proximus pool-name=GA_Pool request=prefix
/ipv6 firewall address-list
add address=::/128 comment="defconf: unspecified address" list=bad_ipv6
add address=::1/128 comment="defconf: lo" list=bad_ipv6
add address=fec0::/10 comment="defconf: site-local" list=bad_ipv6
add address=::ffff:0.0.0.0/96 comment="defconf: ipv4-mapped" list=bad_ipv6
add address=::/96 comment="defconf: ipv4 compat" list=bad_ipv6
add address=100::/64 comment="defconf: discard only " list=bad_ipv6
add address=2001:db8::/32 comment="defconf: documentation" list=bad_ipv6
add address=2001:10::/28 comment="defconf: ORCHID" list=bad_ipv6
add address=3ffe::/16 comment="defconf: 6bone" list=bad_ipv6
/ipv6 firewall filter
add action=accept chain=input comment="accept established,related" connection-state=established,related
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept ICMPv6" protocol=icmpv6
add action=accept chain=input comment="defconf: accept UDP traceroute" port=33434-33534 protocol=udp
add action=accept chain=input comment="defconf: accept DHCPv6-Client prefix delegation." dst-port=546 protocol=udp src-address=fe80::/10
add action=accept chain=forward comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=forward comment="defconf: drop packets with bad src ipv6" src-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: drop packets with bad dst ipv6" dst-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: rfc4890 drop hop-limit=1" hop-limit=equal:1 protocol=icmpv6
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=icmpv6
add action=accept chain=forward comment="defconf: accept HIP - to research if really needed" disabled=yes protocol=139
add action=accept chain=forward comment="Accept LAN to WAN" in-interface-list=LAN out-interface-list=WAN
add action=drop chain=forward comment="Drop any"
add action=drop chain=input comment="Drop any"
/ipv6 nd
set [ find default=yes ] dns=2606:4700:4700::1111,2606:4700:4700::1001 mtu=1492
/system clock
set time-zone-name=Europe/Brussels
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN

DarkNate · Sun Apr 02, 2023 12:19 pm

Your ISP is filtering ICMPv6 and breaking PMTUD. Or they misconfigured MTU for PPPoE underlay (physical backbone) and overlay (server) on their side.

Ask them to deploy RFC4638 as per this guide:
viewtopic.php?t=176358

Znevna · Sun Apr 02, 2023 2:22 pm

Or just do some tests and gather proof before blaming the ISP blindly.

mikey · Sun Apr 02, 2023 5:20 pm

Or just do some tests and gather proof before blaming the ISP blindly.

This interest me. What tests could I do? Is something wrong in my config?

Znevna · Sun Apr 02, 2023 8:44 pm

Cloudflare has a few articles about this on their blog, I've mentioned them on other topics on this forum about IPv6 and MTU:
This: https://blog.cloudflare.com/path-mtu-di ... -practice/
And this: https://blog.cloudflare.com/ip-fragmentation-is-broken/
And this: https://blog.cloudflare.com/increasing-ipv6-mtu/
Short version, open this (for IPv6): http://icmpcheckv6.popcount.org/
If everything is fine regarding PMTUD you should see after a few seconds "All good! ICMP path MTU message was successfully delivered to you."
And under windows you can check the destination cache for that particular website, listing the discovered PMTU (a2e9 is part of the suffix of the current IPv6 addr of that website):

netsh int ipv6 show destinationcache | find "a2e9"

Which should return this (output is in format: 'PMTU' 'Destination Address' 'Next Hop Address'):

1285 2a01:7e01::f03c:91ff:fe16:a2e9                fe80::ce2d:e0ff:fe06:7166

You can also fire up Wireshark and filter capture only by "ip6" and watch what's going on.

mikey · Sun Apr 02, 2023 11:09 pm

Cloudflare has a few articles about this on their blog, I've mentioned them on other topics on this forum about IPv6 and MTU:
This: https://blog.cloudflare.com/path-mtu-di ... -practice/
And this: https://blog.cloudflare.com/ip-fragmentation-is-broken/
And this: https://blog.cloudflare.com/increasing-ipv6-mtu/
Short version, open this (for IPv6): http://icmpcheckv6.popcount.org/
If everything is fine regarding PMTUD you should see after a few seconds "All good! ICMP path MTU message was successfully delivered to you."
And under windows you can check the destination cache for that particular website, listing the discovered PMTU (a2e9 is part of the suffix of the current IPv6 addr of that website):
Code: Select all
netsh int ipv6 show destinationcache | find "a2e9"
Which should return this (output is in format: 'PMTU' 'Destination Address' 'Next Hop Address'):
Code: Select all
1285 2a01:7e01::f03c:91ff:fe16:a2e9                fe80::ce2d:e0ff:fe06:7166
You can also fire up Wireshark and filter capture only by "ip6" and watch what's going on.

Thank you for your answer. If I don't fill in the MTU in the router advertisement, I cannot even reach http://icmpcheckv6.popcount.org/. It loads indefinitely. When I fill in 1492 MTU in the router advertisement, I can reach it and it tells me everything is working as expected.

Znevna · Sun Apr 02, 2023 11:17 pm

Interesting, did you try to disable your IPv6 firewall drop rules for that test? Saw you have a custom set of rules there.
Also what OS is your test client running?

mikey · Sun Apr 02, 2023 11:40 pm

My client is running Windows 11. I tried the http://icmpcheckv6.popcount.org also on Windows 10 and one android device all with the same results.

I just tried to disable the drop rules in the forward and input rule set. Sadly, this had exactly the same results.

What I also noticed when not setting the MTU manually to 1492 in the router advertisement is that most website will not load (like icmpcheckv6, Mikrotik forum, ...) . However, a selected few like google will load fine. IPv4 was also disabled for this test to make sure it does not fall back on it.

What confuses me, is that the PMTU returns 1500 for the working google website when I do not manually set the MTU to 1492 in the RA.

netsh int ipv6 show destinationcache | find "63" 
1500 2a00:1450:400c:c01::63                        fe80::2ec8:1bff:fefd:72f8

Znevna · Mon Apr 03, 2023 12:59 pm

You'll have to open a support ticket with your ISP so that they don't block ICMPv6 communication in order to allow PMTUD to work, this might be an easier task for them instead of adding support for RFC4638, and you'll want working ICMPv6 either way.
Regarding google, it uses QUIC which is a totally different beast than TCP, and they use a slightly lower, forced MTU, to avoid issues with broken ISPs

so PMTUD doesn't kick in and "discovered" MTU remains equal to default interface MTU.

mikey · Tue Apr 04, 2023 11:47 am

Thank you allot for your help. I just had a constructive call with my ISP, now let's hope they will implement the changes. Till then I will keep my MTU on 1492 as a work around.

Znevna · Tue Apr 04, 2023 9:07 pm

I also have several PPPoE wan connections with MTU limited to 1492, since the ISP doesn't support rfc4638 everywhere.
But even if ICMP isn't blocked and PMTUD works fine, I keep 1492 in RA settings, because not everything is TCP out there and even if you fix your side of things there's no guarantee that you won't encounter issues with certain websites that do block ICMP.
Also, even if PMTUD works perfectly fine there is a slight delay that I'm uncomfortable with when the PMTU for that destination is no longer cached and you try to access that website and PMTUD kicks in and you have to wait untill both sides agree on the discovered MTU, again.. see my point?
PMTUD isn't some magical perfect solution.
Sure, 1500 MTU everywhere would be perfect, but not everywhere we can have that.

IPv6 MTU Discovery not working properly [SOLVED]

IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly [SOLVED]

Re: IPv6 MTU Discovery not working properly

Re: IPv6 MTU Discovery not working properly