Open VPN Server Issue on a Dual WAN setup

ashishwadekar · Wed Feb 16, 2022 2:27 pm

Hi there!

I have a dual WAN setup with PCC & NTH. I wanted to add Open VPN Server & have completed configuration for the same.

OpenVPN client is not able to connect. I can see that packets are flowing till the dst-nat rule but going nowhere from there or somewhere that I am not able to understand

I am sharing my config here:

# feb/16/2022 17:43:11 by RouterOS 6.49.2
# software id = M149-EZZ8
#
# model = RB4011iGS+
# serial number = F0380F91C2B8
/interface bridge
add admin-mac=DC:2C:6E:32:27:9D auto-mac=no comment=defconf name=bridge
/interface ethernet
set [ find default-name=ether1 ] advertise=1000M-half,1000M-full comment=\
    "Primary ISP" name=WAN1
set [ find default-name=ether2 ] comment="Secondary ISP" name=WAN2
set [ find default-name=ether9 ] advertise=1000M-full
/interface pppoe-client
add disabled=no interface=WAN1 name=WAN1_PPPOEOUT password=password \
    use-peer-dns=yes user=username
add disabled=no interface=WAN2 name=WAN2_PPPOEOUT password=password \
    use-peer-dns=yes user=username
/interface ethernet switch port
set 0 default-vlan-id=0
set 1 default-vlan-id=0
set 2 default-vlan-id=0
set 3 default-vlan-id=0
set 4 default-vlan-id=0
set 5 default-vlan-id=0
set 6 default-vlan-id=0
set 7 default-vlan-id=0
set 8 default-vlan-id=0
set 9 default-vlan-id=0
set 10 default-vlan-id=0
set 11 default-vlan-id=0
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip pool
add name=default-dhcp ranges=172.16.1.10-172.16.1.254
add name=open_vpn_pool ranges=172.17.1.2-172.17.1.250
/ip dhcp-server
add address-pool=default-dhcp disabled=no interface=bridge name=defconf
/ppp profile
add local-address=172.17.1.1 name=open_vpn remote-address=open_vpn_pool
/interface bridge port
add bridge=bridge comment=defconf interface=ether3
add bridge=bridge comment=defconf interface=ether4
add bridge=bridge comment=defconf interface=ether5
add bridge=bridge comment=defconf interface=ether6
add bridge=bridge comment=defconf interface=ether7
add bridge=bridge comment=defconf interface=ether8
add bridge=bridge comment=defconf interface=ether9
add bridge=bridge comment=defconf interface=ether10
add bridge=bridge comment=defconf interface=sfp-sfpplus1
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface list member
add comment=defconf interface=bridge list=LAN
add interface=WAN1_PPPOEOUT list=WAN
add interface=WAN2_PPPOEOUT list=WAN
/interface ovpn-server server
set auth=sha1 certificate=Server cipher=aes256 default-profile=open_vpn \
    enabled=yes require-client-certificate=yes
/ip address
add address=172.16.1.1/24 comment=defconf interface=bridge network=172.16.1.0
/ip dhcp-client
add comment=defconf disabled=no interface=WAN1
/ip dhcp-server lease
add address=172.16.1.2 mac-address=E0:63:DA:5C:5B:75 use-src-mac=yes
/ip dhcp-server network
add address=172.16.1.0/24 comment=defconf gateway=172.16.1.1
/ip dns
set allow-remote-requests=yes servers=1.0.0.2
/ip dns static
add address=192.168.88.1 comment=defconf name=router.lan
/ip firewall address-list
add address=172.16.1.0/24 list=not_in_internet
add address=192.168.2.0/24 list=not_in_internet
/ip firewall filter
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input comment="Allow OpenVPN Connection" dst-port=\
    1194 protocol=tcp
add action=accept chain=input comment="Allow Input by OVPN Clients" \
    in-interface=all-ppp
add action=drop chain=input comment="defconf: drop all not coming from LAN" \
    in-interface-list=!LAN
add action=accept chain=forward comment="defconf: accept in ipsec policy" \
    ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" \
    ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
    connection-state=new in-interface-list=WAN
/ip firewall mangle
add action=mark-connection chain=prerouting comment="WAN1 Connection Markimg" \
    connection-mark=no-mark in-interface=WAN1_PPPOEOUT new-connection-mark=\
    WAN1_CONN passthrough=no
add action=mark-connection chain=prerouting comment="WAN2 Connection Markimg" \
    connection-mark=no-mark in-interface=WAN2_PPPOEOUT new-connection-mark=\
    WAN2_CONN passthrough=no
add action=mark-connection chain=prerouting comment=\
    "PCC Rule for WAN1 HTTP, HTTPS traffic" connection-mark=no-mark \
    dst-address-list=!not_in_internet dst-address-type=!local dst-port=80,443 \
    in-interface=bridge new-connection-mark=WAN1_CONN passthrough=yes \
    per-connection-classifier=both-addresses-and-ports:2/0 protocol=tcp
add action=mark-connection chain=prerouting comment=\
    "PCC Rule for WAN1 HTTP3 QUIC traffic" connection-mark=no-mark \
    dst-address-list=!not_in_internet dst-address-type=!local dst-port=80,443 \
    in-interface=bridge new-connection-mark=WAN1_CONN passthrough=yes \
    per-connection-classifier=both-addresses-and-ports:2/0 protocol=udp
add action=mark-connection chain=prerouting comment="NTH Rule for WAN1" \
    connection-mark=no-mark dst-address-list=!not_in_internet \
    dst-address-type=!local in-interface=bridge new-connection-mark=WAN1_CONN \
    nth=2,1 passthrough=yes
add action=mark-connection chain=prerouting comment=\
    "PCC Rule for WAN2 HTTP, HTTPS traffic" connection-mark=no-mark \
    dst-address-list=!not_in_internet dst-address-type=!local dst-port=80,443 \
    in-interface=bridge new-connection-mark=WAN2_CONN passthrough=yes \
    per-connection-classifier=both-addresses-and-ports:2/1 protocol=tcp
add action=mark-connection chain=prerouting comment=\
    "PCC Rule for WAN2 HTTP3 QUIC traffic" connection-mark=no-mark \
    dst-address-list=!not_in_internet dst-address-type=!local dst-port=80,443 \
    in-interface=bridge new-connection-mark=WAN2_CONN passthrough=yes \
    per-connection-classifier=both-addresses-and-ports:2/0 protocol=udp
add action=mark-connection chain=prerouting comment="NTH Rule for WAN2" \
    connection-mark=no-mark dst-address-list=!not_in_internet \
    dst-address-type=!local in-interface=bridge new-connection-mark=WAN2_CONN \
    nth=2,2 passthrough=yes
add action=mark-routing chain=prerouting comment="LAN Routing for WAN1" \
    connection-mark=WAN1_CONN in-interface=bridge new-routing-mark=TO_WAN1 \
    passthrough=no
add action=mark-routing chain=prerouting comment="LAN Routing for WAN2" \
    connection-mark=WAN2_CONN in-interface=bridge new-routing-mark=TO_WAN2 \
    passthrough=no
add action=mark-routing chain=output comment=\
    "Routing for Incoming traffic from WAN1" connection-mark=WAN1_CONN \
    new-routing-mark=TO_WAN1 out-interface=WAN1_PPPOEOUT passthrough=no
add action=mark-routing chain=output comment=\
    "Routing for Incoming traffic from WAN2" connection-mark=WAN2_CONN \
    new-routing-mark=TO_WAN2 out-interface=WAN2_PPPOEOUT passthrough=no
/ip firewall nat
add action=masquerade chain=srcnat comment="WAN1 Masquerade" out-interface=\
    WAN1_PPPOEOUT src-address=172.16.1.0/24
add action=masquerade chain=srcnat comment="WAN2 Masquerade" out-interface=\
    WAN2_PPPOEOUT src-address=172.16.1.0/24
add action=dst-nat chain=dstnat comment=\
    "TCP 80,443 Syncthing(22000) RDP(25000) WAN1" dst-port=80,443,22000,25000 \
    in-interface=WAN1_PPPOEOUT log-prefix="TCP FWD" protocol=tcp \
    to-addresses=172.16.1.2
add action=dst-nat chain=dstnat comment=\
    "TCP 80,443 Syncthing(22000) RDP(25000) WAN2" dst-port=80,443,22000,25000 \
    in-interface=WAN2_PPPOEOUT log-prefix="TCP FWD" protocol=tcp \
    to-addresses=172.16.1.2
add action=dst-nat chain=dstnat comment="UDP L2TP IPSEC(500,1701,4500) WAN1" \
    dst-port=500,1701,4500,25000 in-interface=WAN1_PPPOEOUT log-prefix=\
    "UDP FWD WAN1" protocol=udp to-addresses=172.16.1.2
add action=dst-nat chain=dstnat comment="UDP L2TP IPSEC(500,1701,4500) WAN2" \
    dst-port=500,1701,4500,25000 in-interface=WAN2_PPPOEOUT log-prefix=\
    "UDP FWD WAN2" protocol=udp to-addresses=172.16.1.2
add action=dst-nat chain=dstnat comment="OpenVPN Server" dst-port=1194 \
    in-interface=WAN2_PPPOEOUT log=yes log-prefix=OPEN_VPN protocol=tcp \
    to-addresses=172.17.1.1
add action=masquerade chain=srcnat comment="Open VPN Masquerade" \
    out-interface=WAN2_PPPOEOUT src-address=172.17.1.0/24
/ip route
add check-gateway=ping comment="Load Balance to ISP1" distance=1 gateway=\
    WAN1_PPPOEOUT routing-mark=TO_WAN1
add check-gateway=ping comment="Load Balance to ISP2" distance=1 gateway=\
    WAN2_PPPOEOUT routing-mark=TO_WAN2
/ppp secret
add name=username password=userpassword profile=open_vpn service=ovpn
/system clock
set time-zone-name=Asia/Kolkata
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN

Also sharing OPEN VPN client log:

# OPENVPN Log begins
2022-02-16 17:45:09 1
2022-02-16 17:45:09 ----- OpenVPN Start -----
OpenVPN core 3.git::58b92569 ios arm64 64-bit
2022-02-16 17:45:09 OpenVPN core 3.git::58b92569 ios arm64 64-bit
2022-02-16 17:45:09 Frame=512/2048/512 mssfix-ctrl=1250
2022-02-16 17:45:09 UNUSED OPTIONS
4 [resolv-retry] [infinite] 
5 [nobind] 
6 [persist-key] 
7 [persist-tun] 
13 [verb] [3] 
2022-02-16 17:45:09 EVENT: RESOLVE
2022-02-16 17:45:09 Contacting [x.x.x.x]:1194/TCP via TCPv4
2022-02-16 17:45:09 EVENT: WAIT
2022-02-16 17:45:20 Server poll timeout, trying next remote entry...
2022-02-16 17:45:20 EVENT: RECONNECTING
2022-02-16 17:45:20 EVENT: RESOLVE
2022-02-16 17:45:20 Contacting [x.x.x.x]:1194/TCP via TCPv4
2022-02-16 17:45:20 EVENT: WAIT
2022-02-16 17:45:31 Server poll timeout, trying next remote entry...
2022-02-16 17:45:31 EVENT: RECONNECTING
2022-02-16 17:45:31 EVENT: RESOLVE
2022-02-16 17:45:31 Contacting [x.x.x.x]:1194/TCP via TCPv4
2022-02-16 17:45:31 EVENT: WAIT
2022-02-16 17:45:42 EVENT: CONNECTION_TIMEOUT [ERR]
2022-02-16 17:45:42 Raw stats on disconnect:
  CONNECTION_TIMEOUT : 1
  N_RECONNECT : 2
2022-02-16 17:45:42 Performance stats on disconnect:
  CPU usage (microseconds): 38178
  Network bytes per CPU second: 0
  Tunnel bytes per CPU second: 0
2022-02-16 17:45:42 EVENT: DISCONNECTED
2022-02-16 17:45:42 Raw stats on disconnect:
  CONNECTION_TIMEOUT : 1
  N_RECONNECT : 2
2022-02-16 17:45:42 Performance stats on disconnect:
  CPU usage (microseconds): 48199
  Network bytes per CPU second: 0
  Tunnel bytes per CPU second: 0
2022-02-16 17:48:07 RESUME TEST: Internet:ReachableViaWWAN/WR t------
2022-02-16 17:48:07 STANDARD RESUME
# OPENVPN Log ends

Can someone please help me out with this?

Thanks in advance.

Cheers.

Sob · Wed Feb 16, 2022 6:29 pm

That dstnat rule is nonsense. When you have server on router, you just need to allow connetions to it in input chain (which you did). I'm guessing you added dstnat later as desperate attempt to change something, after normal config didn't work?

The real problem is that anything from LAN (bridge) to VPN clients won't go there, because it's marked and routed to one of ISPs. You can either add VPN range to your not_in_internet list, or you can use foolproof routing rule for all local subnets:

/ip route rule
add dst-address=172.16.0.0/12 action=lookup-only-in-table table=main
...

ashishwadekar · Thu Feb 17, 2022 10:22 am

Thanks for your response Sob.

Yes, the dst-nat rule was out of desperation / frustration

I tried adding the network to not_in_internet_list or setting up route rule but still the response is same.

Server poll timeout still persists.

Would you like to have some other logs or something for further diagnosis or help?

Cheers,
Ashish

arnaldo · Thu Feb 17, 2022 8:20 pm

With dual WAN you need to ensure that inbound packages to the routers will have their responses sent out through the same interface.

- Inbound connections on the WANs that are not connection marked need to be tagged (in magle/preroute). One rule per WAN.
- On the output flow (mangle/output), if a connection is conn-marked, you need to set the corresponding route-mark.

Otherwise the outbound packages will flow through the active WAN (greater priority). So, packages come thru WAN_A and return will be sent over WAN_B. Will cause the timeout you see.

I have a very similar setup. Works nicely with OpenVPN/TCP, but still can't get it to work 100% with OpenVPN/UDP (ROSv7) as conn-mark seems to be having a problem with UDP.

Sob · Fri Feb 18, 2022 2:05 am

On second look, what I wrote is correct, but there's also some other problem, when it doesn't connect at all. But I don't see what it is. Do you see this rule accepting any packets (after you removed that dstnat rule)?

/ip firewall filter
add action=accept chain=input comment="Allow OpenVPN Connection" dst-port=1194 protocol=tcp

Most likely yes. Then try if logs show anything useful:

/system logging add topics=ovpn

ashishwadekar · Fri Feb 18, 2022 3:04 pm

With dual WAN you need to ensure that inbound packages to the routers will have their responses sent out through the same interface.

- Inbound connections on the WANs that are not connection marked need to be tagged (in magle/preroute). One rule per WAN.
- On the output flow (mangle/output), if a connection is conn-marked, you need to set the corresponding route-mark.

Otherwise the outbound packages will flow through the active WAN (greater priority). So, packages come thru WAN_A and return will be sent over WAN_B. Will cause the timeout you see.

I have a very similar setup. Works nicely with OpenVPN/TCP, but still can't get it to work 100% with OpenVPN/UDP (ROSv7) as conn-mark seems to be having a problem with UDP.

Thanks for sharing your insight Arnaldo.

Everything is working fine except OpenVPN & I am not able to understand why.

Can you share your config to have a look at it? Maybe I can get some idea from it.

Cheers.

ashishwadekar · Fri Feb 18, 2022 3:05 pm

On second look, what I wrote is correct, but there's also some other problem, when it doesn't connect at all. But I don't see what it is. Do you see this rule accepting any packets (after you removed that dstnat rule)?
Code: Select all
/ip firewall filter
add action=accept chain=input comment="Allow OpenVPN Connection" dst-port=1194 protocol=tcp
Most likely yes. Then try if logs show anything useful:
Code: Select all
/system logging add topics=ovpn

Yes this rule is accepting packets after dst-nat rule is removed.

As suggested, I have added the topic ovpn but nothing is getting logged related to this topic.

Something else that I can try?

Thanks in advance.

Sob · Fri Feb 18, 2022 5:26 pm

I'd try different client (Windows, Linux), to see if it works or at least has more useful error messages.

ashishwadekar · Fri Feb 18, 2022 6:48 pm

Ok Sob. I will try with Windows client & share my observations soon.

Cheers.

ashishwadekar · Fri Feb 18, 2022 7:06 pm

I'd try different client (Windows, Linux), to see if it works or at least has more useful error messages.

This is what I get with a Windows client:

[Feb 18, 2022, 22:33:04] OpenVPN core 3.git::d3f8b18b win x86_64 64-bit built on Dec  8 2021 12:04:20
⏎[Feb 18, 2022, 22:33:04] Frame=512/2048/512 mssfix-ctrl=1250
⏎[Feb 18, 2022, 22:33:04] UNUSED OPTIONS
4 [resolv-retry] [infinite]
5 [nobind]
6 [persist-key]
7 [persist-tun]
13 [verb] [3]
⏎[Feb 18, 2022, 22:33:04] EVENT: RESOLVE ⏎[Feb 18, 2022, 22:33:04] EVENT: WAIT ⏎[Feb 18, 2022, 22:33:04] WinCommandAgent: transmitting bypass route to x.x.x.x
{
	"host" : "x.x.x.x",
	"ipv6" : false
}

⏎[Feb 18, 2022, 22:33:14] Server poll timeout, trying next remote entry...
⏎[Feb 18, 2022, 22:33:14] EVENT: RECONNECTING ⏎[Feb 18, 2022, 22:33:14] EVENT: RESOLVE ⏎[Feb 18, 2022, 22:33:14] EVENT: WAIT ⏎[Feb 18, 2022, 22:33:14] WinCommandAgent: transmitting bypass route to x.x.x.x
{
	"host" : "x.x.x.x",
	"ipv6" : false
}

⏎[Feb 18, 2022, 22:33:24] Server poll timeout, trying next remote entry...
⏎[Feb 18, 2022, 22:33:24] EVENT: RECONNECTING ⏎[Feb 18, 2022, 22:33:24] EVENT: RESOLVE ⏎[Feb 18, 2022, 22:33:24] EVENT: WAIT ⏎[Feb 18, 2022, 22:33:24] WinCommandAgent: transmitting bypass route to x.x.x.x
{
	"host" : "x.x.x.x",
	"ipv6" : false
}

⏎[Feb 18, 2022, 22:33:34] Server poll timeout, trying next remote entry...
⏎[Feb 18, 2022, 22:33:34] EVENT: RECONNECTING ⏎[Feb 18, 2022, 22:33:34] EVENT: RESOLVE ⏎[Feb 18, 2022, 22:33:34] EVENT: WAIT ⏎[Feb 18, 2022, 22:33:34] WinCommandAgent: transmitting bypass route to x.x.x.x
{
	"host" : "x.x.x.x",
	"ipv6" : false
}

⏎[Feb 18, 2022, 22:33:44] Server poll timeout, trying next remote entry...
⏎[Feb 18, 2022, 22:33:44] EVENT: RECONNECTING ⏎[Feb 18, 2022, 22:33:44] EVENT: RESOLVE ⏎[Feb 18, 2022, 22:33:44] EVENT: WAIT ⏎[Feb 18, 2022, 22:33:44] WinCommandAgent: transmitting bypass route to x.x.x.x
{
	"host" : "x.x.x.x",
	"ipv6" : false
}

⏎[Feb 18, 2022, 22:33:54] Server poll timeout, trying next remote entry...
⏎[Feb 18, 2022, 22:33:54] EVENT: RECONNECTING ⏎[Feb 18, 2022, 22:33:54] EVENT: RESOLVE ⏎[Feb 18, 2022, 22:33:54] EVENT: WAIT ⏎[Feb 18, 2022, 22:33:54] WinCommandAgent: transmitting bypass route to x.x.x.x
{
	"host" : "x.x.x.x",
	"ipv6" : false
}

⏎[Feb 18, 2022, 22:34:04] EVENT: CONNECTION_TIMEOUT  CONNECTION_TIMEOUT : 1
 N_RECONNECT : 5
⏎[Feb 18, 2022, 22:34:04] EVENT: DISCONNECTED ⏎

Anything that I can do that you can suggest?

Thanks.

Sob · Fri Feb 18, 2022 7:26 pm

What client is that? It doesn't look like the usual one (from https://openvpn.net/community-downloads/). I'm not saying that yours can't work, just that I don't know it.

One more thing you can try is to connect from LAN, that would rule out whole load balancing as the problem.

ashishwadekar · Mon Feb 21, 2022 9:55 am

What client is that? It doesn't look like the usual one (from https://openvpn.net/community-downloads/). I'm not saying that yours can't work, just that I don't know it.

One more thing you can try is to connect from LAN, that would rule out whole load balancing as the problem.

Hi Sob

The client is OpenVPN Connect on Windows 10 machine.

So as suggested I tried connecting via LAN & these are my observations:

The VPN connection is successful via LAN from my iPhone but does not connect via MAC nor Windows in LAN strange!
I tried turning off my NonVPN WAN i.e. WAN1 in this case & still VPN connection is successful from my iPhone while in LAN
I tried turning off my VPN WAN i.e. WAN2 in this case to check some goof up with routes or mangling & as expected VPN connection is unsuccessful from my iPhone while in LAN
I tried turning off my NonVPN WAN i.e. WAN1 & tried connecting via LTE on my iPhone but it is unsuccessful.

My hunch is something is wrong with mangling / route / packet marking.

The input allow rule counters get updated while connecting from outside network but still client is not able to discover the OpenVPN server. On the other hand while connecting from LAN obviously the input accept rule is not affected but VPN server is found. Frustrating

Sob · Tue Feb 22, 2022 4:47 am

That's weird. The idea behind connecting from LAN was to do a test where load balacing does not influence anything. All your connection marking rules have either in-interface=WANX_PPPOEOUT or dst-address-type=!local, so none of them will touch any connection from LAN to any of router's addresses. As long as there's no misconfiguration or incompatibility between client and server, connections from LAN must work.

ashishwadekar · Tue Feb 22, 2022 3:54 pm

That's weird. The idea behind connecting from LAN was to do a test where load balacing does not influence anything. All your connection marking rules have either in-interface=WANX_PPPOEOUT or dst-address-type=!local, so none of them will touch any connection from LAN to any of router's addresses. As long as there's no misconfiguration or incompatibility between client and server, connections from LAN must work.

I am clueless.

I finally just wiped the dual WAN config & have setup L2TP VPN in plain jane way.

I will try replicating this setup in some other piece of hardware & then pursue this further.

Sob, appreciate all of your time & thanks for guiding. Stay safe. Cheers.

Open VPN Server Issue on a Dual WAN setup

Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup

Re: Open VPN Server Issue on a Dual WAN setup