MikroTik

Community discussions
https://forum.mikrotik.com/

PCC load balancing and PCQ queue

https://forum.mikrotik.com/viewtopic.php?t=174623

Page 1 of 1

PCC load balancing and PCQ queue

Posted: Tue Apr 20, 2021 6:06 pm
by calevizo
Hello everyone :)

I'm trying to setup Load Balancing with two WAN connections, my WAN connections are LTE and Speedtest shows me a speed of 40Mbps/10Mbps for each WAN.
I expect ~80 concurrent internet users, the WANs are supposed to have unlimited data, but I have to find a way to limit traffic so everyone has a decent speed without drawing too much attention from my ISP. I thought about limiting the speed to 1M/768k for all users.

network.png

Reading through the Wiki I was able to achieve some results with PCQ queues over PCC load balancing, but its my fist time creating such a setup, I don't really understand some of the configuration and I have not found a way to properly test it.

I'm worried about the queue size (50) and queue total size (2000), I could not understand these concepts, does the quotient of 2000/50=40 just show you how many users the queue can handle before splitting the queue rate? Does that mean that I have to double the total size to accommodate ~80 users?

could you please review my config and see if it fits my plan?
thank you!!!

Code: Select all
# apr/20/2021 17:45:13 by RouterOS 6.47.9
# software id = Q1DQ-GYUS
#
# model = RB4011iGS+
# serial number = xxx
/interface bridge
add admin-mac=08:55:31:83:98:E0 auto-mac=no comment=defconf name=bridge
/interface ethernet
set [ find default-name=ether1 ] name=cosmote
set [ find default-name=ether2 ] name=cosmote_nat
/interface vlan
add interface=bridge name=vlan11-reception vlan-id=11
add interface=ether3 name=vlan12-beach vlan-id=12
add interface=ether3 name=vlan13-telephones vlan-id=13
add interface=ether3 name=vlan14-restaurant vlan-id=14
add interface=ether3 name=vlan15-aps vlan-id=15
add interface=ether3 name=vlan100-clients vlan-id=100
/interface ethernet switch port
set 0 default-vlan-id=0
set 1 default-vlan-id=0
set 2 default-vlan-id=0
set 3 default-vlan-id=0
set 4 default-vlan-id=0
set 5 default-vlan-id=0
set 6 default-vlan-id=0
set 7 default-vlan-id=0
set 8 default-vlan-id=0
set 9 default-vlan-id=0
set 10 default-vlan-id=0
set 11 default-vlan-id=0
/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN
add name=MANAGEMENT
/interface wireless security-profiles
set [ find default=yes ] supplicant-identity=MikroTik
/ip ipsec peer
add name=l2tpserver passive=yes
/ip ipsec proposal
set [ find default=yes ] enc-algorithms=3des
/ip pool
add name=pool-vlan11-reception ranges=192.168.11.100-192.168.11.254
add name=pool-vlan13-telephones ranges=192.168.13.100-192.168.13.254
add name=pool-vlan14-restaurant ranges=192.168.14.100-192.168.14.254
add name=pool-vlan15-aps ranges=192.168.15.100-192.168.15.253
add name=pool-vlan100-clients ranges=172.16.0.2-172.16.1.254
add name=pool-vpn ranges=192.168.102.100-192.168.102.254
add name=pool-vlan12-beach ranges=192.168.12.100-192.168.12.254
/ip dhcp-server
add address-pool=pool-vlan11-reception disabled=no interface=vlan11-reception \
    name=dhcp-vlan11-reception
add address-pool=pool-vlan13-telephones disabled=no interface=\
    vlan13-telephones name=dhcp-vlan13-telephones
add address-pool=pool-vlan14-restaurant disabled=no interface=\
    vlan14-restaurant name=dhcp-vlan14-restaurant
add address-pool=pool-vlan15-aps disabled=no interface=vlan15-aps name=\
    dhcp-vlan15-aps
add address-pool=pool-vlan100-clients disabled=no interface=vlan100-clients \
    name=dhcp-vlan100-clients
add address-pool=pool-vlan12-beach disabled=no interface=vlan12-beach name=\
    dhcp-vlan12-beach
/ppp profile
add dns-server=192.168.102.1 local-address=192.168.102.1 name=ipsec_vpn
/queue type
add kind=pcq name=PCQ_download pcq-classifier=dst-address pcq-rate=1M
add kind=pcq name=PCQ_upload pcq-classifier=src-address pcq-rate=768k
/queue tree
add name=queue3 packet-mark=cosmote_client_download parent=global queue=\
    PCQ_download
add name=queue4 packet-mark=cosmote_client_upload parent=global queue=\
    PCQ_upload
add name=queue5 packet-mark=cosmote_nat_client_download parent=global queue=\
    PCQ_download
add name=queue6 packet-mark=cosmote_nat_client_upload parent=global queue=\
    PCQ_upload
/interface bridge port
add bridge=bridge comment=defconf interface=ether3
add bridge=bridge comment=defconf interface=ether4
add bridge=bridge comment=defconf interface=ether5
add bridge=bridge comment=defconf interface=sfp-sfpplus1
/ip neighbor discovery-settings
set discover-interface-list=LAN
/interface bridge vlan
add bridge=bridge disabled=yes tagged=ether3 vlan-ids=14
add bridge=bridge disabled=yes tagged=ether3,bridge vlan-ids=11
add bridge=bridge disabled=yes tagged=ether3 vlan-ids=12
add bridge=bridge disabled=yes tagged=ether3 vlan-ids=13
add bridge=bridge disabled=yes tagged=ether3 vlan-ids=15
add bridge=bridge disabled=yes tagged=ether3 vlan-ids=100
/interface l2tp-server server
set authentication=mschap1,mschap2 default-profile=ipsec_vpn enabled=yes
/interface list member
add interface=bridge list=LAN
add interface=cosmote list=WAN
add interface=vlan11-reception list=LAN
add interface=vlan13-telephones list=LAN
add interface=vlan14-restaurant list=LAN
add interface=vlan15-aps list=LAN
add interface=vlan100-clients list=LAN
add interface=vlan12-beach list=LAN
add interface=cosmote_nat list=WAN
add interface=bridge list=MANAGEMENT
add interface=vlan11-reception list=MANAGEMENT
/ip address
add address=192.168.10.1/24 interface=bridge network=192.168.10.0
add address=10.0.0.10/24 interface=cosmote network=10.0.0.0
add address=192.168.13.1/24 interface=vlan13-telephones network=192.168.13.0
add address=192.168.14.1/24 interface=vlan14-restaurant network=192.168.14.0
add address=172.16.0.1/23 interface=vlan100-clients network=172.16.0.0
add address=192.168.15.1/24 interface=vlan15-aps network=192.168.15.0
add address=192.168.11.1/24 interface=vlan11-reception network=192.168.11.0
add address=192.168.12.1/24 interface=vlan12-beach network=192.168.12.0
add address=10.0.1.10/24 interface=cosmote_nat network=10.0.1.0
/ip dhcp-server network
add address=172.16.0.0/23 dns-server=172.16.0.1 gateway=172.16.0.1
add address=192.168.10.0/24 comment=defconf dns-server=192.168.10.1 gateway=\
    192.168.10.1 netmask=24
add address=192.168.11.0/24 dns-server=192.168.11.1 gateway=192.168.11.1
add address=192.168.12.0/24 dns-server=192.168.12.1 gateway=192.168.12.1
add address=192.168.13.0/24 dns-server=192.168.13.1 gateway=192.168.13.1
add address=192.168.14.0/24 dns-server=192.168.14.1 gateway=192.168.14.1
add address=192.168.15.0/24 dns-server=192.168.15.1 gateway=192.168.15.1
/ip dns
set allow-remote-requests=yes servers=1.1.1.1
/ip firewall address-list
add address=192.168.11.0/24 list=list_public_ip
add address=192.168.10.0/24 list=list_public_ip
add address=192.168.13.0/24 list=list_public_ip
add address=192.168.102.0/24 list=vpn
add address=192.168.14.0/24 list=restaurant
add address=192.168.13.0/24 list=allow_forward_established_related
add address=192.168.15.0/24 list=allow_forward_established_related
add address=192.168.14.0/24 list=allow_forward_established_related
add address=192.168.12.0/24 list=drop_forwarded
add address=192.168.15.0/24 list=drop_forwarded_except_internet
add address=172.16.0.0/23 list=drop_forwarded_except_internet
add address=192.168.14.0/24 list=drop_forwarded_except_internet
add address=192.168.10.0/24 list=admin_network
add address=192.168.11.0/24 list=admin_network
add address=192.168.102.0/24 list=admin_network
/ip firewall filter
add action=accept chain=input comment="open the IPSec ports from the WAN" \
    protocol=ipsec-esp
add action=accept chain=input comment="open the IPSec ports from the WAN" \
    port=1701,500,4500 protocol=udp
add action=accept chain=input comment=\
    "defconf: accept established,related,untracked" connection-state=\
    established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=\
    invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment=\
    "defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input comment=\
    "allow input to admin network address list" src-address-list=\
    admin_network
add action=accept chain=forward comment="Allow forward to VPN address list" \
    src-address-list=vpn
add action=drop chain=input comment="defconf: drop all not coming from LAN" \
    disabled=yes in-interface-list=!LAN
add action=accept chain=input comment="allow input from LAN to UDP DNS" \
    dst-port=53 in-interface-list=LAN protocol=udp
add action=accept chain=input comment="allow input from LAN to TCP DNS" \
    dst-port=53 in-interface-list=LAN protocol=tcp
add action=accept chain=input comment="allow input from LAN to NTP" dst-port=\
    123 in-interface-list=LAN protocol=udp
add action=drop chain=input comment=\
    "drop input except admin network address list" src-address-list=\
    !admin_network
add action=drop chain=forward comment="forward clients only to the internet" \
    dst-address=!0.0.0.0/0 in-interface=vlan100-clients
add action=accept chain=forward comment="defconf: accept in ipsec policy" \
    ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" \
    ipsec-policy=out,ipsec
add action=accept chain=forward comment=\
    "allow forward from beach to beachbum server" dst-address=192.168.11.11 \
    src-address=192.168.12.0/24
add action=drop chain=forward comment="drop forwarded address list" \
    src-address-list=drop_forwarded
add action=accept chain=forward comment=\
    "allow forward established related address list" connection-state=\
    established,related src-address-list=allow_forward_established_related
add action=drop chain=forward comment=\
    "drop forwarded except internet address list" dst-address=!0.0.0.0/0 \
    src-address-list=drop_forwarded_except_internet
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" \
    connection-state=established,related disabled=yes
add action=accept chain=forward comment=\
    "defconf: accept established,related, untracked" connection-state=\
    established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" \
    connection-state=invalid
add action=drop chain=forward comment=\
    "defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat \
    connection-state=new in-interface-list=WAN
/ip firewall mangle
add action=accept chain=prerouting comment=PCC dst-address=10.0.0.0/24 \
    in-interface-list=LAN
add action=accept chain=prerouting dst-address=10.0.1.0/24 in-interface-list=\
    LAN
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=cosmote new-connection-mark=cosmote_conn
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=cosmote_nat new-connection-mark=cosmote_nat_conn
add action=mark-connection chain=prerouting connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=\
    cosmote_conn per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=\
    cosmote_nat_conn per-connection-classifier=both-addresses:2/1
add action=mark-routing chain=prerouting connection-mark=cosmote_conn \
    in-interface-list=LAN new-routing-mark=to_cosmote
add action=mark-routing chain=prerouting connection-mark=cosmote_nat_conn \
    in-interface-list=LAN new-routing-mark=to_cosmote_nat
add action=mark-routing chain=output connection-mark=cosmote_conn \
    new-routing-mark=to_cosmote
add action=mark-routing chain=output connection-mark=cosmote_nat_conn \
    new-routing-mark=to_cosmote_nat
add action=mark-packet chain=prerouting comment=PCQ in-interface-list=LAN \
    new-packet-mark=cosmote_client_upload passthrough=yes
add action=mark-packet chain=prerouting in-interface-list=WAN \
    new-packet-mark=cosmote_client_download passthrough=no
/ip ipsec identity
add generate-policy=port-override peer=l2tpserver
/ip ipsec policy
set 0 dst-address=0.0.0.0/0 src-address=0.0.0.0/0
/ip route
add check-gateway=ping distance=1 gateway=10.0.0.1 routing-mark=to_cosmote
add check-gateway=ping distance=1 gateway=10.0.1.1 routing-mark=\
    to_cosmote_nat
add check-gateway=ping distance=1 gateway=10.0.0.1
add check-gateway=ping distance=2 gateway=10.0.1.1
/ip upnp
set enabled=yes
/ip upnp interfaces
add interface=bridge type=internal
add interface=cosmote type=external
add interface=ether3 type=internal
add interface=cosmote_nat type=external
/ppp secret
add name=chris profile=ipsec_vpn remote-address=192.168.102.2 service=l2tp
/system clock
set time-zone-name=Europe/Athens
/system identity
set name=res-rtr
/system logging
add action=disk topics=critical
add action=disk topics=error
add action=disk topics=info
add action=disk topics=warning
/system ntp client
set enabled=yes primary-ntp=216.239.35.0 secondary-ntp=216.239.35.4
/system ntp server
set enabled=yes
/tool mac-server
set allowed-interface-list=MANAGEMENT
/tool mac-server mac-winbox
set allowed-interface-list=MANAGEMENT

Re: PCC load balancing and PCQ queue

Posted: Wed Apr 21, 2021 1:53 pm
by calevizo
I had some progress.

Speedtest gives me 40Mbps/10Mbps for each WAN, I will use a little less than that to account for fluctuations in speed, lets say 38/9Mpbs.

This is 38*10^6 bits/s download and 9*10^6 bits/s upload. So that means that I have to set max-limit in the Queue Tree to 38.000.000/9.000.000 bytes.
I gave each Wan its own download/upload Queue Trees, this is the max-limit that you find in the Wiki.

Code: Select all
/queue tree
add max-limit=38M name=cosmote_client_download packet-mark=\
    cosmote_client_download parent=global queue=PCQ_cosmote_download
add max-limit=9M name=cosmote_client_upload packet-mark=cosmote_client_upload \
    parent=global queue=PCQ_cosmote_upload
add max-limit=38M name=cosmote_nat_client_download packet-mark=\
    cosmote_nat_client_download parent=global queue=PCQ_cosmote_nat_download
add max-limit=9M name=cosmote_nat_client_upload packet-mark=\
    cosmote_nat_client_upload parent=global queue=PCQ_cosmote_nat_upload

Packets are marked separately for each WAN

Code: Select all
add action=mark-packet chain=prerouting comment="PCQ cosmote" \
    connection-mark=cosmote_conn in-interface-list=LAN new-packet-mark=\
    cosmote_client_upload passthrough=no
add action=mark-packet chain=prerouting connection-mark=cosmote_conn \
    in-interface-list=WAN new-packet-mark=cosmote_client_download \
    passthrough=no
add action=mark-packet chain=prerouting comment="PCQ cosmote_nat" \
    connection-mark=cosmote_nat_conn in-interface-list=LAN new-packet-mark=\
    cosmote_nat_client_upload passthrough=no
add action=mark-packet chain=prerouting connection-mark=cosmote_nat_conn \
    in-interface-list=WAN new-packet-mark=cosmote_nat_client_download \
    passthrough=no

I expect 80 concurrent users, I think that I should leave the sub-stream packet buffer to 50KiB, the default, that means 80*50=4.000, but since I have two WANs its 4.000/2=2.000 again the default. So I only set the pcq-rate.

Code: Select all
/queue type
add kind=pcq name=PCQ_cosmote_download pcq-classifier=dst-address pcq-rate=1M
add kind=pcq name=PCQ_cosmote_upload pcq-classifier=src-address pcq-rate=768k
add kind=pcq name=PCQ_cosmote_nat_download pcq-classifier=dst-address \
    pcq-rate=1M
add kind=pcq name=PCQ_cosmote_nat_upload pcq-classifier=src-address pcq-rate=\
    768k

complete config attached.

Re: PCC load balancing and PCQ queue

Posted: Wed Apr 21, 2021 8:11 pm
by calevizo
update: I was passing all traffic from the PCQ Queues so I added a address list to limit the mangle rules.

Re: PCC load balancing and PCQ queue

Posted: Thu Apr 22, 2021 12:33 pm
by calevizo
these are my new mangle rules, is there a more elegant way to mark the packets? Also the clients get double the speed than set in the queue rate, is this because of the PCC? Is there a way around that?

thank you
Code: Select all
/ip firewall mangle
add action=accept chain=prerouting comment=PCC dst-address=10.0.0.0/24 \
    in-interface-list=LAN
add action=accept chain=prerouting dst-address=10.0.1.0/24 in-interface-list=\
    LAN
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=cosmote new-connection-mark=cosmote_conn
add action=mark-connection chain=prerouting connection-mark=no-mark \
    in-interface=cosmote_nat new-connection-mark=cosmote_nat_conn
add action=mark-connection chain=prerouting connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=\
    cosmote_conn per-connection-classifier=both-addresses:2/0
add action=mark-connection chain=prerouting connection-mark=no-mark \
    dst-address-type=!local in-interface-list=LAN new-connection-mark=\
    cosmote_nat_conn per-connection-classifier=both-addresses:2/1
add action=mark-routing chain=prerouting connection-mark=cosmote_conn \
    in-interface-list=LAN new-routing-mark=to_cosmote
add action=mark-routing chain=prerouting connection-mark=cosmote_nat_conn \
    in-interface-list=LAN new-routing-mark=to_cosmote_nat
add action=mark-routing chain=output connection-mark=cosmote_conn \
    new-routing-mark=to_cosmote
add action=mark-routing chain=output connection-mark=cosmote_nat_conn \
    new-routing-mark=to_cosmote_nat
add action=mark-packet chain=prerouting comment="PCQ cosmote" \
    connection-mark=cosmote_conn dst-address=0.0.0.0/0 in-interface-list=LAN \
    new-packet-mark=cosmote_client_upload passthrough=no src-address-list=\
    queue
add action=mark-packet chain=prerouting connection-mark=cosmote_conn \
    dst-address-list=queue in-interface-list=WAN new-packet-mark=\
    cosmote_client_download passthrough=no src-address=0.0.0.0/0
add action=mark-packet chain=prerouting comment="PCQ cosmote_nat" \
    connection-mark=cosmote_nat_conn dst-address=0.0.0.0/0 in-interface-list=\
    LAN new-packet-mark=cosmote_nat_client_upload passthrough=no \
    src-address-list=queue
add action=mark-packet chain=prerouting connection-mark=cosmote_nat_conn \
    dst-address-list=queue in-interface-list=WAN new-packet-mark=\
    cosmote_nat_client_download passthrough=no src-address=0.0.0.0/0
and the queues
Code: Select all
/queue type
add kind=pcq name=PCQ_cosmote_download pcq-classifier=dst-address pcq-rate=1M
add kind=pcq name=PCQ_cosmote_upload pcq-classifier=src-address pcq-rate=500k
add kind=pcq name=PCQ_cosmote_nat_download pcq-classifier=dst-address \
    pcq-rate=1M
add kind=pcq name=PCQ_cosmote_nat_upload pcq-classifier=src-address pcq-rate=\
    500k
/queue tree
add max-limit=38M name=cosmote_client_download packet-mark=\
    cosmote_client_download parent=global queue=PCQ_cosmote_download
add max-limit=9M name=cosmote_client_upload packet-mark=cosmote_client_upload \
    parent=global queue=PCQ_cosmote_upload
add max-limit=38M name=cosmote_nat_client_download packet-mark=\
    cosmote_nat_client_download parent=global queue=PCQ_cosmote_nat_download
add max-limit=9M name=cosmote_nat_client_upload packet-mark=\
    cosmote_nat_client_upload parent=global queue=PCQ_cosmote_nat_upload
All times are UTC+02:00
Page 1 of 1
Powered by phpBB® Forum Software © phpBB Limited
https://www.phpbb.com/