SRX Services Gateway
Highlighted
SRX Services Gateway

FBF filter-based forwarding dual ISP failover

3 weeks ago

I'm trying to configure a trust interface to failover from one ISP on ge-0/0/0 to another on ge-0/0/1, but my trust int only seems to route to one. I am following juniper kb solution here . Here's what I have so far:

interfaces {
    ge-0/0/0 {
        unit 0 {
            description isp1;
            family inet {
                address 1.1.1.2/24;
            }
        }
    }
    ge-0/0/1 {
        unit 0 {
            description isp2;
            family inet {
                address 2.2.2.2/24;
            }
        }
    }
    ge-0/0/2 {
        unit 0 {
            description pos;
            family inet {
                filter {
                    input F1;
                }
                address 192.168.8.1/24;
            }
        }
    }
    ge-0/0/3 {
        unit 0 {
            description office;
            family inet {
                address 192.168.20.1/24;
            }
        }
    }
    ge-0/0/4 {
        unit 0 {
            description publicwifi;
            family inet {
                address 172.16.30.1/24;
            }
        }
    }
}
routing-options {
    interface-routes {
        rib-group inet IMPORT-PHY;
    }
    static {
        route 0.0.0.0/0 next-hop 1.1.1.1;
    }
    rib-groups {
        IMPORT-PHY {
            import-rib [ inet.0 FBF-1.inet.0 FBF-2.inet.0 ];
        }
    }
}
security {
    nat {
        source {
            rule-set interface-nat {
                from zone office;
                to zone untrustisp1;
                rule rule1 {
                    match {
                        source-address 0.0.0.0/0;
                        destination-address 0.0.0.0/0;
                    }
                    then {
                        source-nat {
                            interface;
                        }
                    }
                }
            }
            rule-set pos-nat {
                from zone pos;
                to zone untrustisp1;
                rule rule0 {
                    match {
                        source-address 0.0.0.0/0;
                        destination-address 0.0.0.0/0;
                    }
                    then {
                        source-nat {
                            interface;
                        }
                    }
                }
            }
            rule-set wifi-nat {
                from zone publicwifi;
                to zone untrustisp1;
                rule rulewifi {
                    match {
                        source-address 0.0.0.0/0;
                        destination-address 0.0.0.0/0;
                    }
                    then {
                        source-nat {
                            interface;
                        }
                    }
                }
            }
        }
    }
    policies {
        from-zone pos to-zone untrustisp1 {
            policy policy-name {
                match {
                    source-address any;
                    destination-address any;
                    application any;
                }
                then {
                    permit;
                }
            }
        }
        from-zone office to-zone untrustisp1 {
            policy policy-name {
                match {
                    source-address any;
                    destination-address any;
                    application any;
                }
                then {
                    permit;
                }
            }
        }
        from-zone publicwifi to-zone untrustisp1 {
            policy policy-name {
                match {
                    source-address any;
                    destination-address any;
                    application any;
                }
                then {
                    permit;
                }
            }
        }
        from-zone pos to-zone untrustisp2 {
            policy policy-name {
                match {
                    source-address any;
                    destination-address any;
                    application any;
                }
                then {
                    permit;
                }
            }
        }
        from-zone office to-zone untrustisp2 {
            policy policy-name {
                match {
                    source-address any;
                    destination-address any;
                    application any;
                }
                then {
                    permit;
                }
            }
        }
        from-zone publicwifi to-zone untrustisp2 {
            policy policy-name {
                match {
                    source-address any;
                    destination-address any;
                    application any;
                }
                then {
                    permit;
                }
            }
        }
    }
    zones {
        security-zone untrustisp1 {
            interfaces {
                ge-0/0/0.0 {
                    host-inbound-traffic {
                        system-services {
                            ping;
                        }
                    }
                }
            }
        }
        security-zone untrustisp2 {
            interfaces {
                ge-0/0/1.0 {
                    host-inbound-traffic {
                        system-services {
                            ping;
                        }
                    }
                }
            }
        }
        security-zone pos {
            interfaces {
                ge-0/0/2.0 {
                    host-inbound-traffic {
                        system-services {
                            dhcp;
                            ping;
                            http;
                            https;
                        }
                    }
                }
            }
        }
        security-zone office {
            interfaces {
                ge-0/0/3.0 {
                    host-inbound-traffic {
                        system-services {
                            dhcp;
                            ping;
                            rpm;
                        }
                    }
                }
            }
        }
        security-zone publicwifi {
            interfaces {
                ge-0/0/4.0 {
                    host-inbound-traffic {
                        system-services {
                            dhcp;
                            ping;
                        }
                    }
                }
            }
        }
    }
}
firewall {
    filter F1 {
        term 1 {
            from {
                source-address {
                    192.168.8.100/32;
                }
            }
            then {
                routing-instance FBF-2;
            }
        }
        term 2 {
            from {
                source-address {
                    0.0.0.0/0;
                }
            }
            then {
                routing-instance FBF-1;
            }
        }
    }
}
routing-instances {
    FBF-1 {
        instance-type forwarding;
        routing-options {
            static {
                route 0.0.0.0/0 next-hop 1.1.1.1;
            }
        }
    }
    FBF-2 {
        instance-type forwarding;
        routing-options {
            static {
                route 0.0.0.0/0 next-hop 2.2.2.1;
            }
        }
    }
}
services {
    rpm {
        probe Probe-Server {
            test testsvr {
                target address 1.1.1.1;
                probe-count 10;
                probe-interval 5;
                test-interval 10;
                thresholds {
                    successive-loss 10;
                    total-loss 5;
                }
                destination-interface ge-0/0/0.0;
                next-hop 1.1.1.1;
            }
        }
        probe Probe-Server1 {
            test testsvr {
                target address 2.2.2.1;
                probe-count 10;
                probe-interval 5;
                test-interval 10;
                thresholds {
                    successive-loss 10;
                    total-loss 5;
                }
                destination-interface ge-0/0/1.0;
                next-hop 2.2.2.1;
            }
        }
    }
    ip-monitoring {
        policy Server-Tracking {
            match {
                rpm-probe Probe-Server;
            }
            then {
                preferred-route {
                    routing-instances FBF-1 {
                        route 0.0.0.0/0 {
                            next-hop 1.1.1.1;
                        }
                    }
                }
            }
        }
        policy Server-Tracking1 {
            match {
                rpm-probe Probe-Server1;
            }
            then {
                preferred-route {
                    routing-instances FBF-2 {
                        route 0.0.0.0/0 {
                            next-hop 2.2.2.1;
                        }
                    }
                }
            }
        }
    }
}

I don't quite understand what firewall filter F1 and RPM configuration is really trying to do.

 

Should I be using ECMP for this instead, or something else?

9 REPLIES 9
Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

3 weeks ago

Hi silverst8p,

 

Greetings, This is expected because you have a failover mechanism which means that only if one link fails the other one takes part of the action.. Now if you are looking for some load balancing ECMP is the way to go, let me clarify each question for you:

 

F1 in your case is a regular firewall filter that is necessary to do an FBF ( filter-based forwarding) once the intended traffic hits the filter it will redirect it to the routing instance that is configured. 

 

 

RPM
The real-time performance monitoring (RPM) feature allows you to accurately measure the performance between two network endpoints. With the RPM tool, you configure and send probes to a specified target and monitor the analyzed results to determine packet loss, round-trip time, and jitter, In your case it means that if one the probe fails and it is currently the active path it will do the failover to the other one that succeed the probe. If the probe from the active path succeeds there is not failover.

 

Regarding your question about ECMP, this is completely different than a failover mechanism as mentioned above, ECMP stand for equal-cost multipath and this is to load balance traffic across multiple paths when the cost to reach to their destination is the same ( equal)

 

Now take into consideration that if you have multiple next hops to the same destination if one fails the other one will take over so, in essence, it is a failover as well.

 

To summarize:

 

F1 is a firewall filter that classifies traffic into a routing instance.

 

RPM is to check your network performance and reachability then based on the results, an action can be taken.

 

ECMP Is formed when the routing table contains multiple next-hop addresses for the same destination with an equal cost. (Routes of equal cost have the same preference and metric values.) If there is an ECMP set for the active route, Junos OS uses a hash algorithm to choose one of the next-hop addresses in the ECMP set to install in the forwarding table.

 

Please check this doc about ECMP and how to go after it gentlemen! 

 

https://www.juniper.net/documentation/en_US/junos/topics/topic-map/ecmp-flow-based.html

 

If this solves your problem, please mark this post as "Accepted Solution" so we can help others too \Smiley Happy/

 

Regards,

Lil Dexx
JNCIE-ENT#863, 3X JNCIP-[SP-ENT-DC], 4X JNCIA [cloud-DevOps-Junos-Design], Champions Ingenius, SSYB

Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

3 weeks ago

Thanks so much for this explanation @LilDexx Smiley Happy

 

The intended functionality is to prefer ISP1 over ISP2 for all traffic, but automatically failover all traffic to ISP2 in case ISP1 becomes unreachable, so I think FBF should work for that?

 

Right now the traffic doesn't seem to be failing over to the backup ISP connection if I disconnect the the primary ISP link. I think the FBF *should* be doing that? Maybe I didn't leave it disconnected long enough for RPM to see the primary ISP down? I'll test some more.

Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

[ Edited ]
3 weeks ago

Hello silverst8p,

 

Instead of using cumbersome RPM, you can also achieve this by configuring BFD for the static route.

Once the BFD realizes that next-hop has become un-reahcable it will remove the static route configuration. Once the next-hop is reachable the static route will configured again.

 

Here is really good example:

 

https://www.juniper.net/documentation/en_US/junos/topics/topic-map/policy-bfd-static-routes.html

 

To prefer one ISP over another you can use metric with the static route.

 

PS: Please mark my response as solution if it answer your query, kudos are appreciated too!

 

Thanks

Vishal

Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

3 weeks ago

Hi silverst8p

 

After reviewing the configuration the problem seems to be th following one:

 

1. Traffic from hosts on 192.168.8.0/24 (except for .100) will reach the SRX on interface ge-0/0/2 and will hit F1 filter which places the traffic under routing-instance FBF-1 which sends the traffic over ISP 1. This is fine but what happens when you disconnect ge-0/0/0 (ISP1)?

2. Routing-instance FBF-1 loses the next-hop (1.1.1.1) for its default route hence its default route is no longer valid, however at this point the traffic from 192.168.8.0/24 is still placed on this routing-instance. We need to make sure FBF-1 has a backup default route, however the RPM configuration is not taking care of this situation because the default route that is injecting is not the correct one:

 

ip-monitoring {
        policy Server-Tracking {
            match {
                rpm-probe Probe-Server;
            }
            then {
                preferred-route {
                    routing-instances FBF-1 {
                        route 0.0.0.0/0 {
                            next-hop 1.1.1.1;
                        }
                    }

 

The route that we need to inject is "0.0.0.0/0 next-hop 2.2.2.1" so that FBF-1 could send the traffic over ISP2.

 

Please mark my answer as the Solution if it applies.
Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

3 weeks ago

Note the same situation occurs with the RPM policy Server-Tracking1, where the route that it should inject is "0.0.0.0/0 next-hop 1.1.1.1" so that FBF-2 can send traffic via ISP1 should the monitoring over ISP 2 fails.

 

Whenever you disconnect any of the ISPs make sure the probes are failing and that the routes are getting injected properly:

 

show services ip-monitoring status

show services rpm probe-results

 

 

Please mark my answer as the Solution if it applies.
Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

[ Edited ]
3 weeks ago

I changed the routes to look like:

policy Server-Tracking {
    match {
        rpm-probe Probe-Server;
    }
    then {
        preferred-route {
            routing-instances FBF-1 {
                route 0.0.0.0/0 {
                    next-hop 2.2.2.1;
                }
} } } } policy Server-Tracking1 { match { rpm-probe Probe-Server1; } then { preferred-route { routing-instances FBF-2 { route 0.0.0.0/0 { next-hop 1.1.1.1; } } } } }

But I'm not sure that's what you meant, I tried to add a backup route instead of a preferred route in

 

ip-monitoring {
    policy Server-Tracking {
        match {
            rpm-probe Probe-Server;
        }
        then {
            preferred-route {
                routing-instances FBF-1 {
                    route 0.0.0.0/0 {
                        next-hop 2.2.2.1;

But I didn't see an option for that.

 

I ran the show services first with both connected, then ISP1 disconnected, then ISP2 disconnected and got:

 

show services ip-monitoring status

Policy - Server-Tracking (Status: FAIL)
  RPM Probes:
    Probe name             Test Name       Address          Status
    ---------------------- --------------- ---------------- ---------
    Probe-Server           testsvr         1.1.1.1      FAIL
  Route-Action:
    route-instance    route             next-hop         state
    ----------------- ----------------- ---------------- -------------
    FBF-1             0.0.0.0/0         2.2.2.1        APPLIED

Policy - Server-Tracking1 (Status: PASS)
  RPM Probes:
    Probe name             Test Name       Address          Status
    ---------------------- --------------- ---------------- ---------
    Probe-Server1          testsvr         2.2.2.1        PASS
  Route-Action:
    route-instance    route             next-hop         state
    ----------------- ----------------- ---------------- -------------
    FBF-2             0.0.0.0/0         1.1.1.1      NOT-APPLIED

root@taylors> show services ip-monitoring status

Policy - Server-Tracking (Status: FAIL)
  RPM Probes:
    Probe name             Test Name       Address          Status
    ---------------------- --------------- ---------------- ---------
    Probe-Server           testsvr         1.1.1.1      FAIL
  Route-Action:
    route-instance    route             next-hop         state
    ----------------- ----------------- ---------------- -------------
    FBF-1             0.0.0.0/0         2.2.2.1        APPLIED

Policy - Server-Tracking1 (Status: PASS)
  RPM Probes:
    Probe name             Test Name       Address          Status
    ---------------------- --------------- ---------------- ---------
    Probe-Server1          testsvr         2.2.2.1        PASS
  Route-Action:
    route-instance    route             next-hop         state
    ----------------- ----------------- ---------------- -------------
    FBF-2             0.0.0.0/0         1.1.1.1      NOT-APPLIED

root@taylors> show services ip-monitoring status

Policy - Server-Tracking (Status: FAIL)
  RPM Probes:
    Probe name             Test Name       Address          Status
    ---------------------- --------------- ---------------- ---------
    Probe-Server           testsvr         1.1.1.1      FAIL
  Route-Action:
    route-instance    route             next-hop         state
    ----------------- ----------------- ---------------- -------------
    FBF-1             0.0.0.0/0         2.2.2.1        APPLIED

Policy - Server-Tracking1 (Status: PASS)
  RPM Probes:
    Probe name             Test Name       Address          Status
    ---------------------- --------------- ---------------- ---------
    Probe-Server1          testsvr         2.2.2.1        PASS
  Route-Action:
    route-instance    route             next-hop         state
    ----------------- ----------------- ---------------- -------------
    FBF-2             0.0.0.0/0         1.1.1.1      NOT-APPLIED

It looks like RPM does detect when one link goes down, so I guess that part is working? If I could just get RPM to trigger the right routes I guess it would work?

 

I think my firmware version is too old for BFD, so not sure that's an option.

Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

3 weeks ago

Hey silverst8p/all,

 

You are more than welcome, I strongly believe that this should work because the default route is coming from inet.0 through the rib-group sharing and not from a dynamic protocol injected by the next-hop, which means that if the next-hop is not reachable the route will still be there furthermore when the next-hop is not reachable the ip-monitor policy should take place, for faster failover you can modify the RPM thresholds the way you desired e.i

 

probe-count 5
probe-interval 1
test-interval 5
total-loss 3

 

Also, make sure if one of the probes is failing otherwise the failover would not happen e.i

[edit]
root@SRX-Firewall# run show services ip-monitoring status

Policy - GR1 (Status: FAIL)
RPM Probes:
Probe name Test Name Address Status
---------------------- --------------- ---------------- ---------
Failover1 probe-ge1 2.1.1.2 FAIL
Route-Action:
route-instance route next-hop state
----------------- ----------------- ---------------- -------------
inet.0 0.0.0.0/0 2.1.1.6 APPLIED

 

Here is a sample without RI's But the principle is the same: https://kb.juniper.net/InfoCenter/index?page=content&id=KB32556

 

If this solves your problem, please mark this post as "Accepted Solution" so we can help others too \Smiley Happy/

Regards,

Lil Dexx
JNCIE-ENT#863, 3X JNCIP-[SP-ENT-DC], 4X JNCIA [cloud-DevOps-Junos-Design], Champions Ingenius, SSYB

Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

[ Edited ]
2 weeks ago

I tried disconnecting ISP1 on ge-0/0/0 and after a bit then got:

show services rpm probe-results
    Owner: Probe-Server, Test: testsvr
    Target address: 1.1.1.1, Probe type: icmp-ping
    Destination interface name: ge-0/0/0.0
    Test size: 10 probes
    Probe results:
      Internal error, Thu Mar 19 07:49:45 2020
    Results over current test:
      Probes sent: 10, Probes received: 2, Loss percentage: 80
      Measurement: Round trip time
        Samples: 2, Minimum: 2644 usec, Maximum: 9208 usec, Average: 5926 usec,
        Peak to peak: 6564 usec, Stddev: 3282 usec, Sum: 11852 usec
    Results over last test:
      Probes sent: 10, Probes received: 2, Loss percentage: 80
      Test completed on Thu Mar 19 07:49:50 2020
      Measurement: Round trip time
        Samples: 2, Minimum: 2644 usec, Maximum: 9208 usec, Average: 5926 usec,
        Peak to peak: 6564 usec, Stddev: 3282 usec, Sum: 11852 usec
    Results over all tests:
      Probes sent: 1850, Probes received: 1841, Loss percentage: 0
      Measurement: Round trip time
        Samples: 1841, Minimum: 2485 usec, Maximum: 48585 usec,
        Average: 4015 usec, Peak to peak: 46100 usec, Stddev: 5453 usec,
        Sum: 7391625 usec

    Owner: Probe-Server1, Test: testsvr
    Target address: 2.2.2.1, Probe type: icmp-ping
    Destination interface name: ge-0/0/1.0
    Test size: 10 probes
    Probe results:
      Response received, Thu Mar 19 07:49:55 2020, No hardware timestamps
      Rtt: 2044 usec
    Results over current test:
      Probes sent: 1, Probes received: 1, Loss percentage: 0
      Measurement: Round trip time
        Samples: 1, Minimum: 2044 usec, Maximum: 2044 usec, Average: 2044 usec,
        Peak to peak: 0 usec, Stddev: 0 usec, Sum: 2044 usec
    Results over last test:
      Probes sent: 10, Probes received: 10, Loss percentage: 0
      Test completed on Thu Mar 19 07:49:45 2020
      Measurement: Round trip time
        Samples: 10, Minimum: 1845 usec, Maximum: 1969 usec, Average: 1928 usec,
        Peak to peak: 124 usec, Stddev: 34 usec, Sum: 19275 usec
    Results over all tests:
      Probes sent: 1851, Probes received: 1851, Loss percentage: 0
      Measurement: Round trip time
        Samples: 1851, Minimum: 1756 usec, Maximum: 3600661 usec,
        Average: 4064 usec, Peak to peak: 3598905 usec, Stddev: 83622 usec,
        Sum: 7521863 usec

and

show services ip-monitoring status

Policy - Server-Tracking (Status: FAIL)
  RPM Probes:
    Probe name             Test Name       Address          Status
    ---------------------- --------------- ---------------- ---------
    Probe-Server           testsvr        1.1.1.1     FAIL
  Route-Action:
    route-instance    route             next-hop         state
    ----------------- ----------------- ---------------- -------------
    FBF-1             0.0.0.0/0         2.2.2.1     APPLIED

Policy - Server-Tracking1 (Status: PASS)
  RPM Probes:
    Probe name             Test Name       Address          Status
    ---------------------- --------------- ---------------- ---------
    Probe-Server1          testsvr         2.2.2.1     PASS
  Route-Action:
    route-instance    route             next-hop         state
    ----------------- ----------------- ---------------- -------------
    FBF-2             0.0.0.0/0         1.1.1.1    NOT-APPLIED

So it looks like RPM is detecting a failure, but I still can't ping the upstream gateway, it seems like my routing instances are either wrong or the wrong one is getting inserted? I'm still trying to get my head around what mistakes I may have made here:

firewall {
    filter F1 {
        term 1 {
            from {
                source-address {
                    192.168.8.100/32;
                }
            }
            then {
                routing-instance FBF-2;
            }
        }
        term 2 {
            from {
                source-address {
                    0.0.0.0/0;
                }
            }
            then {
                routing-instance FBF-1;
            }
        }
    }
}
routing-instances {
    FBF-1 {
        instance-type forwarding;
        routing-options {
            static {
                route 0.0.0.0/0 next-hop 1.1.1.1;
            }
        }
    }
    FBF-2 {
        instance-type forwarding;
        routing-options {
            static {
                route 0.0.0.0/0 next-hop 2.2.2.1;
            }
        }
    }
}

I think I'm mixing up something simple? Here are my routes:

show route

inet.0: 7 destinations, 7 routes (7 active, 0 holddown, 0 hidden)
+ = Active Route, - = Last Active, * = Both

1.1.1.2/32    *[Local/0] 03:10:18
                      Reject
172.16.30.1/32     *[Local/0] 03:10:17
                      Reject
192.168.8.0/24     *[Direct/0] 03:10:13
                    > via ge-0/0/2.0
192.168.8.1/32     *[Local/0] 03:10:17
                      Local via ge-0/0/2.0
2.2.2.0/24    *[Direct/0] 03:10:14
                    > via ge-0/0/1.0
2.2.2.150/32  *[Local/0] 03:10:18
                      Local via ge-0/0/1.0
192.168.20.1/32    *[Local/0] 03:10:17
                      Reject

FBF-1.inet.0: 8 destinations, 8 routes (8 active, 0 holddown, 0 hidden)
+ = Active Route, - = Last Active, * = Both

0.0.0.0/0          *[Static/1] 00:09:48, metric2 0
                    > to 192.168.13.1 via ge-0/0/1.0
1.1.1.2/32    *[Local/0] 00:21:22
                      Reject
172.16.30.1/32     *[Local/0] 03:10:17
                      Reject
192.168.8.0/24     *[Direct/0] 03:10:13
                    > via ge-0/0/2.0
192.168.8.1/32     *[Local/0] 03:10:13
                      Local via ge-0/0/2.0
2.2.2.0/24    *[Direct/0] 03:10:14
                    > via ge-0/0/1.0
2.2.2.150/32  *[Local/0] 03:10:14
                      Local via ge-0/0/1.0
192.168.20.1/32    *[Local/0] 03:10:17
                      Reject

FBF-2.inet.0: 8 destinations, 8 routes (8 active, 0 holddown, 0 hidden)
+ = Active Route, - = Last Active, * = Both

0.0.0.0/0          *[Static/5] 03:10:14
                    > to 192.168.13.1 via ge-0/0/1.0
1.1.1.2/32    *[Local/0] 00:21:22
                      Reject
172.16.30.1/32     *[Local/0] 03:10:17
                      Reject
192.168.8.0/24     *[Direct/0] 03:10:13
                    > via ge-0/0/2.0
192.168.8.1/32     *[Local/0] 03:10:13
                      Local via ge-0/0/2.0
2.2.2.0/24    *[Direct/0] 03:10:14
                    > via ge-0/0/1.0
2.2.2.150/32  *[Local/0] 03:10:14
                      Local via ge-0/0/1.0
192.168.20.1/32    *[Local/0] 03:10:17
                      Reject
Highlighted
SRX Services Gateway

Re: FBF filter-based forwarding dual ISP failover

Wednesday

The “show route” you shared is from when the ISP 1 was disconnected? I don’t see the default-route getting applied:

 

FBF-2.inet.0: 8 destinations, 8 routes (8 active, 0 holddown, 0 hidden)
+ = Active Route, - = Last Active, * = Both

0.0.0.0/0          *[Static/5] 03:10:14
                    > to 192.168.13.1 via ge-0/0/1.0

 

Please mark my answer as the Solution if it applies.