Using ESP8266, NodeMCU and MQTT to create a wirelessly connected 'installation' - part 2

This weekend was time for some field tests of the networking component.

I'd had this working at home for a while but we're setting up at an Airsoft site which is a fairly dense wood with large changes in elevation. There's also an assortment of fake buildings bashed together from wood and for some reason, old aluminium garage doors.

The site is roughly 90 acres in size and while we won't be using all of it we are going to be using the areas that have buildings. The radio tech I used last year really struggled with propagation through the trees and ended up with about a 50-100m range. This doesn't really cut it so it was imperative that I tested the new setup.

Each node is a plastic storage box filled with the following.

  • Fortigate firewall, I had two 50Bs and two 60ADSLs. Old ones like this are plentiful and cheap on eBay and I already had these. I work with these professionally so I'm 100% comfortable configuring them. You get pretty much everything you might want from a firewall/router appliance even when forced to run very old versions of FortiOS. These SOHO models even have a small network switch integrated.
  • Huawei E160G USB 3G modem. These slightly old 3G modems are also plentiful on eBay and I know they 100% work with FortiOS. Simply plug them into a USB port of the firewall and with a few lines of config you're ready to go.
  • Netgear WG102 wireless access point. I just happened to have picked four up previously and they support a point-to-multpoint bridging mode that could connect all the nodes. They're old and only support 802.11b/g over 2.4Ghz but performance isn't the thing we need.
  • 4W 2.4Ghz Wifi amplifier from China. We're working in a remote area with nobody to interfere with and I really needed the range this would give.
  • 30AH 12V sealed lead-acid battery, another thing I had four of lurking at home. With all the components working off external PSUs that supply 12V then the ~12.5-13V these kick out meant I didn't have to mess around with any other DC-DC conversion to power things.
I'm glad to say the test worked perfectly, with me able to cover the whole area of the site we intend to use acceptably in Wifi. The most distant node used the 3G backup as those Wifi amplifiers do work but trees in leaf do very bad things to radio propagation.

For those interested in the config, here's what I put on each firewall. It was done as a base script you can just copy and paste on then a second one to modify it specifically for each node. This could also mostly be done through the firewall GUI but making a command line script helps with making four identical nodes.

execute batch start
config system admin
    edit "admin"
        set password XXXXX
    next
end

config system global
    set admintimeout 90
    set hostname nodeX
    set timezone 25
    set dst enable
end

config system ntp
    set ntpsync enable
    set syncinterval 30
    config ntpserver
        edit 1
            set server pool.ntp.org
        next
    end
end

config system modem
    set status enable
    set dial-on-demand disable
    set auto-dial enable
    set idle-timer 1
    set redial 10
    set phone1 "*99#"
    set distance 100
end

config system interface
    edit "internal"
        set mode static
        unset ip
        set allowaccess ping https ssh
    next
    edit wan1
        set mode static
        unset ip
        set allowaccess ping https ssh
            config secondaryip
                edit 1
                    set detectserver "0.0.0.0"
                    set ip 192.168.0.1 255.255.255.0
                next
            end
    next
    edit wan2
        set mode static
        unset ip
        set allowaccess ping https ssh
    next
    edit "wifi_clients"
        set vdom root
        set type vlan
        set vlanid 2
        set interface wan1
        set mode static
        unset ip
        set allowaccess ping https ssh
    next
    edit "modem"
        set allowaccess ping https
        set ddns enable
        set ddns-server dyndns.org
        set ddns-domain "XXXXXXXXXX.homeip.net"
        set ddns-username "XXXXXXXXXXXXXX"
        set ddns-password XXXXXXXXXXXXXXXXX
    next
end

config system dhcp server
    delete "internal_dhcp_server"
    edit "internal"
        set default-gateway 10.254.1.1
        set start-ip 10.254.1.2
        set end-ip 10.254.1.254
        set dns-server1 8.8.8.8
        set interface "internal"
        set netmask 255.255.255.0
    next
    edit "wan1"
        set default-gateway 10.0.0.254
        set start-ip 10.0.0.5
        set end-ip 10.0.0.254
        set dns-server1 8.8.8.8
        set interface "wan1"
        set netmask 255.255.255.0
    next
    edit "wifi_clients"
        set default-gateway 10.254.3.1
        set start-ip 10.254.3.2
        set end-ip 10.254.3.254
        set dns-server1 8.8.8.8
        set interface "wifi_clients"
        set netmask 255.255.255.0
    next
end

config system dhcp reserved-address
    edit "ap1"
        set ip 10.0.0.5
        set mac 00:1b:2f:96:2b:cb
    next
    edit "ap2"
        set ip 10.0.0.6
        set mac 00:1b:2f:96:29:ab
    next
    edit "ap3"
        set ip 10.0.0.7
        set mac 00:1b:2f:98:40:d1
    next
    edit "ap4"
        set ip 10.0.0.8
        set mac 00:1e:2a:15:a4:4a
    next
end

config vpn ipsec phase1-interface
    edit "tunnel"
        set interface "modem"
        set dpd enable
        set nattraversal enable
        set proposal 3des-sha1 3des-md5
        set mode aggressive
        set remote-gw 1.2.3.4
        set psksecret XXXXXXXXXXXXX
        set localid XXXXXXXXX
        set peertype one
        set peerid XXXXXXXXXXX
    next
end

config router static
    delete 1
end

config firewall policy
    delete 1
end

config system zone
    edit this_node
        set interface "internal" "wifi_clients"
        set intrazone allow
    next
    edit elsewhere
        set interface "modem" "tunnel"
        set intrazone allow
    next
    edit mesh
        set interface "wan1"
        set intrazone allow
    next
end

config firewall address
    edit "mesh"
        set subnet 10.0.0.0 255.255.255.0
    next
    edit "wifi_clients"
        set subnet 10.1.0.0 255.255.255.0
    next
    edit "node1"
        set subnet 10.1.0.0 255.255.0.0
    next
    edit "node2"
        set subnet 10.2.0.0 255.255.0.0
    next
    edit "node3"
        set subnet 10.3.0.0 255.255.0.0
    next
    edit "node4"
        set subnet 10.4.0.0 255.255.0.0
    next
    edit "ap_default"
        set subnet 192.168.0.229 255.255.255.255
    next
end

config firewall addrgrp
    edit "this_node"
        set member "node2"
    next
    edit "other_nodes"
        set member "node1" "node3" "node4"
    next
end

config vpn ipsec phase2-interface
    edit "tunnel"
        set phase1name "tunnel"
        set keepalive enable
        set pfs enable
        set proposal 3des-sha1 3des-md5
        set src-addr-type name
        set dst-addr-type name
        set src-name "this_node"
        set dst-name "other_nodes"
        set auto-negotiate enable
    next
end


config router ospf
        config area
            edit 10.0.0.0
                set authentication md5
            next
        end
        config network
            edit 1
                set area 10.0.0.0
                set prefix 10.0.0.0 255.255.255.0
            next
        end
        config redistribute "connected"
            set status enable
        end
    set router-id 10.0.0.254
    set default-information-originate enable
    set passive-interface tunnel internal
end

config firewall policy
    edit 1
        set srcintf this_node
        set dstintf elsewhere
        set srcaddr this_node
        set dstaddr other_nodes
        set service ANY
        set action accept
        set schedule always
        set nat disable
    next
    edit 2
        set srcintf this_node
        set dstintf elsewhere
        set srcaddr this_node
        set dstaddr all
        set service ANY
        set action accept
        set schedule always
        set nat enable
    next
    edit 3
        set srcintf elsewhere
        set dstintf this_node
        set srcaddr other_nodes
        set dstaddr this_node
        set service ANY
        set action accept
        set schedule always
        set nat disable
    next
    edit 4
        set srcintf elsewhere
        set dstintf elsewhere
        set srcaddr other_nodes
        set dstaddr other_nodes
        set service ANY
        set action accept
        set schedule always
        set nat disable
    next
    edit 5
        set srcintf elsewhere
        set dstintf elsewhere
        set srcaddr other_nodes
        set dstaddr all
        set service ANY
        set action accept
        set schedule always
        set nat enable
    next
    edit 6
        set srcintf "this_node"
        set dstintf "mesh"
            set srcaddr "this_node"
            set dstaddr "ap_default"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat enable
    next
    edit 7
        set srcintf "this_node"
        set dstintf "mesh"
            set srcaddr "this_node"
            set dstaddr "other_nodes"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat disable
    next
    edit 8
        set srcintf "this_node"
        set dstintf "mesh"
            set srcaddr "this_node"
            set dstaddr "mesh"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat enable
    next
    edit 9
        set srcintf "mesh"
        set dstintf "this_node"
            set srcaddr "mesh" "other_nodes"
            set dstaddr "this_node"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat disable
    next
    edit 10
        set srcintf "mesh"
        set dstintf "elsewhere"
            set srcaddr "all"
            set dstaddr "all"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat enable
    next
end

config log memory setting
    set status enable
end

config log memory filter
    set event enable
    set admin enable
    set auth enable
    set cpu-memory-usage enable
    set dhcp enable
    set ha enable
    set ipsec enable
    set ldb-monitor enable
    set pattern enable
    set ppp enable
    set sslvpn-log-adm enable
    set sslvpn-log-auth enable
    set sslvpn-log-session enable
    set system enable
end

Then for each node I then had something like this.

config system global
    set hostname node1
end

config system interface
    edit "internal"
        set ip 10.1.1.1 255.255.255.0
    next
    edit "wan1"
        set ip 10.0.0.1 255.255.255.0
    next
    edit "wifi_clients"
        set ip 10.1.2.1 255.255.255.0
    next
    edit "modem"
        set ddns-domain "XXXXXXXXX.homeip.net"
    next
end

config router ospf
    set router-id 10.0.0.1
end

config system dhcp server
    edit "internal"
        set default-gateway 10.1.1.1
        set start-ip 10.1.1.2
        set end-ip 10.1.1.254
    next
    edit "wan1"
        set default-gateway 10.1.0.1
    next
    edit "wifi_clients"
        set default-gateway 10.1.2.1
        set start-ip 10.1.2.2
        set end-ip 10.1.2.254
    next
end

config vpn ipsec phase1-interface
    edit "tunnel"
        set psksecret XXXXXXXXXXXXXXXXXX
        set peerid Hub1
        set localid Node1
    next
end

config system interface
    edit "tunnel"
        set ip 10.253.0.2 255.255.255.255
        set remote-ip 10.253.0.1
        set allowaccess ping https ssh
    next
end

config router static
    edit 1
        set device "tunnel"
        set dst 10.2.0.0 255.255.0.0
        set distance 128
    next
    edit 2
        set device "tunnel"
        set dst 10.3.0.0 255.255.0.0
        set distance 128
    next
    edit 3
        set device "tunnel"
        set dst 10.4.0.0 255.255.0.0
        set distance 128
    next
end

config firewall addrgrp
    edit "this_node"
        set member "node1"
    next
    edit "other_nodes"
        set member "node2" "node3" "node4"
    next
end
execute batch end
There's no real attempt at security or firewalling the Fortinet's just being used as a router with basic OSPF and a VPN plus a couple of local networks at each node. I may tidy it up later.

At the other end they connect to there's config like this.

config vpn ipsec phase1-interface
    edit "Node1"
        set type dynamic
        set interface "portA1"
        set peertype one
        set mode aggressive
        set proposal 3des-sha1 3des-md5
        set localid "Hub1"
        set peerid "Node1"
        set psksecret XXXXXXXXXXXXXXXXXX
    next
    edit "Node2"
        set type dynamic
        set interface "portA1"
        set peertype one
        set mode aggressive
        set proposal 3des-sha1 3des-md5
        set localid "Hub2"
        set peerid "Node2"
        set psksecret XXXXXXXXXXXXXXXXXX
    next
    edit "Node3"
        set type dynamic
        set interface "portA1"
        set peertype one
        set mode aggressive
        set proposal 3des-sha1 3des-md5
        set localid "Hub3"
        set peerid "Node3"
        set psksecret XXXXXXXXXXXXXXXXXX
    next
    edit "Node4"
        set type dynamic
        set interface "portA1"
        set peertype one
        set mode aggressive
        set proposal 3des-sha1 3des-md5
        set localid "Hub4"
        set peerid "Node4"
        set psksecret XXXXXXXXXXXXXXXXXX
    next
end
config firewall address
    edit "Node1"
        set subnet 10.1.0.0 255.255.0.0
    next
    edit "Node2"
        set subnet 10.2.0.0 255.255.0.0
    next
    edit "Node3"
        set subnet 10.3.0.0 255.255.0.0
    next
    edit "Node4"
        set subnet 10.4.0.0 255.255.0.0
    next
end 
config vpn ipsec phase2-interface
    edit "Node1"
        set dst-addr-type name
        set phase1name "Node1"
        set proposal 3des-sha1 aes128-sha1
        set src-addr-type name
        set dst-name "Node1"
        set src-name "Node 2,3,4"
    next
    edit "Node2"
        set dst-addr-type name
        set phase1name "Node2"
        set proposal 3des-sha1 aes128-sha1
        set src-addr-type name
        set dst-name "Node2"
        set src-name "Node 1,3,4"
    next
    edit "Node3"
        set dst-addr-type name
        set phase1name "Node3"
        set proposal 3des-sha1 aes128-sha1
        set src-addr-type name
        set dst-name "Node3"
        set src-name "Node 1,2,4"
    next
    edit "Node4"
        set dst-addr-type name
        set phase1name "Node4"
        set proposal 3des-sha1 aes128-sha1
        set src-addr-type name
        set dst-name "Node4"
        set src-name "Node 1,2,3"
    next
end
config firewall policy
    edit 1000
        set srcintf "Node1"
        set dstintf "Internet"
            set srcaddr "Node1"
            set dstaddr "all"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat enable
    next
    edit 2000
        set srcintf "Node2"
        set dstintf "Internet"
            set srcaddr "Node2"
            set dstaddr "all"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat enable
    next
    edit 1002
        set srcintf "Node1"
        set dstintf "Node2"
            set srcaddr "Node1"
            set dstaddr "Node2"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 2001
        set srcintf "Node2"
        set dstintf "Node1"
            set srcaddr "Node2"
            set dstaddr "Node1"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 1003
        set srcintf "Node1"
        set dstintf "Node3"
            set srcaddr "Node1"
            set dstaddr "Node3"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 2003
        set srcintf "Node2"
        set dstintf "Node3"
            set srcaddr "Node2"
            set dstaddr "Node3"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 3000
        set srcintf "Node3"
        set dstintf "Internet"
            set srcaddr "Node3"
            set dstaddr "all"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat enable
    next
    edit 3001
        set srcintf "Node3"
        set dstintf "Node1"
            set srcaddr "Node3"
            set dstaddr "Node1"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 3002
        set srcintf "Node3"
        set dstintf "Node2"
            set srcaddr "Node3"
            set dstaddr "Node2"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 23
        set srcintf "Node1"
        set dstintf "Node4"
            set srcaddr "Node1"
            set dstaddr "Node4"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 24
        set srcintf "Node2"
        set dstintf "Node4"
            set srcaddr "Node2"
            set dstaddr "Node4"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 25
        set srcintf "Node3"
        set dstintf "Node4"
            set srcaddr "Node3"
            set dstaddr "Node4"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 26
        set srcintf "Node4"
        set dstintf "Internet"
            set srcaddr "Node4"
            set dstaddr "all"
        set action accept
        set schedule "always"
            set service "ANY"
        set nat enable
    next
    edit 27
        set srcintf "Node4"
        set dstintf "Node1"
            set srcaddr "Node4"
            set dstaddr "Node1"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 28
        set srcintf "Node4"
        set dstintf "Node2"
            set srcaddr "Node4"
            set dstaddr "Node2"
        set action accept
        set schedule "always"
            set service "ANY"
    next
    edit 29
        set srcintf "Node4"
        set dstintf "Node3"
            set srcaddr "Node4"
            set dstaddr "Node3"
        set action accept
        set schedule "always"
            set service "ANY"
    next
end

No comments: