Using ESP8266, NodeMCU and MQTT to create a wirelessly connected 'installation' - part 2

This weekend was time for some field tests of the networking component.

I'd had this working at home for a while but we're setting up at an Airsoft site which is a fairly dense wood with large changes in elevation. There's also an assortment of fake buildings bashed together from wood and for some reason, old aluminium garage doors.

The site is roughly 90 acres in size and while we won't be using all of it we are going to be using the areas that have buildings. The radio tech I used last year really struggled with propagation through the trees and ended up with about a 50-100m range. This doesn't really cut it so it was imperative that I tested the new setup.

Each node is a plastic storage box filled with the following.

• Fortigate firewall, I had two 50Bs and two 60ADSLs. Old ones like this are plentiful and cheap on eBay and I already had these. I work with these professionally so I'm 100% comfortable configuring them. You get pretty much everything you might want from a firewall/router appliance even when forced to run very old versions of FortiOS. These SOHO models even have a small network switch integrated.
• Huawei E160G USB 3G modem. These slightly old 3G modems are also plentiful on eBay and I know they 100% work with FortiOS. Simply plug them into a USB port of the firewall and with a few lines of config you're ready to go.
• Netgear WG102 wireless access point. I just happened to have picked four up previously and they support a point-to-multpoint bridging mode that could connect all the nodes. They're old and only support 802.11b/g over 2.4Ghz but performance isn't the thing we need.
• 4W 2.4Ghz Wifi amplifier from China. We're working in a remote area with nobody to interfere with and I really needed the range this would give.
• 30AH 12V sealed lead-acid battery, another thing I had four of lurking at home. With all the components working off external PSUs that supply 12V then the ~12.5-13V these kick out meant I didn't have to mess around with any other DC-DC conversion to power things.

I'm glad to say the test worked perfectly, with me able to cover the whole area of the site we intend to use acceptably in Wifi. The most distant node used the 3G backup as those Wifi amplifiers do work but trees in leaf do very bad things to radio propagation.

For those interested in the config, here's what I put on each firewall. It was done as a base script you can just copy and paste on then a second one to modify it specifically for each node. This could also mostly be done through the firewall GUI but making a command line script helps with making four identical nodes.

execute batch start

config system admin

    edit "admin"

        set password XXXXX

    next

end

config system global

    set admintimeout 90

    set hostname nodeX

    set timezone 25

    set dst enable

end

config system ntp

    set ntpsync enable

    set syncinterval 30

    config ntpserver

        edit 1

            set server pool.ntp.org

        next

    end

end

config system modem

    set status enable

    set dial-on-demand disable

    set auto-dial enable

    set idle-timer 1

    set redial 10

    set phone1 "*99#"

    set distance 100

end

config system interface

    edit "internal"

        set mode static

        unset ip

        set allowaccess ping https ssh

    next

    edit wan1

        set mode static

        unset ip

        set allowaccess ping https ssh

            config secondaryip

                edit 1

                    set detectserver "0.0.0.0"

                    set ip 192.168.0.1 255.255.255.0

                next

            end

    next

    edit wan2

        set mode static

        unset ip

        set allowaccess ping https ssh

    next

    edit "wifi_clients"

        set vdom root

        set type vlan

        set vlanid 2

        set interface wan1

        set mode static

        unset ip

        set allowaccess ping https ssh

    next

    edit "modem"

        set allowaccess ping https

        set ddns enable

        set ddns-server dyndns.org

        set ddns-domain "XXXXXXXXXX.homeip.net"

        set ddns-username "XXXXXXXXXXXXXX"

        set ddns-password XXXXXXXXXXXXXXXXX

    next

end

config system dhcp server

    delete "internal_dhcp_server"

    edit "internal"

        set default-gateway 10.254.1.1

        set start-ip 10.254.1.2

        set end-ip 10.254.1.254

        set dns-server1 8.8.8.8

        set interface "internal"

        set netmask 255.255.255.0

    next

    edit "wan1"

        set default-gateway 10.0.0.254

        set start-ip 10.0.0.5

        set end-ip 10.0.0.254

        set dns-server1 8.8.8.8

        set interface "wan1"

        set netmask 255.255.255.0

    next

    edit "wifi_clients"

        set default-gateway 10.254.3.1

        set start-ip 10.254.3.2

        set end-ip 10.254.3.254

        set dns-server1 8.8.8.8

        set interface "wifi_clients"

        set netmask 255.255.255.0

    next

end

config system dhcp reserved-address

    edit "ap1"

        set ip 10.0.0.5

        set mac 00:1b:2f:96:2b:cb

    next

    edit "ap2"

        set ip 10.0.0.6

        set mac 00:1b:2f:96:29:ab

    next

    edit "ap3"

        set ip 10.0.0.7

        set mac 00:1b:2f:98:40:d1

    next

    edit "ap4"

        set ip 10.0.0.8

        set mac 00:1e:2a:15:a4:4a

    next

end

config vpn ipsec phase1-interface

    edit "tunnel"

        set interface "modem"

        set dpd enable

        set nattraversal enable

        set proposal 3des-sha1 3des-md5

        set mode aggressive

        set remote-gw 1.2.3.4

        set psksecret XXXXXXXXXXXXX

        set localid XXXXXXXXX

        set peertype one

        set peerid XXXXXXXXXXX

    next

end

config router static

    delete 1

end

config firewall policy

    delete 1

end

config system zone

    edit this_node

        set interface "internal" "wifi_clients"

        set intrazone allow

    next

    edit elsewhere

        set interface "modem" "tunnel"

        set intrazone allow

    next

    edit mesh

        set interface "wan1"

        set intrazone allow

    next

end

config firewall address

    edit "mesh"

        set subnet 10.0.0.0 255.255.255.0

    next

    edit "wifi_clients"

        set subnet 10.1.0.0 255.255.255.0

    next

    edit "node1"

        set subnet 10.1.0.0 255.255.0.0

    next

    edit "node2"

        set subnet 10.2.0.0 255.255.0.0

    next

    edit "node3"

        set subnet 10.3.0.0 255.255.0.0

    next

    edit "node4"

        set subnet 10.4.0.0 255.255.0.0

    next

    edit "ap_default"

        set subnet 192.168.0.229 255.255.255.255

    next

end

config firewall addrgrp

    edit "this_node"

        set member "node2"

    next

    edit "other_nodes"

        set member "node1" "node3" "node4"

    next

end

config vpn ipsec phase2-interface

    edit "tunnel"

        set phase1name "tunnel"

        set keepalive enable

        set pfs enable

        set proposal 3des-sha1 3des-md5

        set src-addr-type name

        set dst-addr-type name

        set src-name "this_node"

        set dst-name "other_nodes"

        set auto-negotiate enable

    next

end

config router ospf

        config area

            edit 10.0.0.0

                set authentication md5

            next

        end

        config network

            edit 1

                set area 10.0.0.0

                set prefix 10.0.0.0 255.255.255.0

            next

        end

        config redistribute "connected"

            set status enable

        end

    set router-id 10.0.0.254

    set default-information-originate enable

    set passive-interface tunnel internal

end

config firewall policy

    edit 1

        set srcintf this_node

        set dstintf elsewhere

        set srcaddr this_node

        set dstaddr other_nodes

        set service ANY

        set action accept

        set schedule always

        set nat disable

    next

    edit 2

        set srcintf this_node

        set dstintf elsewhere

        set srcaddr this_node

        set dstaddr all

        set service ANY

        set action accept

        set schedule always

        set nat enable

    next

    edit 3

        set srcintf elsewhere

        set dstintf this_node

        set srcaddr other_nodes

        set dstaddr this_node

        set service ANY

        set action accept

        set schedule always

        set nat disable

    next

    edit 4

        set srcintf elsewhere

        set dstintf elsewhere

        set srcaddr other_nodes

        set dstaddr other_nodes

        set service ANY

        set action accept

        set schedule always

        set nat disable

    next

    edit 5

        set srcintf elsewhere

        set dstintf elsewhere

        set srcaddr other_nodes

        set dstaddr all

        set service ANY

        set action accept

        set schedule always

        set nat enable

    next

    edit 6

        set srcintf "this_node"

        set dstintf "mesh"

            set srcaddr "this_node"

            set dstaddr "ap_default"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat enable

    next

    edit 7

        set srcintf "this_node"

        set dstintf "mesh"

            set srcaddr "this_node"

            set dstaddr "other_nodes"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat disable

    next

    edit 8

        set srcintf "this_node"

        set dstintf "mesh"

            set srcaddr "this_node"

            set dstaddr "mesh"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat enable

    next

    edit 9

        set srcintf "mesh"

        set dstintf "this_node"

            set srcaddr "mesh" "other_nodes"

            set dstaddr "this_node"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat disable

    next

    edit 10

        set srcintf "mesh"

        set dstintf "elsewhere"

            set srcaddr "all"

            set dstaddr "all"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat enable

    next

end

config log memory setting

    set status enable

end

config log memory filter

    set event enable

    set admin enable

    set auth enable

    set cpu-memory-usage enable

    set dhcp enable

    set ha enable

    set ipsec enable

    set ldb-monitor enable

    set pattern enable

    set ppp enable

    set sslvpn-log-adm enable

    set sslvpn-log-auth enable

    set sslvpn-log-session enable

    set system enable

end

Then for each node I then had something like this.

config system global

    set hostname node1

end

config system interface

    edit "internal"

        set ip 10.1.1.1 255.255.255.0

    next

    edit "wan1"

        set ip 10.0.0.1 255.255.255.0

    next

    edit "wifi_clients"

        set ip 10.1.2.1 255.255.255.0

    next

    edit "modem"

        set ddns-domain "XXXXXXXXX.homeip.net"

    next

end

config router ospf

    set router-id 10.0.0.1

end

config system dhcp server

    edit "internal"

        set default-gateway 10.1.1.1

        set start-ip 10.1.1.2

        set end-ip 10.1.1.254

    next

    edit "wan1"

        set default-gateway 10.1.0.1

    next

    edit "wifi_clients"

        set default-gateway 10.1.2.1

        set start-ip 10.1.2.2

        set end-ip 10.1.2.254

    next

end

config vpn ipsec phase1-interface

    edit "tunnel"

        set psksecret XXXXXXXXXXXXXXXXXX

        set peerid Hub1

        set localid Node1

    next

end

config system interface

    edit "tunnel"

        set ip 10.253.0.2 255.255.255.255

        set remote-ip 10.253.0.1

        set allowaccess ping https ssh

    next

end

config router static

    edit 1

        set device "tunnel"

        set dst 10.2.0.0 255.255.0.0

        set distance 128

    next

    edit 2

        set device "tunnel"

        set dst 10.3.0.0 255.255.0.0

        set distance 128

    next

    edit 3

        set device "tunnel"

        set dst 10.4.0.0 255.255.0.0

        set distance 128

    next

end

config firewall addrgrp

    edit "this_node"

        set member "node1"

    next

    edit "other_nodes"

        set member "node2" "node3" "node4"

    next

end

execute batch end

There's no real attempt at security or firewalling the Fortinet's just being used as a router with basic OSPF and a VPN plus a couple of local networks at each node. I may tidy it up later.

At the other end they connect to there's config like this.

config vpn ipsec phase1-interface

    edit "Node1"

        set type dynamic

        set interface "portA1"

        set peertype one

        set mode aggressive

        set proposal 3des-sha1 3des-md5

        set localid "Hub1"

        set peerid "Node1"

        set psksecret XXXXXXXXXXXXXXXXXX

    next

    edit "Node2"

        set type dynamic

        set interface "portA1"

        set peertype one

        set mode aggressive

        set proposal 3des-sha1 3des-md5

        set localid "Hub2"

        set peerid "Node2"

        set psksecret XXXXXXXXXXXXXXXXXX

    next

    edit "Node3"

        set type dynamic

        set interface "portA1"

        set peertype one

        set mode aggressive

        set proposal 3des-sha1 3des-md5

        set localid "Hub3"

        set peerid "Node3"

        set psksecret XXXXXXXXXXXXXXXXXX

    next

    edit "Node4"

        set type dynamic

        set interface "portA1"

        set peertype one

        set mode aggressive

        set proposal 3des-sha1 3des-md5

        set localid "Hub4"

        set peerid "Node4"

        set psksecret XXXXXXXXXXXXXXXXXX

    next

end

config firewall address

    edit "Node1"

        set subnet 10.1.0.0 255.255.0.0

    next

    edit "Node2"

        set subnet 10.2.0.0 255.255.0.0

    next

    edit "Node3"

        set subnet 10.3.0.0 255.255.0.0

    next

    edit "Node4"

        set subnet 10.4.0.0 255.255.0.0

    next

end 

config vpn ipsec phase2-interface

    edit "Node1"

        set dst-addr-type name

        set phase1name "Node1"

        set proposal 3des-sha1 aes128-sha1

        set src-addr-type name

        set dst-name "Node1"

        set src-name "Node 2,3,4"

    next

    edit "Node2"

        set dst-addr-type name

        set phase1name "Node2"

        set proposal 3des-sha1 aes128-sha1

        set src-addr-type name

        set dst-name "Node2"

        set src-name "Node 1,3,4"

    next

    edit "Node3"

        set dst-addr-type name

        set phase1name "Node3"

        set proposal 3des-sha1 aes128-sha1

        set src-addr-type name

        set dst-name "Node3"

        set src-name "Node 1,2,4"

    next

    edit "Node4"

        set dst-addr-type name

        set phase1name "Node4"

        set proposal 3des-sha1 aes128-sha1

        set src-addr-type name

        set dst-name "Node4"

        set src-name "Node 1,2,3"

    next

end

config firewall policy

    edit 1000

        set srcintf "Node1"

        set dstintf "Internet"

            set srcaddr "Node1"

            set dstaddr "all"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat enable

    next

    edit 2000

        set srcintf "Node2"

        set dstintf "Internet"

            set srcaddr "Node2"

            set dstaddr "all"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat enable

    next

    edit 1002

        set srcintf "Node1"

        set dstintf "Node2"

            set srcaddr "Node1"

            set dstaddr "Node2"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 2001

        set srcintf "Node2"

        set dstintf "Node1"

            set srcaddr "Node2"

            set dstaddr "Node1"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 1003

        set srcintf "Node1"

        set dstintf "Node3"

            set srcaddr "Node1"

            set dstaddr "Node3"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 2003

        set srcintf "Node2"

        set dstintf "Node3"

            set srcaddr "Node2"

            set dstaddr "Node3"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 3000

        set srcintf "Node3"

        set dstintf "Internet"

            set srcaddr "Node3"

            set dstaddr "all"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat enable

    next

    edit 3001

        set srcintf "Node3"

        set dstintf "Node1"

            set srcaddr "Node3"

            set dstaddr "Node1"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 3002

        set srcintf "Node3"

        set dstintf "Node2"

            set srcaddr "Node3"

            set dstaddr "Node2"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 23

        set srcintf "Node1"

        set dstintf "Node4"

            set srcaddr "Node1"

            set dstaddr "Node4"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 24

        set srcintf "Node2"

        set dstintf "Node4"

            set srcaddr "Node2"

            set dstaddr "Node4"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 25

        set srcintf "Node3"

        set dstintf "Node4"

            set srcaddr "Node3"

            set dstaddr "Node4"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 26

        set srcintf "Node4"

        set dstintf "Internet"

            set srcaddr "Node4"

            set dstaddr "all"

        set action accept

        set schedule "always"

            set service "ANY"

        set nat enable

    next

    edit 27

        set srcintf "Node4"

        set dstintf "Node1"

            set srcaddr "Node4"

            set dstaddr "Node1"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 28

        set srcintf "Node4"

        set dstintf "Node2"

            set srcaddr "Node4"

            set dstaddr "Node2"

        set action accept

        set schedule "always"

            set service "ANY"

    next

    edit 29

        set srcintf "Node4"

        set dstintf "Node3"

            set srcaddr "Node4"

            set dstaddr "Node3"

        set action accept

        set schedule "always"

            set service "ANY"

    next

end