Solaris 11 Networking

Switch to manual configuration

To disable automatic procedures to take back your changes you have to enable the manual configuration mode.

# netadm enable -p ncp DefaultFixed

Nodename

# svccfg -s svc:/system/identity:node setprop config/nodename = astring: camponotus
# svcadm refresh svc:/system/identity:node
# svcadm restart svc:/system/identity:node

Interfaces

Initial setup

# ipadm create-ip net1 
# ipadm create-addr -T static -a local=192.168.5.101/24 net1/v4mailcluster1

IPMP

# ipadm create-ip net2
# ipadm create-ip net3
# ipadm create-addr -T static -a 192.168.5.102/24 net2/v4ipmptestadress
# ipadm create-addr -T static -a 192.168.5.103/24 net3/v4ipmptestadress
# ipadm create-ipmp ipmp0
# ipadm add-ipmp -i net2 -i net3 ipmp0
# ipadm create-addr -T static -a 192.168.5.101/24 ipmp0/v4mailcluster0

# ipmpstat -i
INTERFACE   ACTIVE  GROUP       FLAGS     LINK      PROBE     STATE
net2        yes     ipmp0       -------   up        ok        ok
net3        yes     ipmp0       --mbM--   up        ok        ok

# ipmpstat -an
ADDRESS                   STATE  GROUP       INBOUND     OUTBOUND
::                        down   ipmp0       --          --
192.168.5.101              up     ipmp0       net3        net2 net3

Set one interface to standby:

# ipadm set-ifprop -p standby=on -m ip net2
# ipmpstat -i
INTERFACE   ACTIVE  GROUP       FLAGS     LINK      PROBE     STATE
net3        yes     ipmp0       --mbM--   up        ok        ok
net2        no      ipmp0       is-----   up        ok        ok
# ipmpstat -g
GROUP       GROUPNAME   STATE     FDT       INTERFACES
ipmp0       ipmp0       ok        10.00s    net3 (net2)

More sophisticated with aggregations and vnics

# dladm show-phys -L
LINK              DEVICE       LOC
net0              igb12        /SYS/MB
net1              igb13        /SYS/MB
net2              igb14        /SYS/MB
net3              igb15        /SYS/MB
net4              igb0         /SYS/MB/PCI_MEZZ/PCIE3
net5              igb1         /SYS/MB/PCI_MEZZ/PCIE3
net6              igb2         /SYS/MB/PCI_MEZZ/PCIE3
net7              igb3         /SYS/MB/PCI_MEZZ/PCIE3
net8              igb4         /SYS/MB/RISER2/PCIE2
net9              igb5         /SYS/MB/RISER2/PCIE2
net10             igb6         /SYS/MB/RISER2/PCIE2
net11             igb7         /SYS/MB/RISER2/PCIE2
net12             igb8         /SYS/MB/RISER0/PCIE0
net13             igb9         /SYS/MB/RISER0/PCIE0
net14             igb10        /SYS/MB/RISER0/PCIE0
net15             igb11        /SYS/MB/RISER0/PCIE0
net16             usbecm2      --
# dladm create-aggr -P L2,L3 -l net8 -l net9 -l net10 -l net11 PCIE2 
# dladm create-aggr -P L2,L3 -l net4 -l net5 -l net6 -l net7 PCIE3
# dladm show-link
...
PCIE2               aggr      1500   up       net8 net9 net10 net11
PCIE3               aggr      1500   up       net4 net5 net6 net7
...
# dladm create-vnic -l PCIE2 zone01_ipmp0
# dladm create-vnic -l PCIE3 zone01_ipmp1
# dladm show-link
...
zone01_ipmp1      vnic      1500   up       PCIE3
zone01_ipmp0      vnic      1500   up       PCIE2
...
# zonecfg -z zone01
zonecfg:zone01> add net
zonecfg:zone01:net> set configure-allowed-address=true
zonecfg:zone01:net> set physical=zone01_ipmp0
zonecfg:zone01:net> end
zonecfg:zone01> add net
zonecfg:zone01:net> set configure-allowed-address=true
zonecfg:zone01:net> set physical=zone01_ipmp1
zonecfg:zone01:net> end
zonecfg:zone01> verify
zonecfg:zone01> commit
zonecfg:zone01> exit

Change address

1. Create new interface:

# ipadm create-addr -T static -a 192.168.5.111/24 ipmp0/v4mailcluster1

2. Login to new IP.

3. Delete the old interface:

# ipadm delete-addr ipmp0/v4mailcluster0

DNS

Client

# svccfg -s svc:/network/dns/client setprop config/nameserver = net_address: "( 0.0.0.0 192.168.1.1 )"
# svccfg -s svc:/network/dns/client setprop config/search = astring: "timmann.de blindhuhn.de"
# svcadm refresh svc:/network/dns/client:default
# svcadm restart svc:/network/dns/client:default

Activate dns in nameservice switch (nsswitch.conf):

# perl -pi -e "s/^hosts:\s+files$/hosts: files dns/g" /etc/nsswitch.conf
# nscfg import -f svc:/system/name-service/switch:default
# svcadm refresh name-service/switch
# svcprop -p config/host svc:/system/name-service/switch:default
files\ dns

Server

# groupadd -g 53 dns
# useradd -u 53 -g dns -d /var/named -m dns
# usermod -A solaris.smf.manage.bind dns
# svccfg -s svc:network/dns/server:default setprop start/group = dns
# svccfg -s svc:network/dns/server:default setprop start/user  = dns
# svccfg -s svc:network/dns/server:default setprop options/ip_interfaces = IPv4
# svccfg -s svc:network/dns/server:default setprop options/configuration_file = /etc/named.conf
# svcadm refresh svc:network/dns/server:default
# svcadm enable  svc:network/dns/server:default

Set tcp/udp parameter (formerly ndd)

# ipadm show-prop -p smallest_anon_port tcp
PROTO PROPERTY            PERM CURRENT      PERSISTENT   DEFAULT      POSSIBLE
tcp   smallest_anon_port  rw   1024         --           1024         1024-65535

# ipadm set-prop -p smallest_anon_port=9000 tcp
# ipadm set-prop -p smallest_anon_port=9000 udp
# ipadm set-prop -p largest_anon_port=65500 tcp
# ipadm set-prop -p largest_anon_port=65500 udp

Jumbo Frames

The MTU of an ipadm-interface can never be greater than its underlying dladm-interface. To change the dladm-interface the ipadm-interface has to be disabled (DOWNTIME!? BE CAREFUL!).

# ipadm disable-if -t iscsi0
# dladm set-linkprop -p mtu=9000  iscsi0
# ipadm enable-if -t iscsi0
# ipadm set-ifprop -m ipv4 -p mtu=9000 iscsi0

Aggregate for iSCSI

This is cruel but worked on our ciscos:

# dladm create-aggr -m trunk -P L4 -L off "-l iscsi"{0..7} iscsi_aggr0 | /bin/sh
# dladm show-aggr -P iscsi_aggr0
LINK              MODE  POLICY   ADDRPOLICY           LACPACTIVITY LACPTIMER
iscsi_aggr0       trunk L4       auto                 off          short
# dladm show-aggr -L iscsi_aggr0
LINK                PORT         AGGREGATABLE SYNC COLL DIST DEFAULTED EXPIRED
iscsi_aggr0         iscsi0       no           no   no   no   yes       no
--                  iscsi1       no           no   no   no   yes       no
--                  iscsi2       no           no   no   no   yes       no
--                  iscsi3       no           no   no   no   yes       no
--                  iscsi4       no           no   no   no   yes       no
--                  iscsi5       no           no   no   no   yes       no
--                  iscsi6       no           no   no   no   yes       no
--                  iscsi7       no           no   no   no   yes       no

Set TCP parameters in immutable zones

In normal immutable mode zlogin -U does not change it:

root@global# zlogin -U immutable-zone ipadm set-prop -p _time_wait_interval=30000 tcp
ipadm: set-prop: _time_wait_interval: Invalid argument provided

root@global# zlogin immutable-zone ipadm show-prop -p _time_wait_interval tcp     
PROTO PROPERTY              PERM CURRENT      PERSISTENT   DEFAULT      POSSIBLE
tcp   _time_wait_interval   rw   30000        --           60000        1000-600000

Need to boot into writable:

root@global# zoneadm -z immutable-zone reboot -w

root@global# zlogin -U immutable-zone ipadm set-prop -p _time_wait_interval=30000 tcp

root@global# zlogin immutable-zone ipadm show-prop -p _time_wait_interval tcp     
PROTO PROPERTY              PERM CURRENT      PERSISTENT   DEFAULT      POSSIBLE
tcp   _time_wait_interval   rw   30000        30000        60000        1000-600000

root@global# zoneadm -z immutable-zone reboot