Updating config.toml with more configurable parameters

* Make VXLAN and Gossip port configurable
* Configurable Default gateway network name
* Add DataDir configuration params

Signed-off-by: Madhu Venugopal <madhu@docker.com>

Author: mavenugo

cmd/dnet/dnet.go

@@ -123,6 +123,14 @@ func processConfig(cfg *config.Config) []config.Option {
 		options = append(options, config.OptionKVProviderURL(dcfg.Client.Address))
 	}
 
+	if cfg.Daemon.DefaultGwNetwork != "" {
+		options = append(options, config.OptionDefaultGwNetwork(cfg.Daemon.DefaultGwNetwork))
+	}
+
+	if cfg.Daemon.DataDir != "" {
+		options = append(options, config.OptionDataDir(cfg.Daemon.DataDir))
+	}
+
 	dOptions, err := startDiscovery(&cfg.Cluster)
 	if err != nil {
 		logrus.Infof("Skipping discovery : %s", err.Error())

config/config.go

@@ -1,7 +1,6 @@
 package config
 
 import (
-	"fmt"
 	"os"
 	"strings"
 
@@ -37,6 +36,7 @@ type DaemonCfg struct {
 	Experimental           bool
 	DataDir                string
 	DefaultNetwork         string
+	DefaultGwNetwork       string
 	DefaultDriver          string
 	Labels                 []string
 	DriverCfg              map[string]interface{}
@@ -81,14 +81,12 @@ func ParseConfig(tomlCfgFile string) (*Config, error) {
 	if !(strings.Contains(cfg.Cluster.Discovery, "etcd") || strings.Contains(cfg.Cluster.Discovery, "consul")) {
 		cfg.Cluster.Discovery = "etcd://" + cfg.Cluster.Discovery
 	}
-	fmt.Printf("cluster address=%s, discovery=%s \n", cfg.Cluster.Address, cfg.Cluster.Discovery)
 
 	if _, ok := cfg.Scopes[datastore.GlobalScope]; !ok {
 		kvParts := strings.SplitN(cfg.Cluster.Discovery, "://", 2)
 		if len(kvParts) == 2 {
 			gCfg := datastore.ScopeClientCfg{Provider: kvParts[0], Address: kvParts[1]}
 			cfg.Scopes[datastore.GlobalScope] = &datastore.ScopeCfg{gCfg}
-			fmt.Printf("KVStore provider=%s, address=%s\n", kvParts[0], kvParts[1])
 		}
 
 	}
@@ -119,11 +117,17 @@ type Option func(c *Config)
 // OptionDefaultNetwork function returns an option setter for a default network
 func OptionDefaultNetwork(dn string) Option {
 	return func(c *Config) {
-		logrus.Debugf("Option DefaultNetwork: %s", dn)
 		c.Daemon.DefaultNetwork = strings.TrimSpace(dn)
 	}
 }
 
+// OptionDefaultGwNetwork function returns an option setter for a default Gateway network
+func OptionDefaultGwNetwork(dn string) Option {
+	return func(c *Config) {
+		c.Daemon.DefaultGwNetwork = strings.TrimSpace(dn)
+	}
+}
+
 // OptionDefaultDriver function returns an option setter for default driver
 func OptionDefaultDriver(dd string) Option {
 	return func(c *Config) {

config/libnetwork.toml

@@ -2,17 +2,19 @@ title = "LibNetwork Configuration file"
 
 [daemon]
   debug = true
-  labels = ["com.docker.network.driver.overlay.hostmode=true"]
+  labels = ["com.docker.network.driver.overlay.hostmode=true", "com.docker.network.driver.overlay.vxlan-port=4790", "com.docker.network.driver.overlay.gossip-port=8000"]
+  defaultGwNetwork = "dnet-gwbridge"
+  DataDir = "/var/run/libnetwork"
 [cluster]
-  discovery = "etcd://localhost:2379"
-  Address = "192.168.56.101"
+  discovery = "etcd://192.168.56.101:3379"
+  address = "192.168.56.101"
 [datastore]
   embedded = false
 [datastore.client]
   provider = "etcd"
-  Address = "localhost:2379"
+  Address = "192.168.56.101:3379"
 [scopes]
   [scopes.global]
       [scopes.global.client]
             provider = "etcd"
-                  address = "localhost:2379"
+                  address = "192.168.56.101:3379"

controller.go

@@ -191,6 +191,7 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 		return nil, err
 	}
 
+	c.initDefaultGwNetwork()
 	drvRegistry, err := drvregistry.New(c.getStore(datastore.LocalScope), c.getStore(datastore.GlobalScope), c.RegisterDriver, nil, c.cfg.PluginGetter)
 	if err != nil {
 		return nil, err

default_gateway.go

@@ -29,6 +29,12 @@ var procGwNetwork = make(chan (bool), 1)
    - its deleted when an endpoint with GW joins the container
 */
 
+func (c *controller) initDefaultGwNetwork() {
+	if c.cfg.Daemon.DefaultGwNetwork == "" {
+		c.cfg.Daemon.DefaultGwNetwork = libnGWNetwork
+	}
+}
+
 func (sb *sandbox) setupDefaultGW() error {
 
 	// check if the container already has a GW endpoint
@@ -40,7 +46,7 @@ func (sb *sandbox) setupDefaultGW() error {
 
 	// Look for default gw network. In case of error (includes not found),
 	// retry and create it if needed in a serialized execution.
-	n, err := c.NetworkByName(libnGWNetwork)
+	n, err := c.NetworkByName(sb.controller.DefaultGwNetworkName())
 	if err != nil {
 		if n, err = c.defaultGwNetwork(); err != nil {
 			return err
@@ -146,15 +152,15 @@ func (sb *sandbox) needDefaultGW() bool {
 
 func (sb *sandbox) getEndpointInGWNetwork() *endpoint {
 	for _, ep := range sb.getConnectedEndpoints() {
-		if ep.getNetwork().name == libnGWNetwork && strings.HasPrefix(ep.Name(), "gateway_") {
+		if ep.getNetwork().name == sb.controller.DefaultGwNetworkName() && strings.HasPrefix(ep.Name(), "gateway_") {
 			return ep
 		}
 	}
 	return nil
 }
 
 func (ep *endpoint) endpointInGWNetwork() bool {
-	if ep.getNetwork().name == libnGWNetwork && strings.HasPrefix(ep.Name(), "gateway_") {
+	if ep.getNetwork().name == ep.getNetwork().ctrlr.DefaultGwNetworkName() && strings.HasPrefix(ep.Name(), "gateway_") {
 		return true
 	}
 	return false
@@ -178,7 +184,7 @@ func (c *controller) defaultGwNetwork() (Network, error) {
 	procGwNetwork <- true
 	defer func() { <-procGwNetwork }()
 
-	n, err := c.NetworkByName(libnGWNetwork)
+	n, err := c.NetworkByName(c.DefaultGwNetworkName())
 	if err != nil {
 		if _, ok := err.(types.NotFoundError); ok {
 			n, err = c.createGWNetwork()
@@ -187,6 +193,10 @@ func (c *controller) defaultGwNetwork() (Network, error) {
 	return n, err
 }
 
+func (c *controller) DefaultGwNetworkName() string {
+	return c.cfg.Daemon.DefaultGwNetwork
+}
+
 // Returns the endpoint which is providing external connectivity to the sandbox
 func (sb *sandbox) getGatewayEndpoint() *endpoint {
 	for _, ep := range sb.getConnectedEndpoints() {

default_gateway_linux.go

@@ -15,12 +15,12 @@ func getPlatformOption() EndpointOption {
 
 func (c *controller) createGWNetwork() (Network, error) {
 	netOption := map[string]string{
-		bridge.BridgeName:         libnGWNetwork,
+		bridge.BridgeName:         c.DefaultGwNetworkName(),
 		bridge.EnableICC:          strconv.FormatBool(false),
 		bridge.EnableIPMasquerade: strconv.FormatBool(true),
 	}
 
-	n, err := c.NewNetwork("bridge", libnGWNetwork, "",
+	n, err := c.NewNetwork("bridge", c.DefaultGwNetworkName(), "",
 		NetworkOptionDriverOpts(netOption),
 		NetworkOptionEnableIPv6(false),
 	)

default_gateway_solaris.go

@@ -15,12 +15,12 @@ func getPlatformOption() EndpointOption {
 
 func (c *controller) createGWNetwork() (Network, error) {
 	netOption := map[string]string{
-		bridge.BridgeName:         libnGWNetwork,
+		bridge.BridgeName:         c.DefaultGwNetworkName(),
 		bridge.EnableICC:          strconv.FormatBool(false),
 		bridge.EnableIPMasquerade: strconv.FormatBool(true),
 	}
 
-	n, err := c.NewNetwork("bridge", libnGWNetwork, "",
+	n, err := c.NewNetwork("bridge", c.DefaultGwNetworkName(), "",
 		NetworkOptionDriverOpts(netOption),
 		NetworkOptionEnableIPv6(false),
 	)

drivers/overlay/encryption.go

@@ -113,13 +113,13 @@ func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal
 
 	if add {
 		for _, rIP := range nodes {
-			if err := setupEncryption(lIP, aIP, rIP, vxlanID, d.secMap, d.keys); err != nil {
+			if err := d.setupEncryption(lIP, aIP, rIP, vxlanID, d.secMap, d.keys); err != nil {
 				logrus.Warnf("Failed to program network encryption between %s and %s: %v", lIP, rIP, err)
 			}
 		}
 	} else {
 		if len(nodes) == 0 {
-			if err := removeEncryption(lIP, rIP, d.secMap); err != nil {
+			if err := d.removeEncryption(lIP, rIP, d.secMap); err != nil {
 				logrus.Warnf("Failed to remove network encryption between %s and %s: %v", lIP, rIP, err)
 			}
 		}
@@ -128,18 +128,18 @@ func (d *driver) checkEncryption(nid string, rIP net.IP, vxlanID uint32, isLocal
 	return nil
 }
 
-func setupEncryption(localIP, advIP, remoteIP net.IP, vni uint32, em *encrMap, keys []*key) error {
+func (d *driver) setupEncryption(localIP, advIP, remoteIP net.IP, vni uint32, em *encrMap, keys []*key) error {
 	logrus.Debugf("Programming encryption for vxlan %d between %s and %s", vni, localIP, remoteIP)
 	rIPs := remoteIP.String()
 
 	indices := make([]*spi, 0, len(keys))
 
-	err := programMangle(vni, true)
+	err := programMangle(vni, d.vxlanPort, true)
 	if err != nil {
 		logrus.Warn(err)
 	}
 
-	err = programInput(vni, true)
+	err = programInput(vni, d.vxlanPort, true)
 	if err != nil {
 		logrus.Warn(err)
 	}
@@ -171,7 +171,7 @@ func setupEncryption(localIP, advIP, remoteIP net.IP, vni uint32, em *encrMap, k
 	return nil
 }
 
-func removeEncryption(localIP, remoteIP net.IP, em *encrMap) error {
+func (d *driver) removeEncryption(localIP, remoteIP net.IP, em *encrMap) error {
 	em.Lock()
 	indices, ok := em.nodes[remoteIP.String()]
 	em.Unlock()
@@ -198,7 +198,7 @@ func removeEncryption(localIP, remoteIP net.IP, em *encrMap) error {
 	return nil
 }
 
-func programMangle(vni uint32, add bool) (err error) {
+func programMangle(vni uint32, vxlanPort int, add bool) (err error) {
 	var (
 		p      = strconv.FormatUint(uint64(vxlanPort), 10)
 		c      = fmt.Sprintf("0>>22&0x3C@12&0xFFFFFF00=%d", int(vni)<<8)
@@ -225,7 +225,7 @@ func programMangle(vni uint32, add bool) (err error) {
 	return
 }
 
-func programInput(vni uint32, add bool) (err error) {
+func programInput(vni uint32, vxlanPort int, add bool) (err error) {
 	var (
 		port       = strconv.FormatUint(uint64(vxlanPort), 10)
 		vniMatch   = fmt.Sprintf("0>>22&0x3C@12&0xFFFFFF00=%d", int(vni)<<8)

drivers/overlay/joinleave.go

@@ -44,7 +44,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 		return fmt.Errorf("could not find subnet for endpoint %s", eid)
 	}
 
-	if s.gwIP == nil {
+	if n.hostAccess && s.gwIP == nil {
 		gwIP, err := jinfo.RequestAddress(s.subnetIP)
 		if err != nil {
 			logrus.Errorf("RequestAddress failed %s %v", s.subnetIP.String(), err)

drivers/overlay/ov_network.go

@@ -214,8 +214,8 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d
 	// Make sure no rule is on the way from any stale secure network
 	if !n.secure {
 		for _, vni := range vnis {
-			programMangle(vni, false)
-			programInput(vni, false)
+			programMangle(vni, d.vxlanPort, false)
+			programInput(vni, d.vxlanPort, false)
 		}
 	}
 
@@ -266,8 +266,8 @@ func (d *driver) DeleteNetwork(nid string) error {
 
 	if n.secure {
 		for _, vni := range vnis {
-			programMangle(vni, false)
-			programInput(vni, false)
+			programMangle(vni, d.vxlanPort, false)
+			programInput(vni, d.vxlanPort, false)
 		}
 	}
 
@@ -418,13 +418,13 @@ func populateVNITbl() {
 		})
 }
 
-func networkOnceInit() {
+func (d *driver) networkOnceInit() {
 	populateVNITbl()
 	if hostMode {
 		return
 	}
 
-	err := createVxlan("testvxlan", 1, 0)
+	err := createVxlan("testvxlan", 1, d.vxlanPort, 0)
 	if err != nil {
 		logrus.Errorf("Failed to create testvxlan interface: %v", err)
 		return
@@ -565,7 +565,7 @@ func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error
 		return fmt.Errorf("bridge creation in sandbox failed for subnet %q: %v", s.subnetIP.String(), err)
 	}
 
-	err := createVxlan(vxlanName, n.vxlanID(s), n.maxMTU())
+	err := createVxlan(vxlanName, n.vxlanID(s), n.driver.vxlanPort, n.maxMTU())
 	if err != nil {
 		return err
 	}
@@ -665,7 +665,7 @@ func (n *network) initSandbox(restore bool) error {
 	n.initEpoch++
 	n.Unlock()
 
-	networkOnce.Do(networkOnceInit)
+	networkOnce.Do(n.driver.networkOnceInit)
 
 	if !restore {
 		if hostMode {