Create parent znodes as part of watching or creating ephemeral nodes

The trick here is doing it in a way that's safe to the node being
deleted mid-process. To make that safe, we wrap the Create and
ChildrenW calls in a retry loop.

Author: colinmarc

partitions.go

@@ -48,11 +48,6 @@ func watchPartitions(zkWatcher *zkWatcher, peers *peers, db, version string, num
 		noneMissing:   make(chan bool),
 	}
 
-	// Create the partitions path we're going to watch, in case no one has done
-	// that yet.
-	p.zkPath = path.Join("partitions", db, version)
-	zkWatcher.createPath(p.zkPath)
-
 	updates, _ := zkWatcher.watchChildren(p.zkPath)
 	p.updateRemotePartitions(<-updates)
 	go p.sync(updates)

peers.go

@@ -42,7 +42,6 @@ func watchPeers(zkWatcher *zkWatcher, shardID, address string) *peers {
 		resetConvergenceTimer: make(chan bool),
 	}
 
-	zkWatcher.createPath("nodes")
 	node := path.Join("nodes", fmt.Sprintf("%s@%s", p.shardID, p.address))
 	zkWatcher.createEphemeral(node)
 

zk_watcher.go

@@ -16,6 +16,7 @@ const (
 	coordinationVersion = "v1"
 	zkReconnectPeriod   = 1 * time.Second
 	defaultZKPort       = 2181
+	maxCreateRetries    = 5
 )
 
 var defaultZkACL = zk.WorldACL(zk.PERM_ALL)
@@ -234,9 +235,22 @@ func (w *zkWatcher) hookCreateEphemeral(node string) error {
 	w.RLock()
 	defer w.RUnlock()
 
-	_, err := w.conn.Create(node, "", zk.EPHEMERAL, defaultZkACL)
-	if err != nil {
-		return err
+	// Retry a few times, in case the node is removed in between the two following
+	// steps.
+	for i := 0; i < maxCreateRetries; i++ {
+		_, err := w.conn.Create(node, "", zk.EPHEMERAL, defaultZkACL)
+		if err == nil {
+			break
+		} else if err != nil && !isNoNode(err) {
+			return err
+		}
+
+		// Create the parent nodes.
+		parent, _ := path.Split(node)
+		err = w.createAll(parent)
+		if err != nil {
+			return fmt.Errorf("create %s: %s", node, err)
+		}
 	}
 
 	return nil
@@ -279,7 +293,7 @@ func (w *zkWatcher) hookWatchChildren(node string, wn watchedNode) error {
 	w.RLock()
 	defer w.RUnlock()
 
-	children, _, events, err := w.conn.ChildrenW(node)
+	children, _, events, err := w.childrenW(node)
 	if err != nil {
 		return err
 	}
@@ -317,7 +331,7 @@ func (w *zkWatcher) hookWatchChildren(node string, wn watchedNode) error {
 			}
 
 			w.RLock()
-			children, _, events, err = w.conn.ChildrenW(node)
+			children, _, events, err = w.childrenW(node)
 			w.RUnlock()
 
 			if err != nil {
@@ -331,30 +345,36 @@ func (w *zkWatcher) hookWatchChildren(node string, wn watchedNode) error {
 	return nil
 }
 
-// createPath creates a node and all its parents permanently.
-func (w *zkWatcher) createPath(node string) error {
-	w.RLock()
-	defer w.RUnlock()
+func (w *zkWatcher) childrenW(node string) (children []string, stat *zk.Stat, events <-chan zk.Event, err error) {
+	// Retry a few times, in case the node is removed in between the two following
+	// steps.
+	for i := 0; i < maxCreateRetries; i++ {
+		children, stat, events, err = w.conn.ChildrenW(node)
+		if !isNoNode(err) {
+			return
+		}
 
-	node = path.Join(w.prefix, node)
-	err := w.createAll(node)
-	if err != nil {
-		return fmt.Errorf("create %s: %s", node, err)
-	} else {
-		return err
+		// Create the node so we can watch it.
+		err = w.createAll(node)
+		if err != nil {
+			err = fmt.Errorf("create %s: %s", node, err)
+			return
+		}
 	}
+
+	return
 }
 
-func (w *zkWatcher) createAll(fullNode string) error {
-	base, _ := path.Split(path.Clean(fullNode))
+func (w *zkWatcher) createAll(node string) error {
+	base, _ := path.Split(path.Clean(node))
 	if base != "" && base != "/" {
 		err := w.createAll(base)
 		if err != nil {
 			return err
 		}
 	}
 
-	_, err := w.conn.Create(path.Clean(fullNode), "", 0, defaultZkACL)
+	_, err := w.conn.Create(path.Clean(node), "", 0, defaultZkACL)
 	if err != nil && !isNodeExists(err) {
 		return err
 	}
@@ -387,3 +407,11 @@ func isNodeExists(err error) bool {
 
 	return false
 }
+
+func isNoNode(err error) bool {
+	if zkErr, ok := err.(*zk.Error); ok && zkErr.Code == zk.ZNONODE {
+		return true
+	}
+
+	return false
+}

zk_watcher_test.go

@@ -103,9 +103,6 @@ func TestZKWatcher(t *testing.T) {
 	defer w.close()
 	defer tzk.close()
 
-	err := w.createPath("/foo")
-	require.NoError(t, err, "createPath should work")
-
 	updates, _ := w.watchChildren("/foo")
 	go func() {
 		w.createEphemeral("/foo/bar")
@@ -123,9 +120,6 @@ func TestZKWatcherReconnect(t *testing.T) {
 	defer w.close()
 	defer tzk.close()
 
-	err := w.createPath("/foo")
-	require.NoError(t, err, "createPath should work")
-
 	updates, _ := w.watchChildren("/foo")
 	go func() {
 		w.createEphemeral("/foo/bar")
@@ -144,9 +138,6 @@ func TestZKWatchesCanceled(t *testing.T) {
 	defer w.close()
 	defer tzk.close()
 
-	err := w.createPath("/foo")
-	require.NoError(t, err, "createPath should work")
-
 	w.watchChildren("/foo")
 
 	for i := 0; i < 3; i++ {
@@ -161,9 +152,6 @@ func TestZKRemoveWatch(t *testing.T) {
 	defer w.close()
 	defer tzk.close()
 
-	err := w.createPath("/foo")
-	require.NoError(t, err, "createPath should work")
-
 	updates, disconnected := w.watchChildren("/foo")
 
 	w.createEphemeral("/foo/bar")