From be543afbd862973e69612393bdedb55c777a4297 Mon Sep 17 00:00:00 2001
From: Jim Garlick <garlick.jim@gmail.com>
Date: Fri, 23 Jul 2021 08:13:20 -0700
Subject: [PATCH 1/3] overlay/attr: defend against invalid arguments

Problem: attribute code does not check 'attr' argument
for NULL before using.

Fail with EINVAL if attr argument is NULL.
---
 src/broker/attr.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/broker/attr.c b/src/broker/attr.c
index 93dd082d25ef..ca2dde1ba929 100644
--- a/src/broker/attr.c
+++ b/src/broker/attr.c
@@ -89,7 +89,7 @@ int attr_add (attr_t *attrs, const char *name, const char *val, int flags)
 {
     struct entry *e;
 
-    if (name == NULL || (flags & FLUX_ATTRFLAG_ACTIVE)) {
+    if (attrs == NULL || name == NULL || (flags & FLUX_ATTRFLAG_ACTIVE)) {
         errno = EINVAL;
         return -1;
     }
@@ -109,6 +109,10 @@ int attr_add_active (attr_t *attrs, const char *name, int flags,
     struct entry *e;
     int rc = -1;
 
+    if (!attrs) {
+        errno = EINVAL;
+        goto done;
+    }
     if ((e = zhash_lookup (attrs->hash, name))) {
         if (!set) {
             errno = EEXIST;
@@ -134,6 +138,10 @@ int attr_get (attr_t *attrs, const char *name, const char **val, int *flags)
     struct entry *e;
     int rc = -1;
 
+    if (!attrs || !name) {
+        errno = EINVAL;
+        goto done;
+    }
     if (!(e = zhash_lookup (attrs->hash, name))) {
         errno = ENOENT;
         goto done;

From ddf3e352d3ead8f4055f3ced36104d1a7e822c70 Mon Sep 17 00:00:00 2001
From: Jim Garlick <garlick.jim@gmail.com>
Date: Fri, 23 Jul 2021 08:15:04 -0700
Subject: [PATCH 2/3] broker: rework fanout options

Problem: there is an immutable tbon.arity attribute and
a -k broker option.  Since many attributes are settable
on the broker command line with -S, it is confusing to have
an attribute that can only be set with an option.  Also,
tbon.arity is a strange name for the TBON fanout.

Drop -k,--k-ary=N broker option.
Rename tbon.arity to tbon.fanout.
Allow tbon.fanout to be set on the command line.

Update users, including tests, the systemd unit file,
and flux-sharness.sh.

Fixes #3795
---
 etc/flux.service.in           |   2 +-
 src/bindings/lua/flux-lua.c   |  16 +++---
 src/broker/boot_config.c      |  11 ++--
 src/broker/boot_config.h      |   2 +-
 src/broker/boot_pmi.c         |  14 ++---
 src/broker/boot_pmi.h         |   2 +-
 src/broker/broker.c           |  16 +++---
 src/broker/broker.h           |   1 -
 src/broker/overlay.c          | 102 ++++++++++++++++++++++++----------
 src/broker/overlay.h          |  15 ++---
 src/broker/test/overlay.c     |  55 +++++++++++-------
 src/connectors/loop/loop.c    |   4 +-
 t/lua/t0002-rpc.t             |   2 +-
 t/sharness.d/flux-sharness.sh |   2 +-
 t/t0001-basic.t               |  15 ++---
 15 files changed, 160 insertions(+), 99 deletions(-)

diff --git a/etc/flux.service.in b/etc/flux.service.in
index 79a71040f64a..5774c6b853af 100644
--- a/etc/flux.service.in
+++ b/etc/flux.service.in
@@ -7,7 +7,7 @@ TimeoutStopSec=90
 KillMode=mixed
 ExecStart=@X_BINDIR@/flux broker \
   --config-path=@X_SYSCONFDIR@/flux/system/conf.d \
-  --k-ary=256 \
+  -Stbon.fanout=256 \
   -Srundir=@X_RUNSTATEDIR@/flux \
   -Slocal-uri=local://@X_RUNSTATEDIR@/flux/local \
   -Slog-stderr-level=6 \
diff --git a/src/bindings/lua/flux-lua.c b/src/bindings/lua/flux-lua.c
index d082112b52fa..1bd47f15430f 100644
--- a/src/bindings/lua/flux-lua.c
+++ b/src/bindings/lua/flux-lua.c
@@ -235,16 +235,16 @@ static int l_flux_size (lua_State *L)
     return (l_pushresult (L, size));
 }
 
-static int l_flux_arity (lua_State *L)
+static int l_flux_fanout (lua_State *L)
 {
     flux_t *f = lua_get_flux (L, 1);
     const char *s;
-    int arity;
+    int fanout;
 
-    if (!(s = flux_attr_get (f, "tbon.arity")))
-        return lua_pusherror (L, "flux_attr_get tbon.arity error");
-    arity = strtoul (s, NULL, 10);
-    return (l_pushresult (L, arity));
+    if (!(s = flux_attr_get (f, "tbon.fanout")))
+        return lua_pusherror (L, "flux_attr_get tbon.fanout error");
+    fanout = strtoul (s, NULL, 10);
+    return (l_pushresult (L, fanout));
 }
 
 static int l_flux_index (lua_State *L)
@@ -258,8 +258,8 @@ static int l_flux_index (lua_State *L)
         return l_flux_size (L);
     if (strcmp (key, "rank") == 0)
         return l_flux_rank (L);
-    if (strcmp (key, "arity") == 0)
-        return l_flux_arity (L);
+    if (strcmp (key, "fanout") == 0)
+        return l_flux_fanout (L);
 
     lua_getmetatable (L, 1);
     lua_getfield (L, -1, key);
diff --git a/src/broker/boot_config.c b/src/broker/boot_config.c
index b000a6395560..d2c9ee22ac35 100644
--- a/src/broker/boot_config.c
+++ b/src/broker/boot_config.c
@@ -439,11 +439,12 @@ int boot_config_geturibyrank (json_t *hosts,
     return 0;
 }
 
-int boot_config (flux_t *h, struct overlay *overlay, attr_t *attrs, int tbon_k)
+int boot_config (flux_t *h, struct overlay *overlay, attr_t *attrs)
 {
     struct boot_conf conf;
     uint32_t rank;
     uint32_t size;
+    int fanout = overlay_get_fanout (overlay);
     json_t *hosts = NULL;
 
     /* Ingest the [bootstrap] stanza.
@@ -476,10 +477,10 @@ int boot_config (flux_t *h, struct overlay *overlay, attr_t *attrs, int tbon_k)
         rank = 0;
     }
 
-    /* Tell overlay network this broker's rank, size, and branching factor.
+    /* Tell overlay network this broker's rank and size.
      * If a curve certificate was provided, load it.
      */
-    if (overlay_set_geometry (overlay, size, rank, tbon_k) < 0)
+    if (overlay_set_geometry (overlay, size, rank) < 0)
         goto error;
     if (conf.curve_cert) {
         if (overlay_cert_load (overlay, conf.curve_cert) < 0)
@@ -491,7 +492,7 @@ int boot_config (flux_t *h, struct overlay *overlay, attr_t *attrs, int tbon_k)
      * attribute to the URI peers will connect to.  If broker has no
      * downstream peers, set tbon.endpoint to NULL.
      */
-    if (kary_childof (tbon_k, size, rank, 0) != KARY_NONE) {
+    if (kary_childof (fanout, size, rank, 0) != KARY_NONE) {
         char bind_uri[MAX_URI + 1];
         char my_uri[MAX_URI + 1];
 
@@ -543,7 +544,7 @@ int boot_config (flux_t *h, struct overlay *overlay, attr_t *attrs, int tbon_k)
         char parent_uri[MAX_URI + 1];
         if (boot_config_geturibyrank (hosts,
                                       &conf,
-                                      kary_parentof (tbon_k, rank),
+                                      kary_parentof (fanout, rank),
                                       parent_uri,
                                       sizeof (parent_uri)) < 0)
             goto error;
diff --git a/src/broker/boot_config.h b/src/broker/boot_config.h
index 3f4cbfdc903a..0cbb17b82215 100644
--- a/src/broker/boot_config.h
+++ b/src/broker/boot_config.h
@@ -20,7 +20,7 @@
  *   tbon.endpoint (w)
  *   instance-level (w)
  */
-int boot_config (flux_t *h, struct overlay *overlay, attr_t *attrs, int tbon_k);
+int boot_config (flux_t *h, struct overlay *overlay, attr_t *attrs);
 
 /* The following is exported for unit testing.
  */
diff --git a/src/broker/boot_pmi.c b/src/broker/boot_pmi.c
index a330a266444f..aee6b36b3d82 100644
--- a/src/broker/boot_pmi.c
+++ b/src/broker/boot_pmi.c
@@ -148,8 +148,9 @@ static int format_bind_uri (char *buf, int bufsz, attr_t *attrs, int rank)
     return -1;
 }
 
-int boot_pmi (struct overlay *overlay, attr_t *attrs, int tbon_k)
+int boot_pmi (struct overlay *overlay, attr_t *attrs)
 {
+    int fanout = overlay_get_fanout (overlay);
     int rank;
     char key[64];
     char val[1024];
@@ -184,8 +185,7 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs, int tbon_k)
     }
     if (overlay_set_geometry (overlay,
                               pmi_params.size,
-                              pmi_params.rank,
-                              tbon_k) < 0)
+                              pmi_params.rank) < 0)
         goto error;
 
     /* A size=1 instance has no peers, so skip the PMI exchange.
@@ -198,7 +198,7 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs, int tbon_k)
      * N.B. there are no downstream peers if the 0th child of this rank
      * in k-ary tree does not exist.
      */
-    if (kary_childof (tbon_k,
+    if (kary_childof (fanout,
                       pmi_params.size,
                       pmi_params.rank,
                       0) != KARY_NONE) {
@@ -259,7 +259,7 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs, int tbon_k)
     if (pmi_params.rank > 0) {
         char *cp;
 
-        rank = kary_parentof (tbon_k, pmi_params.rank);
+        rank = kary_parentof (fanout, pmi_params.rank);
         if (snprintf (key, sizeof (key), "%d", rank) >= sizeof (key)) {
             log_msg ("pmi key string overflow");
             goto error;
@@ -288,10 +288,10 @@ int boot_pmi (struct overlay *overlay, attr_t *attrs, int tbon_k)
 
     /* Fetch the business card of children and inform overlay of public keys.
      */
-    for (i = 0; i < tbon_k; i++) {
+    for (i = 0; i < fanout; i++) {
         char *cp;
 
-        rank = kary_childof (tbon_k, pmi_params.size, pmi_params.rank, i);
+        rank = kary_childof (fanout, pmi_params.size, pmi_params.rank, i);
         if (rank == KARY_NONE)
             break;
         if (snprintf (key, sizeof (key), "%d", rank) >= sizeof (key)) {
diff --git a/src/broker/boot_pmi.h b/src/broker/boot_pmi.h
index 427eb67ecc91..17aec53a1c8b 100644
--- a/src/broker/boot_pmi.h
+++ b/src/broker/boot_pmi.h
@@ -16,7 +16,7 @@
 #include "attr.h"
 #include "overlay.h"
 
-int boot_pmi (struct overlay *overlay, attr_t *attrs, int tbon_k);
+int boot_pmi (struct overlay *overlay, attr_t *attrs);
 
 #endif /* BROKER_BOOT_PMI_H */
 
diff --git a/src/broker/broker.c b/src/broker/broker.c
index e3176d3d5f0e..426e22b9730a 100644
--- a/src/broker/broker.c
+++ b/src/broker/broker.c
@@ -106,8 +106,6 @@ static const struct flux_handle_ops broker_handle_ops;
 static struct optparse_option opts[] = {
     { .name = "verbose",    .key = 'v', .has_arg = 2, .arginfo = "[LEVEL]",
       .usage = "Be annoyingly informative by degrees", },
-    { .name = "k-ary",      .key = 'k', .has_arg = 1, .arginfo = "K",
-      .usage = "Wire up in a k-ary tree (default: 2)", },
     { .name = "setattr",    .key = 'S', .has_arg = 1, .arginfo = "ATTR=VAL",
       .usage = "Set broker attribute", },
     { .name = "config-path",.key = 'c', .has_arg = 1, .arginfo = "PATH",
@@ -128,9 +126,6 @@ void parse_command_line_arguments (int argc, char *argv[], broker_ctx_t *ctx)
 
     ctx->verbose = optparse_get_int (ctx->opts, "verbose", 0);
 
-    if ((ctx->tbon_k = optparse_get_int (ctx->opts, "k-ary", 2)) < 1)
-        log_msg_exit ("--k-ary value must be >= 1");
-
     optparse_get_str (ctx->opts, "config-path", NULL);
 
     while ((arg = optparse_getopt_next (ctx->opts, "setattr"))) {
@@ -275,7 +270,10 @@ int main (int argc, char *argv[])
         goto cleanup;
     }
 
-    if (!(ctx.overlay = overlay_create (ctx.h, overlay_recv_cb, &ctx))) {
+    if (!(ctx.overlay = overlay_create (ctx.h,
+                                        ctx.attrs,
+                                        overlay_recv_cb,
+                                        &ctx))) {
         log_err ("overlay_create");
         goto cleanup;
     }
@@ -299,13 +297,13 @@ int main (int argc, char *argv[])
      */
     monotime (&boot_start_time);
     if (flux_conf_unpack (conf, NULL, "{s:{}}", "bootstrap") == 0) {
-        if (boot_config (ctx.h, ctx.overlay, ctx.attrs, ctx.tbon_k) < 0) {
+        if (boot_config (ctx.h, ctx.overlay, ctx.attrs) < 0) {
             log_msg ("bootstrap failed");
             goto cleanup;
         }
     }
     else { // PMI
-        if (boot_pmi (ctx.overlay, ctx.attrs, ctx.tbon_k) < 0) {
+        if (boot_pmi (ctx.overlay, ctx.attrs) < 0) {
             log_msg ("bootstrap failed");
             goto cleanup;
         }
@@ -318,7 +316,7 @@ int main (int argc, char *argv[])
     assert (ctx.size > 0);
 
     /* Must be called after overlay setup */
-    if (overlay_register_attrs (ctx.overlay, ctx.attrs) < 0) {
+    if (overlay_register_attrs (ctx.overlay) < 0) {
         log_err ("registering overlay attributes");
         goto cleanup;
     }
diff --git a/src/broker/broker.h b/src/broker/broker.h
index 4f13bdd12fce..ec3a13ba3d55 100644
--- a/src/broker/broker.h
+++ b/src/broker/broker.h
@@ -38,7 +38,6 @@ struct broker {
     zlist_t *subscriptions;     /* subscripts for internal services */
     struct content_cache *cache;
     struct publisher *publisher;
-    int tbon_k;
 
     struct runat *runat;
     struct state_machine *state_machine;
diff --git a/src/broker/overlay.c b/src/broker/overlay.c
index e1d47cb4dff7..5e651d36e779 100644
--- a/src/broker/overlay.c
+++ b/src/broker/overlay.c
@@ -36,6 +36,8 @@
 #define FLUX_ZAP_DOMAIN "flux"
 #define ZAP_ENDPOINT "inproc://zeromq.zap.01"
 
+#define DEFAULT_FANOUT 2
+
 enum {
     KEEPALIVE_STATUS_NORMAL = 0,
     KEEPALIVE_STATUS_DISCONNECT = 1,
@@ -78,13 +80,14 @@ struct overlay {
     flux_watcher_t *zap_w;
 
     flux_t *h;
+    attr_t *attrs;
     flux_reactor_t *reactor;
     flux_msg_handler_t **handlers;
     flux_future_t *f_sync;
 
     uint32_t size;
     uint32_t rank;
-    int tbon_k;
+    int fanout;
     char uuid[UUID_STR_LEN];
     int version;
 
@@ -153,15 +156,11 @@ static int child_count (uint32_t rank, uint32_t size, int k)
     return count;
 }
 
-int overlay_set_geometry (struct overlay *ov,
-                          uint32_t size,
-                          uint32_t rank,
-                          int tbon_k)
+int overlay_set_geometry (struct overlay *ov, uint32_t size, uint32_t rank)
 {
     ov->size = size;
     ov->rank = rank;
-    ov->tbon_k = tbon_k;
-    ov->child_count = child_count (rank, size, tbon_k);
+    ov->child_count = child_count (rank, size, ov->fanout);
     if (ov->child_count > 0) {
         int i;
 
@@ -173,16 +172,21 @@ int overlay_set_geometry (struct overlay *ov,
         zhashx_set_key_destructor (ov->child_hash, NULL);
         for (i = 0; i < ov->child_count; i++) {
             struct child *child = &ov->children[i];
-            child->rank = kary_childof (tbon_k, size, rank, i);
+            child->rank = kary_childof (ov->fanout, size, rank, i);
         }
     }
     if (rank > 0) {
-        ov->parent.rank = kary_parentof (tbon_k, rank);
+        ov->parent.rank = kary_parentof (ov->fanout, rank);
     }
 
     return 0;
 }
 
+int overlay_get_fanout (struct overlay *ov)
+{
+    return ov->fanout;
+}
+
 uint32_t overlay_get_rank (struct overlay *ov)
 {
     return ov->rank;
@@ -284,7 +288,7 @@ static struct child *child_lookup_byrank (struct overlay *ov, uint32_t rank)
     uint32_t first;
     int i;
 
-    if ((first = kary_childof (ov->tbon_k, ov->size, ov->rank, 0)) == KARY_NONE
+    if ((first = kary_childof (ov->fanout, ov->size, ov->rank, 0)) == KARY_NONE
         || (i = rank - first) < 0
         || i >= ov->child_count)
         return NULL;
@@ -297,7 +301,7 @@ static struct child *child_lookup_route (struct overlay *ov, uint32_t rank)
 {
     uint32_t child_rank;
 
-    child_rank = kary_child_route (ov->tbon_k, ov->size, ov->rank, rank);
+    child_rank = kary_child_route (ov->fanout, ov->size, ov->rank, rank);
     if (child_rank == KARY_NONE)
         return NULL;
     return child_lookup_byrank (ov, child_rank);
@@ -1012,34 +1016,39 @@ static int overlay_attr_get_cb (const char *name, const char **val, void *arg)
     return rc;
 }
 
-int overlay_register_attrs (struct overlay *overlay, attr_t *attrs)
+int overlay_register_attrs (struct overlay *overlay)
 {
-    int tbon_level = kary_levelof (overlay->tbon_k, overlay->rank);
-    int tbon_maxlevel = kary_levelof (overlay->tbon_k, overlay->size - 1);
-    int tbon_descendants = kary_sum_descendants (overlay->tbon_k,
-                                                 overlay->size,
-                                                 overlay->rank);
-
-    if (attr_add_active (attrs, "tbon.parent-endpoint",
+    if (attr_add_active (overlay->attrs,
+                         "tbon.parent-endpoint",
                          FLUX_ATTRFLAG_READONLY,
-                         overlay_attr_get_cb, NULL, overlay) < 0)
+                         overlay_attr_get_cb,
+                         NULL,
+                         overlay) < 0)
         return -1;
-    if (attr_add_uint32 (attrs, "rank", overlay->rank,
+    if (attr_add_uint32 (overlay->attrs,
+                         "rank",
+                         overlay->rank,
                          FLUX_ATTRFLAG_IMMUTABLE) < 0)
         return -1;
-    if (attr_add_uint32 (attrs, "size", overlay->size,
+    if (attr_add_uint32 (overlay->attrs,
+                         "size", overlay->size,
                          FLUX_ATTRFLAG_IMMUTABLE) < 0)
         return -1;
-    if (attr_add_int (attrs, "tbon.arity", overlay->tbon_k,
+    if (attr_add_int (overlay->attrs,
+                      "tbon.level",
+                      kary_levelof (overlay->fanout, overlay->rank),
                       FLUX_ATTRFLAG_IMMUTABLE) < 0)
         return -1;
-    if (attr_add_int (attrs, "tbon.level", tbon_level,
+    if (attr_add_int (overlay->attrs,
+                      "tbon.maxlevel",
+                      kary_levelof (overlay->fanout, overlay->size - 1),
                       FLUX_ATTRFLAG_IMMUTABLE) < 0)
         return -1;
-    if (attr_add_int (attrs, "tbon.maxlevel", tbon_maxlevel,
-                      FLUX_ATTRFLAG_IMMUTABLE) < 0)
-        return -1;
-    if (attr_add_int (attrs, "tbon.descendants", tbon_descendants,
+    if (attr_add_int (overlay->attrs,
+                      "tbon.descendants",
+                      kary_sum_descendants (overlay->fanout,
+                                            overlay->size,
+                                            overlay->rank),
                       FLUX_ATTRFLAG_IMMUTABLE) < 0)
         return -1;
 
@@ -1130,6 +1139,35 @@ int overlay_authorize (struct overlay *ov, const char *name, const char *pubkey)
     return 0;
 }
 
+/* Set tbon.fanout attribute and return fanout value or -1 on error.
+ * Allow the value to be set on the broker command line, but no changes
+ * after this function is called.
+ */
+static int overlay_configure_fanout (attr_t *attrs)
+{
+    int fanout = DEFAULT_FANOUT;
+    const char *val;
+    char *endptr;
+
+    if (attr_get (attrs, "tbon.fanout", &val, NULL) == 0) {
+        errno = 0;
+        fanout = strtol (val, &endptr, 10);
+        if (errno != 0 || fanout <= 0 || *endptr != '\0') {
+            log_msg ("tbon.fanout value must be a positive integer");
+            errno = EINVAL;
+            return -1;
+        }
+        if (attr_delete (attrs, "tbon.fanout", true) < 0)
+            return -1;
+    }
+    if (attr_add_int (attrs,
+                      "tbon.fanout",
+                      fanout,
+                      FLUX_ATTRFLAG_IMMUTABLE) < 0)
+        return -1;
+    return fanout;
+}
+
 void overlay_destroy (struct overlay *ov)
 {
     if (ov) {
@@ -1173,13 +1211,17 @@ static const struct flux_msg_handler_spec htab[] = {
     FLUX_MSGHANDLER_TABLE_END,
 };
 
-struct overlay *overlay_create (flux_t *h, overlay_recv_f cb, void *arg)
+struct overlay *overlay_create (flux_t *h,
+                                attr_t *attrs,
+                                overlay_recv_f cb,
+                                void *arg)
 {
     struct overlay *ov;
     uuid_t uuid;
 
     if (!(ov = calloc (1, sizeof (*ov))))
         return NULL;
+    ov->attrs = attrs;
     ov->rank = FLUX_NODEID_ANY;
     ov->parent.lastsent = -1;
     ov->h = h;
@@ -1189,6 +1231,8 @@ struct overlay *overlay_create (flux_t *h, overlay_recv_f cb, void *arg)
     ov->version = FLUX_CORE_VERSION_HEX;
     uuid_generate (uuid);
     uuid_unparse (uuid, ov->uuid);
+    if ((ov->fanout = overlay_configure_fanout (ov->attrs)) < 0)
+        goto error;
     if (flux_msg_handler_addvec (h, htab, ov, &ov->handlers) < 0)
         goto error;
     if (!(ov->f_sync = flux_sync_create (h, sync_min))
diff --git a/src/broker/overlay.h b/src/broker/overlay.h
index 29984e7d3efb..04bd4ff06aa4 100644
--- a/src/broker/overlay.h
+++ b/src/broker/overlay.h
@@ -29,15 +29,15 @@ typedef void (*overlay_recv_f)(const flux_msg_t *msg,
 /* Create overlay network, registering 'cb' to be called with each
  * received message.
  */
-struct overlay *overlay_create (flux_t *h, overlay_recv_f cb, void *arg);
+struct overlay *overlay_create (flux_t *h,
+                                attr_t *attrs,
+                                overlay_recv_f cb,
+                                void *arg);
 void overlay_destroy (struct overlay *ov);
 
-/* Set the overlay network size, rank, and TBON branching factor.
+/* Set the overlay network size and rank of this broker.
  */
-int overlay_set_geometry (struct overlay *ov,
-                          uint32_t size,
-                          uint32_t rank,
-                          int tbon_k);
+int overlay_set_geometry (struct overlay *ov, uint32_t size, uint32_t rank);
 
 /* Send a message on the overlay network.
  * 'where' determines whether the message is routed upstream or downstream.
@@ -65,6 +65,7 @@ int overlay_set_parent_pubkey (struct overlay *ov, const char *pubkey);
 
 /* Misc. accessors
  */
+int overlay_get_fanout (struct overlay *ov);
 uint32_t overlay_get_rank (struct overlay *ov);
 void overlay_set_rank (struct overlay *ov, uint32_t rank); // test only
 uint32_t overlay_get_size (struct overlay *ov);
@@ -100,7 +101,7 @@ void overlay_set_monitor_cb (struct overlay *ov,
 
 /* Register overlay-related broker attributes.
  */
-int overlay_register_attrs (struct overlay *overlay, attr_t *attrs);
+int overlay_register_attrs (struct overlay *overlay);
 
 #endif /* !_BROKER_OVERLAY_H */
 
diff --git a/src/broker/test/overlay.c b/src/broker/test/overlay.c
index 12d18a56b756..f79fb7c7b75c 100644
--- a/src/broker/test/overlay.c
+++ b/src/broker/test/overlay.c
@@ -78,7 +78,11 @@ void ctx_destroy (struct context *ctx)
     free (ctx);
 }
 
-struct context *ctx_create (flux_t *h, const char *name, int size, int rank,
+struct context *ctx_create (flux_t *h,
+                            const char *name,
+                            int size,
+                            int rank,
+                            int fanout,
                             overlay_recv_f cb)
 {
     struct context *ctx;
@@ -88,16 +92,20 @@ struct context *ctx_create (flux_t *h, const char *name, int size, int rank,
 
     if (!(ctx = calloc (1, sizeof (*ctx))))
         BAIL_OUT ("calloc failed");
+    if (!(ctx->attrs = attr_create ()))
+        BAIL_OUT ("attr_create failed");
+    if (fanout != 2) {
+        if (attr_add_int (ctx->attrs, "tbon.fanout", fanout, 0) < 0)
+            BAIL_OUT ("could not add tbon.fanout attribute");
+    }
     ctx->h = h;
     ctx->size = size;
     ctx->rank = rank;
     snprintf (ctx->name, sizeof (ctx->name), "%s-%d", name, rank);
-    if (!(ctx->ov = overlay_create (h, cb, ctx)))
+    if (!(ctx->ov = overlay_create (h, ctx->attrs, cb, ctx)))
         BAIL_OUT ("overlay_create");
     if (!(ctx->uuid = overlay_get_uuid (ctx->ov)))
         BAIL_OUT ("overlay_get_uuid failed");
-    if (!(ctx->attrs = attr_create ()))
-        BAIL_OUT ("attr_create failed");
     diag ("created %s: rank %d size %d uuid %s",
           ctx->name, ctx->rank, ctx->size, ctx->uuid);
 
@@ -106,23 +114,23 @@ struct context *ctx_create (flux_t *h, const char *name, int size, int rank,
 
 void single (flux_t *h)
 {
-    struct context *ctx = ctx_create (h, "single", 1, 0, NULL);
+    struct context *ctx = ctx_create (h, "single", 1, 0, 2, NULL);
     flux_msg_t *msg;
 
-    ok (overlay_set_geometry (ctx->ov, 1, 0, 2) == 0,
-        "%s: overlay_set_geometry size=1 rank=0 tbon_k=2 works", ctx->name);
+    ok (overlay_set_geometry (ctx->ov, 1, 0) == 0,
+        "%s: overlay_set_geometry size=1 rank=0 works", ctx->name);
 
     ok (overlay_get_size (ctx->ov) == 1,
         "%s: overlay_get_size returns 1", ctx->name);
     ok (overlay_get_rank (ctx->ov) == 0,
         "%s: overlay_get_rank returns 0", ctx->name);
 
-    ok (overlay_register_attrs (ctx->ov, ctx->attrs) == 0,
+    ok (overlay_register_attrs (ctx->ov) == 0,
         "%s: overlay_register_attrs works", ctx->name);
     check_attr (ctx, "tbon.parent-endpoint", NULL);
     check_attr (ctx, "rank", "0");
     check_attr (ctx, "size", "1");
-    check_attr (ctx, "tbon.arity", "2");
+    check_attr (ctx, "tbon.fanout", "2");
     check_attr (ctx, "tbon.level", "0");
     check_attr (ctx, "tbon.maxlevel", "0");
     check_attr (ctx, "tbon.descendants", "0");
@@ -245,7 +253,6 @@ void trio (flux_t *h)
 {
     struct context *ctx[2];
     int size = 3;
-    int k_ary = 2;
     char parent_uri[64];
     const char *server_pubkey;
     const char *client_pubkey;
@@ -258,9 +265,9 @@ void trio (flux_t *h)
     zcert_t *cert;
     const char *sender;
 
-    ctx[0] = ctx_create (h, "trio", size, 0, recv_cb);
+    ctx[0] = ctx_create (h, "trio", size, 0, 2, recv_cb);
 
-    ok (overlay_set_geometry (ctx[0]->ov, size, 0, k_ary) == 0,
+    ok (overlay_set_geometry (ctx[0]->ov, size, 0) == 0,
         "%s: overlay_set_geometry works", ctx[0]->name);
 
     ok ((server_pubkey = overlay_cert_pubkey (ctx[0]->ov)) != NULL,
@@ -270,9 +277,9 @@ void trio (flux_t *h)
     ok (overlay_bind (ctx[0]->ov, parent_uri) == 0,
         "%s: overlay_bind %s works", ctx[0]->name, parent_uri);
 
-    ctx[1] = ctx_create (h, "trio", size, 1, recv_cb);
+    ctx[1] = ctx_create (h, "trio", size, 1, 2, recv_cb);
 
-    ok (overlay_set_geometry (ctx[1]->ov, size, 1, k_ary) == 0,
+    ok (overlay_set_geometry (ctx[1]->ov, size, 1) == 0,
         "%s: overlay_init works", ctx[1]->name);
 
     ok ((client_pubkey = overlay_cert_pubkey (ctx[1]->ov)) != NULL,
@@ -468,13 +475,13 @@ void test_create (flux_t *h,
                   int size,
                   struct context *ctx[])
 {
-    int k_ary = size - 1;
+    int fanout = size - 1;
     char uri[64] = { 0 };
     int rank;
 
     for (rank = 0; rank < size; rank++) {
-        ctx[rank] = ctx_create (h, name, size, rank, recv_cb);
-        if (overlay_set_geometry (ctx[rank]->ov, size, rank, k_ary) < 0)
+        ctx[rank] = ctx_create (h, name, size, rank, fanout, recv_cb);
+        if (overlay_set_geometry (ctx[rank]->ov, size, rank) < 0)
             BAIL_OUT ("%s: overlay_set_geometry failed", ctx[rank]->name);
         if (rank == 0)
             snprintf (uri, sizeof (uri), "ipc://@%s", ctx[0]->name);
@@ -593,12 +600,21 @@ void check_monitor (flux_t *h)
 void wrongness (flux_t *h)
 {
     struct overlay *ov;
+    attr_t *attrs;
 
+    if (!(attrs = attr_create ()))
+        BAIL_OUT ("attr_create failed");
     errno = 0;
-    ok (overlay_create (NULL, NULL, NULL) == NULL && errno == EINVAL,
+    ok (overlay_create (NULL, attrs, NULL, NULL) == NULL && errno == EINVAL,
         "overlay_create h=NULL fails with EINVAL");
+    errno = 0;
+    ok (overlay_create (h, NULL, NULL, NULL) == NULL && errno == EINVAL,
+        "overlay_create attrs=NULL fails with EINVAL");
+    attr_destroy (attrs);
 
-    if (!(ov = overlay_create (h, NULL, NULL)))
+    if (!(attrs = attr_create ()))
+        BAIL_OUT ("attr_create failed");
+    if (!(ov = overlay_create (h, attrs, NULL, NULL)))
         BAIL_OUT ("overlay_create failed");
 
     errno = 0;
@@ -606,6 +622,7 @@ void wrongness (flux_t *h)
         "overlay_bind fails if called before rank is known");
 
     overlay_destroy (ov);
+    attr_destroy (attrs);
 }
 
 void diag_logger (const char *buf, int len, void *arg)
diff --git a/src/connectors/loop/loop.c b/src/connectors/loop/loop.c
index eb2381a80836..add5cfb2e327 100644
--- a/src/connectors/loop/loop.c
+++ b/src/connectors/loop/loop.c
@@ -118,11 +118,11 @@ flux_t *connector_init (const char *path, int flags)
         goto error;
     if (!(c->h = flux_handle_create (c, &handle_ops, flags)))
         goto error;
-    /* Fake out size, rank, tbon-arity attributes for testing.
+    /* Fake out size, rank, tbon.fanout attributes for testing.
      */
     if (flux_attr_set_cacheonly(c->h, "rank", "0") < 0
                 || flux_attr_set_cacheonly (c->h, "size", "1") < 0
-                || flux_attr_set_cacheonly (c->h, "tbon-arity", "2") < 0)
+                || flux_attr_set_cacheonly (c->h, "tbon.fanout", "2") < 0)
         goto error;
     c->cred.userid = getuid ();
     c->cred.rolemask = FLUX_ROLE_OWNER;
diff --git a/t/lua/t0002-rpc.t b/t/lua/t0002-rpc.t
index ef4bce34f61e..70953c539604 100755
--- a/t/lua/t0002-rpc.t
+++ b/t/lua/t0002-rpc.t
@@ -14,7 +14,7 @@ is (err, nil, "error is nil")
 
 is (f.rank, 0, "running on rank 0")
 is (f.size, 2, "session size is 2")
-is (f.arity, 2, "session arity is 2")
+is (f.fanout, 2, "session fanout is 2")
 
 --
 --  Use 'ping' packet to test rpc
diff --git a/t/sharness.d/flux-sharness.sh b/t/sharness.d/flux-sharness.sh
index a13db46a3da6..a7457190f2c8 100644
--- a/t/sharness.d/flux-sharness.sh
+++ b/t/sharness.d/flux-sharness.sh
@@ -119,7 +119,7 @@ make_bootstrap_config() {
     echo "--test-exit-timeout=${TEST_UNDER_FLUX_EXIT_TIMEOUT:-0}"
     echo "-o,-Sbroker.quorum=${TEST_UNDER_FLUX_QUORUM:-$full}"
     echo "--test-start-mode=${TEST_UNDER_FLUX_START_MODE:-all}"
-    echo "-o,--k-ary=${TEST_UNDER_FLUX_FANOUT:-$size}"
+    echo "-o,-Stbon.fanout=${TEST_UNDER_FLUX_FANOUT:-$size}"
 }
 
 #
diff --git a/t/t0001-basic.t b/t/t0001-basic.t
index 7b7f66b14fc8..b71c11ca92ae 100755
--- a/t/t0001-basic.t
+++ b/t/t0001-basic.t
@@ -423,13 +423,14 @@ test_expect_success 'broker broker.pid attribute is immutable' '
 test_expect_success 'broker --verbose option works' '
 	flux start ${ARGS} -o,-v /bin/true
 '
-test_expect_success 'broker --k-ary=4 option works' '
-	echo 4 >arity.exp &&
-	flux start ${ARGS} -o,--k-ary=4 flux getattr tbon.arity >arity.out &&
-	test_cmp arity.exp arity.out
-'
-test_expect_success 'broker --k-ary=0 fails' '
-	test_must_fail flux start ${ARGS} -o,--k-ary=0 /bin/true
+test_expect_success 'broker -Stbon.fanout=4 option works' '
+	echo 4 >fanout.exp &&
+	flux start ${ARGS} -o,-Stbon.fanout=4 \
+		flux getattr tbon.fanout >fanout.out &&
+	test_cmp fanout.exp fanout.out
+'
+test_expect_success 'broker -Stbon.fanout=0 fails' '
+	test_must_fail flux start ${ARGS} -o,-Stbon.fanout=0 /bin/true
 '
 test_expect_success 'broker fails on unknown option' '
 	test_must_fail flux start ${ARGS} -o,--not-an-option /bin/true

From 32ec7defc0a5a3ffc8f9b103069688e09fc54020 Mon Sep 17 00:00:00 2001
From: Jim Garlick <garlick.jim@gmail.com>
Date: Fri, 23 Jul 2021 08:17:55 -0700
Subject: [PATCH 3/3] doc: track changes to fanout options

Problem: docs refer to -k option and tbon.arity attribute,
but those have been renamed.

Update flux-broker(1) and flux-broker-attributes(7).
Drop 'arity' from the dictionary.
---
 doc/man1/flux-broker.rst            | 4 ----
 doc/man7/flux-broker-attributes.rst | 2 +-
 doc/test/spell.en.pws               | 1 -
 3 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/doc/man1/flux-broker.rst b/doc/man1/flux-broker.rst
index ec3546a384b7..988e23d72ab0 100644
--- a/doc/man1/flux-broker.rst
+++ b/doc/man1/flux-broker.rst
@@ -47,10 +47,6 @@ OPTIONS
 **-S, --setattr**\ =\ *ATTR=VAL*
    Set initial value for broker attribute.
 
-**-k, --k-ary**\ =\ *N*
-   Set the branching factor of the tree based overlay
-   network (default: 2).
-
 
 RESOURCES
 =========
diff --git a/doc/man7/flux-broker-attributes.rst b/doc/man7/flux-broker-attributes.rst
index ba99e737d2f6..eb731a4ac0d1 100644
--- a/doc/man7/flux-broker-attributes.rst
+++ b/doc/man7/flux-broker-attributes.rst
@@ -40,7 +40,7 @@ content.backing-path
 TOPOLOGY ATTRIBUTES
 ===================
 
-tbon.arity
+tbon.fanout
    Branching factor of the tree based overlay network.
 
 tbon.descendants
diff --git a/doc/test/spell.en.pws b/doc/test/spell.en.pws
index d091fc778ee9..7c74490e3afc 100644
--- a/doc/test/spell.en.pws
+++ b/doc/test/spell.en.pws
@@ -260,7 +260,6 @@ tinfo
 hwloc
 epgm
 args
-arity
 attr
 ATTRFLAG
 ENOENT