Merge pull request #267 from HaiwangYu/apply-pointcloud

Some in-dev code and speed up of conversion code

aux/inc/WireCellAux/TensorDMcommon.h

@@ -10,6 +10,11 @@
 
 namespace WireCell::Aux::TensorDM {
 
+    using located_t = std::unordered_map<std::string, ITensor::pointer>;
+
+    /// Index the tensors by their datapaths.
+    located_t index_datapaths(const ITensor::vector& tens);
+
     /** Build metadata-only (array-less) tensor in the DM.
      */
     ITensor::pointer make_metadata_tensor(const std::string& datatype,
@@ -24,7 +29,8 @@ namespace WireCell::Aux::TensorDM {
      */
     ITensor::pointer top_tensor(const ITensor::vector& tens,
                                 const std::string& datatype,
-                                const std::string& datapath="");
+                                const std::string& datapath="",
+                                const located_t& located = {});
 
 
     /// Build a tensor set from set of tensors.  The tensor data model
@@ -35,10 +41,6 @@ namespace WireCell::Aux::TensorDM {
                                      const Configuration& tsetmd = Json::objectValue);
 
 
-    using located_t = std::map<std::string, ITensor::pointer>;
-
-    /// Index the tensors by their datapaths.
-    located_t index_datapaths(const ITensor::vector& tens);
 
     /// Return first of type
     ITensor::pointer first_of(const ITensor::vector& tens,

aux/inc/WireCellAux/TensorDMdataset.h

@@ -5,6 +5,7 @@
 #define WIRECELLAUX_TENSORDMDATASET
 
 #include "WireCellUtil/PointCloudDataset.h"
+#include "WireCellAux/TensorDMcommon.h"
 #include "WireCellIface/ITensorSet.h"
 
 #include <unordered_map>
@@ -44,13 +45,15 @@ namespace WireCell::Aux::TensorDM {
     /// ValueError is thrown if tensors are badly formed.
     PointCloud::Dataset as_dataset(const ITensor::vector& tensors,
                                    const std::string& datapath="",
+                                   const located_t& located = {},
                                    bool share=false);
 
     /// Convenience function calling above using the tensors from the
     /// ITensorSet.  Note the dataset level metadata is taken from the
     /// ITensor with datatype "pcdataset" and not from ITensorSet.
     PointCloud::Dataset as_dataset(const ITensorSet::pointer& its,
                                    const std::string& datapath="",
+                                   const located_t& located = {},
                                    bool share=false);
 
 

aux/inc/WireCellAux/TensorDMpointtree.h

@@ -64,7 +64,7 @@ namespace WireCell::Aux::TensorDM {
        empty in which case the first pcnamedset found is used.
 
      */
-    named_pointclouds_t as_pcnamedset(const ITensor::vector& tens, const std::string& datapath = "");
+    named_pointclouds_t as_pcnamedset(const ITensor::vector& tens, const std::string& datapath = "", const located_t& located = {});
 
 
 

aux/src/TensorDMcommon.cxx

@@ -20,19 +20,21 @@ ITensor::pointer WireCell::Aux::TensorDM::make_metadata_tensor(
 ITensor::pointer
 WireCell::Aux::TensorDM::top_tensor(const ITensor::vector& tens,
                                     const std::string& datatype,
-                                    const std::string& datapath)
+                                    const std::string& datapath,
+                                    const located_t& located)
 {
-    for (const auto& iten : tens) {
-        const auto& tenmd = iten->metadata();
-        const auto dtype = tenmd["datatype"].asString();
-        const auto dpath = tenmd["datapath"].asString();
-        if (dtype == datatype and (datapath.empty() or datapath == dpath)) {
-            return iten;
-        }
+    const auto it = located.find(datapath);
+    if (it == located.end()) {
+        raise<ValueError>("no array at datapath \"%s\"", datapath);
     }
-    raise<ValueError>("no array of datatype \"%s\" at datapath \"%s\"",
-                      datatype, datapath);
-    return nullptr;
+    const auto& iten = it->second;
+    const auto& tenmd = iten->metadata();
+    const auto dtype = tenmd["datatype"].asString();
+    if (dtype != datatype) {
+        raise<ValueError>("array at datapath \"%s\" is of datatype \"%s\" not \"%s\"",
+                          datapath, dtype, datatype);
+    }
+    return iten;
 }
 
 //     /** Return a map from datapath to tensor.

aux/src/TensorDMdataset.cxx

@@ -106,27 +106,10 @@ PointCloud::Array WireCell::Aux::TensorDM::as_array(const ITensor::pointer& ten,
 PointCloud::Dataset
 WireCell::Aux::TensorDM::as_dataset(const ITensor::vector& tens,
                                     const std::string& datapath,
+                                    const located_t& located,
                                     bool share)
 {
-    // Index the tensors by datapath and find main "pcdataset" tensor.
-    ITensor::pointer top = nullptr;
-    std::unordered_map<std::string, ITensor::pointer> located;
-    for (const auto& iten : tens) {
-        const auto& tenmd = iten->metadata();
-        const auto dtype = tenmd["datatype"].asString();
-        const auto dpath = tenmd["datapath"].asString();
-        if (!top and dtype == "pcdataset") {
-            if (datapath.empty() or datapath == dpath) {
-                top = iten;
-            }
-            continue;
-        }
-        if (dtype == "pcarray") {
-            located[dpath] = iten;
-        }
-        continue;
-    }
-
+    ITensor::pointer top = top_tensor(tens, "pcdataset", datapath, located);
     if (!top) {
         THROW(ValueError() << errmsg{"no array of datatype \"pcdataset\""});
     }
@@ -137,7 +120,7 @@ WireCell::Aux::TensorDM::as_dataset(const ITensor::vector& tens,
     auto arrrefs = topmd["arrays"];
     for (const auto& name : arrrefs.getMemberNames()) {
         const auto path = arrrefs[name].asString();
-        auto it = located.find(path);
+        const auto it = located.find(path);
         if (it == located.end()) {
             THROW(ValueError() << errmsg{"no array \"" + name + "\" at path \"" + path + "\""});
         }
@@ -152,8 +135,9 @@ WireCell::Aux::TensorDM::as_dataset(const ITensor::vector& tens,
 PointCloud::Dataset
 WireCell::Aux::TensorDM::as_dataset(const ITensorSet::pointer& tens,
                                     const std::string& datapath,
+                                    const located_t& located,
                                     bool share)
 {
     auto sv = tens->tensors();
-    return as_dataset(*(sv.get()), datapath, share);
+    return as_dataset(*(sv.get()), datapath, located, share);
 }

aux/src/TensorDMframe.cxx

@@ -308,19 +308,9 @@ IFrame::pointer WireCell::Aux::TensorDM::as_frame(const ITensor::vector& tens,
                                                   const std::string& datapath,
                                                   std::function<float(float)> transform)
 {
-    ITensor::pointer ften=nullptr;
-    // map from datapath to tensor
-    std::unordered_map<std::string, ITensor::pointer> located;
-    for (const auto& iten : tens) {
-        auto dp = iten->metadata()["datapath"].asString();
-        located[dp] = iten;
-        auto dt = iten->metadata()["datatype"].asString();
-        if (!ften and dt == "frame") {
-            if (datapath.empty() or datapath == dp) {
-                ften = iten;
-            }
-        }
-    }
+    const auto& located = index_datapaths(tens);
+    ITensor::pointer ften=top_tensor(tens, "frame", datapath, located);
+
     if (!ften) {
         THROW(ValueError() << errmsg{"no frame tensor found"});
     }
@@ -330,7 +320,7 @@ IFrame::pointer WireCell::Aux::TensorDM::as_frame(const ITensor::vector& tens,
     std::vector<std::shared_ptr<SimpleTrace>> straces;
     for (auto jtrpath : fmd["traces"]) {
         auto trpath = jtrpath.asString();
-        auto trten = located[trpath];
+        auto trten = located.at(trpath);
         auto trmd = trten->metadata();
 
         int tbin = get(trmd, "tbin", 0);
@@ -363,7 +353,7 @@ IFrame::pointer WireCell::Aux::TensorDM::as_frame(const ITensor::vector& tens,
             }
             auto tdpath = jtdpath.asString();
             const bool share = false; // was true
-            PC::Dataset td = as_dataset(tens, tdpath, share);
+            PC::Dataset td = as_dataset(tens, tdpath, located, share);
             // dump_dataset(td, tdpath);
             auto tdmd = td.metadata();
 

aux/src/TensorDMpointgraph.cxx

@@ -13,31 +13,16 @@ WireCell::PointGraph
 WireCell::Aux::TensorDM::as_pointgraph(const ITensor::vector& tens,
                                        const std::string& datapath)
 {
-    std::unordered_map<std::string, ITensor::pointer> located;
-    ITensor::pointer top = nullptr;
-    for (const auto& iten : tens) {
-        const auto& tenmd = iten->metadata();
-        const auto dtype = tenmd["datatype"].asString();
-        const auto dpath = tenmd["datapath"].asString();
-        if (!top and dtype == "pcgraph") {
-            if (datapath.empty() or datapath == dpath) {
-                top = iten;
-            }
-            continue;
-        }
-        if (dtype == "pcdataset") {
-            located[dpath] = iten;
-        }
-        continue;
-    }
+    const auto& located = index_datapaths(tens);
+    ITensor::pointer top = top_tensor(tens, "pcgraph", datapath, located);
 
     if (!top) {
         THROW(ValueError() << errmsg{"no array of datatype \"pcgraph\" at datapath \"" + datapath + "\"" });
     }
 
     const auto& topmd = top->metadata();
-    auto nodes = as_dataset(tens, topmd["nodes"].asString());
-    auto edges = as_dataset(tens, topmd["edges"].asString());
+    auto nodes = as_dataset(tens, topmd["nodes"].asString(), located);
+    auto edges = as_dataset(tens, topmd["edges"].asString(), located);
     return PointGraph(nodes, edges);
 }
 

aux/src/TensorDMpointtree.cxx

@@ -1,24 +1,25 @@
 #include "WireCellAux/TensorDMpointtree.h"
 #include "WireCellAux/TensorDMcommon.h"
 #include "WireCellAux/SimpleTensor.h"
+#include <iostream>
 
 
 using namespace WireCell;
 using namespace WireCell::Aux;
 using namespace WireCell::Aux::TensorDM;
 
 WireCell::Aux::TensorDM::named_pointclouds_t
-WireCell::Aux::TensorDM::as_pcnamedset(const ITensor::vector& tens, const std::string& datapath)
+WireCell::Aux::TensorDM::as_pcnamedset(const ITensor::vector& tens, const std::string& datapath, const located_t& located)
 {
     named_pointclouds_t ret;
-    auto top = top_tensor(tens, "pcnamedset", datapath);
+    auto top = top_tensor(tens, "pcnamedset", datapath, located);
 
     const auto& md = top->metadata();
     auto items = md["items"];
     for (const auto& name : items.getMemberNames()) {
         const auto path = items[name].asString();
 
-        auto ds = as_dataset(tens, path);
+        auto ds = as_dataset(tens, path, located);
         ret.emplace(name, ds);
     }
     return ret;
@@ -74,17 +75,17 @@ WireCell::Aux::TensorDM::as_pctree(const ITensor::vector& tens,
 
     std::unordered_map<std::string, Points::node_t*> nodes_by_datapath;
     Points::node_t::owned_ptr root;
-
+    const auto& located = index_datapaths(tens);
     std::function<void(const std::string& dpath)> dochildren;
     dochildren = [&](const std::string& dpath) -> void {
 
 
-        auto top = top_tensor(tens, "pctreenode", dpath);
+        auto top = top_tensor(tens, "pctreenode", dpath, located);
         auto const& md = top->metadata();        
 
         named_pointclouds_t pcns;
-        if (! md["pointclouds"] ) {
-            pcns = as_pcnamedset(tens, md["pointclouds"].asString());
+        if (! md["pointclouds"].asString().empty() ) {
+            pcns = as_pcnamedset(tens, md["pointclouds"].asString(), located);
         }
 
         std::string ppath = md["parent"].asString();

aux/test/doctest_tensordm_dataset.cxx

@@ -235,7 +235,8 @@ TEST_CASE("tensordm dataset dataset roundtrip")
     // back to dataset
 
     bool share = false;
-    Dataset d2 = TensorDM::as_dataset(itsp, datapath, share);
+    const auto& located = TensorDM::index_datapaths(itenvec);
+    Dataset d2 = TensorDM::as_dataset(itsp, datapath, located, share);
 
     CHECK(d2.size_major() == 3);
 

cfg/pgrapher/experiment/dune-vd/dnnroi.jsonnet

@@ -80,7 +80,7 @@ function (anode, ts, prefix="dnnroi", output_scale=1.0)
 
     local retagger = pg.pnode({
       type: 'Retagger',
-      name: 'dnnroi',
+      name: 'dnnroi%d' % apaid,
       data: {
         // Note: retagger keeps tag_rules an array to be like frame fanin/fanout.
         tag_rules: [{

img/inc/WireCellImg/MultiAlgBlobClustering.h

@@ -0,0 +1,44 @@
+#ifndef WIRECELLIMG_MULTIALGBLOBCLUSTERING
+#define WIRECELLIMG_MULTIALGBLOBCLUSTERING
+
+#include "WireCellIface/ITensorSetFilter.h"
+#include "WireCellIface/IConfigurable.h"
+#include "WireCellAux/Logger.h"
+
+namespace WireCell::Img {
+
+    class MultiAlgBlobClustering : public Aux::Logger, public ITensorSetFilter, public IConfigurable {
+      public:
+        MultiAlgBlobClustering();
+        virtual ~MultiAlgBlobClustering() = default;
+
+        virtual void configure(const WireCell::Configuration& cfg);
+        virtual WireCell::Configuration default_configuration() const;         
+
+        virtual bool operator()(const input_pointer& in, output_pointer& out);
+
+      private:
+        // Count how many times we are called
+        size_t m_count{0};
+
+        /** Config: "inpath"
+         *
+         * The datapath for the input point graph data.  This may be a
+         * regular expression which will be applied in a first-match
+         * basis against the input tensor datapaths.  If the matched
+         * tensor is a pcdataset it is interpreted as providing the
+         * nodes dataset.  Otherwise the matched tensor must be a
+         * pcgraph.
+         */
+        std::string m_inpath{".*"};
+
+        /** Config: "outpath"
+         *
+         * The datapath for the resulting pcdataset.  A "%d" will be
+         * interpolated with the ident number of the input tensor set.
+         */
+        std::string m_outpath{""};
+    };
+}
+
+#endif

img/inc/WireCellImg/PointCloudFacade.h

@@ -0,0 +1,42 @@
+/**
+ *
+ */
+
+#ifndef WIRECELLIMG_POINTCLOUDFACADE
+#define WIRECELLIMG_POINTCLOUDFACADE
+
+#include "WireCellUtil/PointCloudDataset.h"
+#include "WireCellUtil/PointTree.h"
+#include "WireCellUtil/Point.h"
+
+namespace WireCell::PointCloud {
+    using node_t = WireCell::PointCloud::Tree::Points::node_t;
+    using node_ptr = std::unique_ptr<node_t>;
+    using Point = WireCell::Point;
+
+    class Cluster {
+       public:
+        Cluster(const node_ptr& n)
+          : m_node(n.get())
+        {
+        }
+        WireCell::PointCloud::Point calc_ave_pos(const Point& origin, const double dis) const;
+
+       private:
+        node_t* m_node;  /// do not own
+    };
+
+    class Blob {
+       public:
+        Blob(const node_ptr& n)
+          : m_node(n.get())
+        {
+        }
+        double center_pos() const;
+
+       private:
+        node_t* m_node;  /// do not own
+    };
+}  // namespace WireCell::PointCloud
+
+#endif