Parallelize hierarchical DRC processing with OpenMP

build.sh

@@ -42,6 +42,7 @@ HAVE_EXPAT=0
 HAVE_GIT2=1
 HAVE_LSTREAM=1
 HAVE_CPP20=0
+HAVE_OPENMP=0
 
 RUBYINCLUDE=""
 RUBYINCLUDE2=""
@@ -103,6 +104,9 @@ while [ "$*" != "" ]; do
   -without-qtbinding)
     HAVE_QTBINDINGS=0
     ;;
+  -with-openmp)
+    HAVE_OPENMP=1
+    ;;
   -without-qt-uitools)
     HAVE_QT_UITOOLS=0
     ;;
@@ -255,6 +259,7 @@ while [ "$*" != "" ]; do
     echo "  -with-qtbinding       Create Qt bindings for ruby scripts [default]"
     echo "  -without-qtbinding    Don't create Qt bindings for ruby scripts"
     echo "  -without-qt-uitools   Don't include uitools in Qt binding"
+    echo "  -with-openmp          Enable OpenMP parallelization for hierarchical processing"
     echo "  -with-64bit-coord     Use long (64bit) coordinates - EXPERIMENTAL FEATURE"
     echo "                          (only available for gcc>=4.4 for 64bit build)"
     echo "  -without-64bit-coord  Don't use long (64bit) coordinates [default]"
@@ -601,6 +606,7 @@ echo "      HAVE_PNG=$HAVE_PNG"
 echo "      HAVE_EXPAT=$HAVE_EXPAT"
 echo "      HAVE_GIT2=$HAVE_GIT2"
 echo "      HAVE_LSTREAM=$HAVE_LSTREAM"
+echo "      HAVE_OPENMP=$HAVE_OPENMP"
 echo "      RPATH=$RPATH"
 
 mkdir -p $BUILD
@@ -676,6 +682,7 @@ qmake_options=(
   HAVE_GIT2="$HAVE_GIT2"
   HAVE_LSTREAM="$HAVE_LSTREAM"
   HAVE_CPP20="$HAVE_CPP20"
+  HAVE_OPENMP="$HAVE_OPENMP"
   PREFIX="$BIN"
   RPATH="$RPATH"
   KLAYOUT_VERSION="$KLAYOUT_VERSION"

src/db/db/dbCompoundOperation.cc

@@ -766,46 +766,52 @@ CompoundRegionGeometricalBoolOperationNode::implement_bool (CompoundRegionOperat
   one_a.push_back (std::unordered_set<T1> ());
 
   shape_interactions<T, T> computed_a;
-  child (0)->compute_local (cache, layout, cell, interactions_for_child (interactions, 0, computed_a), one_a, proc);
 
-  if (one_a.front ().empty ()) {
-
-    if (m_op == GeometricalOp::And || m_op == GeometricalOp::Not) {
-
-      //  .. no results ..
+  std::vector<std::unordered_set<T2> > one_b;
+  one_b.push_back (std::unordered_set<T2> ());
 
-    } else {
+  shape_interactions<T, T> computed_b;
 
-      std::vector<std::unordered_set<T2> > one_b;
-      one_b.push_back (std::unordered_set<T2> ());
+  bool can_parallel = (m_op != GeometricalOp::And && m_op != GeometricalOp::Not);
 
-      shape_interactions<T, T> computed_b;
+#if defined(_OPENMP)
+  if (can_parallel && proc->threads() > 0) {
+    #pragma omp task shared(one_a, computed_a, cache, layout, cell, interactions, proc)
+    {
+      child (0)->compute_local (cache, layout, cell, interactions_for_child (interactions, 0, computed_a), one_a, proc);
+    }
+    #pragma omp task shared(one_b, computed_b, cache, layout, cell, interactions, proc)
+    {
+      child (1)->compute_local (cache, layout, cell, interactions_for_child (interactions, 1, computed_b), one_b, proc);
+    }
+    #pragma omp taskwait
+  } else
+#endif
+  {
+    child (0)->compute_local (cache, layout, cell, interactions_for_child (interactions, 0, computed_a), one_a, proc);
+    if (!one_a.front().empty()) {
       child (1)->compute_local (cache, layout, cell, interactions_for_child (interactions, 1, computed_b), one_b, proc);
+    } else {
+      if (!can_parallel) { // And or Not and A is empty
+        return; // nothing to do, results remain empty
+      }
+    }
+  }
 
+  if (one_a.front ().empty ()) {
+    if (!can_parallel) {
+      //  .. no results ..
+    } else {
       copy_results (results, one_b);
-
     }
-
   } else {
-
-    std::vector<std::unordered_set<T2> > one_b;
-    one_b.push_back (std::unordered_set<T2> ());
-
-    shape_interactions<T, T> computed_b;
-    child (1)->compute_local (cache, layout, cell, interactions_for_child (interactions, 1, computed_b), one_b, proc);
-
     if (one_b.front ().empty ()) {
-
       if (m_op != GeometricalOp::And) {
         copy_results (results, one_a);
       }
-
     } else {
-
       run_bool (m_op, layout, one_a.front (), one_b.front (), results.front ());
-
     }
-
   }
 }
 
@@ -934,30 +940,54 @@ void compound_region_generic_operation_node<TS, TI, TR>::implement_compute_local
   shape_interactions<TTS, TTI> self_interactions_heap;
   const shape_interactions<TTS, TTI> &self_interactions = interactions_for_child (interactions, 0, self_interactions_heap);
 
-  self->compute_local (cache, layout, cell, self_interactions, self_result, proc);
-
-  db::generic_shape_iterator <TS> is (self_result.front ().begin (), self_result.front ().end ());
-
   std::vector<db::generic_shape_iterator<TI> > iiv;
   std::vector<std::unordered_set<TI> > intruder_results;
-  intruder_results.reserve (children () - 1);  //  important, so that the memory layout will not change while we generate them
+  intruder_results.resize (children () - 1);  //  allocate memory upfront
 
-  for (unsigned int ci = 1; ci < children (); ++ci) {
+  #if defined(_OPENMP)
+  if (proc->threads() > 0) {
+    #pragma omp task shared(self_result, self_interactions_heap, cache, layout, cell, interactions, proc)
+    {
+      self->compute_local (cache, layout, cell, self_interactions, self_result, proc);
+    }
+    for (unsigned int ci = 1; ci < children (); ++ci) {
+      #pragma omp task shared(intruder_results, cache, layout, cell, interactions, proc) firstprivate(ci)
+      {
+        const CompoundRegionOperationNode *intruder = child (ci);
+        std::vector<std::unordered_set<TI> > intruder_result;
+        intruder_result.push_back (std::unordered_set<TI> ());
+
+        shape_interactions<TTS, TTI> intruder_interactions_heap;
+        const shape_interactions<TTS, TTI> &intruder_interactions = interactions_for_child (interactions, ci, intruder_interactions_heap);
+
+        intruder->compute_local (cache, layout, cell, intruder_interactions, intruder_result, proc);
+        intruder_results[ci - 1] = std::move(intruder_result.front());
+      }
+    }
+    #pragma omp taskwait
+  } else
+  #endif
+  {
+    self->compute_local (cache, layout, cell, self_interactions, self_result, proc);
 
-    const CompoundRegionOperationNode *intruder = child (ci);
-    std::vector<std::unordered_set<TI> > intruder_result;
-    intruder_result.push_back (std::unordered_set<TI> ());
+    for (unsigned int ci = 1; ci < children (); ++ci) {
 
-    shape_interactions<TTS, TTI> intruder_interactions_heap;
-    const shape_interactions<TTS, TTI> &intruder_interactions = interactions_for_child (interactions, ci, intruder_interactions_heap);
+      const CompoundRegionOperationNode *intruder = child (ci);
+      std::vector<std::unordered_set<TI> > intruder_result;
+      intruder_result.push_back (std::unordered_set<TI> ());
 
-    intruder->compute_local (cache, layout, cell, intruder_interactions, intruder_result, proc);
+      shape_interactions<TTS, TTI> intruder_interactions_heap;
+      const shape_interactions<TTS, TTI> &intruder_interactions = interactions_for_child (interactions, ci, intruder_interactions_heap);
 
-    intruder_results.push_back (std::unordered_set<TI> ());
-    intruder_results.back ().swap (intruder_result.front ());
+      intruder->compute_local (cache, layout, cell, intruder_interactions, intruder_result, proc);
+      intruder_results[ci - 1] = std::move(intruder_result.front());
+    }
+  }
 
-    iiv.push_back (db::generic_shape_iterator<TI> (intruder_results.back ().begin (), intruder_results.back ().end ()));
+  db::generic_shape_iterator <TS> is (self_result.front ().begin (), self_result.front ().end ());
 
+  for (unsigned int ci = 1; ci < children (); ++ci) {
+    iiv.push_back (db::generic_shape_iterator <TI> (intruder_results[ci - 1].begin (), intruder_results[ci - 1].end ()));
   }
 
   db::local_processor <TS, TI, TR> lproc (layout);

src/db/db/dbCompoundOperation.h

@@ -57,19 +57,23 @@ class CompoundRegionOperationNode;
  *  This cache is important to avoid duplicate evaluation of the same node in
  *  a diamond-graph structure of nodes.
  */
+#include "tlThreads.h"
+
 class DB_PUBLIC CompoundRegionOperationCache
 {
 public:
   template <class TR>
   std::pair<bool, std::vector<std::unordered_set<TR> > *> get (const CompoundRegionOperationNode *node)
   {
+    tl::MutexLocker lock (&m_mutex);
     bool valid = false;
     std::vector<std::unordered_set<TR> > *cache = 0;
     get_cache (cache, valid, node);
     return std::make_pair (valid, cache);
   }
 
 private:
+  tl::Mutex m_mutex;
   std::map<const CompoundRegionOperationNode *, std::vector<std::unordered_set<db::PolygonRefWithProperties> > > m_cache_polyref_wp;
   std::map<const CompoundRegionOperationNode *, std::vector<std::unordered_set<db::PolygonWithProperties> > > m_cache_poly_wp;
   std::map<const CompoundRegionOperationNode *, std::vector<std::unordered_set<db::EdgeWithProperties> > > m_cache_edge_wp;

src/db/db/dbHierNetworkProcessor.cc

@@ -2945,10 +2945,23 @@ template <class T>
 void
 hier_clusters<T>::build_hier_connections_for_cells (cell_clusters_box_converter<T> &cbc, const db::Layout &layout, const std::vector<db::cell_index_type> &cells, const db::Connectivity &conn, const std::set<db::cell_index_type> *breakout_cells, tl::RelativeProgress &progress, instance_interaction_cache_type &instance_interaction_cache, bool separate_attributes)
 {
+#if defined(_OPENMP)
+  #pragma omp parallel for schedule(dynamic)
+  for (long long i = 0; i < (long long)cells.size (); ++i) {
+    db::cell_index_type c = cells[i];
+    build_hier_connections (cbc, layout, layout.cell (c), conn, breakout_cells, instance_interaction_cache, separate_attributes);
+
+    #pragma omp critical
+    {
+      ++progress;
+    }
+  }
+#else
   for (std::vector<db::cell_index_type>::const_iterator c = cells.begin (); c != cells.end (); ++c) {
     build_hier_connections (cbc, layout, layout.cell (*c), conn, breakout_cells, instance_interaction_cache, separate_attributes);
     ++progress;
   }
+#endif
 }
 
 namespace {

src/db/db/dbHierNetworkProcessor.h

@@ -1110,6 +1110,7 @@ class DB_PUBLIC_TEMPLATE instance_interaction_cache
 
   size_t size () const
   {
+    tl::MutexLocker lock (&m_mutex);
     MemStatisticsSimple ms;
     ms << m_map;
     return ms.used ();
@@ -1127,6 +1128,7 @@ class DB_PUBLIC_TEMPLATE instance_interaction_cache
 
   const Value *find (db::cell_index_type ci1, db::cell_index_type ci2, const Key &key) const
   {
+    tl::MutexLocker lock (&m_mutex);
     typename std::map <std::pair<db::cell_index_type, db::cell_index_type>, std::list <std::pair<Key, Value> > >::iterator i1 = m_map.find (std::make_pair (ci1, ci2));
     if (i1 == m_map.end ()) {
       ++m_misses;
@@ -1156,6 +1158,7 @@ class DB_PUBLIC_TEMPLATE instance_interaction_cache
   {
     const size_t instance_cache_variant_threshold = 20;
 
+    tl::MutexLocker lock (&m_mutex);
     std::list <std::pair<Key, Value> > &m = m_map [std::make_pair (ci1, ci2)];
     if (m.size () >= instance_cache_variant_threshold) {
       m.pop_back ();
@@ -1166,6 +1169,7 @@ class DB_PUBLIC_TEMPLATE instance_interaction_cache
   }
 
 private:
+  mutable tl::Mutex m_mutex;
   mutable size_t m_hits, m_misses;
   mutable std::map <std::pair<db::cell_index_type, db::cell_index_type>, std::list <std::pair<Key, Value> > > m_map;
 };