Import next (#259)

Co-authored-by: Malachi Phillips <[email protected]>
Nek5000 · Mar 27, 2021 · 0a1d5ae · 0a1d5ae
1 parent d955b9b
commit 0a1d5ae
Show file tree

Hide file tree

Showing 651 changed files with 219,248 additions and 23,780 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -43,9 +43,18 @@ jobs:
    - stage: test
      name: "ethier (block) velocity and pressure projection with subcycling"
      script: cd $NEKRS_EXAMPLES/ethier && nrsmpi ethier 2 4 
+   - stage: test
+     name: "ethier default + moving mesh"
+     script: cd $NEKRS_EXAMPLES/ethier && nrsmpi ethier 2 5 
+   - stage: test
+     name: "ethier subcycle + moving mesh"
+     script: cd $NEKRS_EXAMPLES/ethier && nrsmpi ethier 2 6 
    - stage: test
      name: "lowMach default"
      script: cd $NEKRS_EXAMPLES/lowMach && nrsmpi lowMach 2 1 
+   - stage: test
+     name: "mv_cyl"
+     script: cd $NEKRS_EXAMPLES/mv_cyl && nrsmpi mv_cyl 2 1 
    - stage: test
      name: "conj_ht"
      script: cd $NEKRS_EXAMPLES/conj_ht && nrsmpi conj_ht 2 1

diff --git a/3rd_party/gslib/ogs/include/ogsInterface.h b/3rd_party/gslib/ogs/include/ogsInterface.h
@@ -45,6 +45,7 @@ extern "C"
   void ogsHostScatterMany(void *v, const int k, const char *type, const char *op, void *gsh);
 
   void ogsHostFree(void *gsh);
+
 }
 
 #endif
diff --git a/3rd_party/gslib/ogs/include/ogstypes.h b/3rd_party/gslib/ogs/include/ogstypes.h
@@ -1,6 +1,8 @@
 #if !defined(ogstypes_h)
 #define ogstypes_h 
 
+#define BLOCKSIZE 256
+
 //float data type
 #if 0
 #define DFLOAT_SINGLE

diff --git a/3rd_party/gslib/ogs/ogs.hpp b/3rd_party/gslib/ogs/ogs.hpp
@@ -113,6 +113,7 @@ SOFTWARE.
 #define OGS_HPP 1
 
 #include <functional>
+#include <cstring>
 #include <math.h>
 #include <stdlib.h>
 #include <occa.hpp>
@@ -134,6 +135,8 @@ SOFTWARE.
 #define ogsMax "max"
 #define ogsMin "min"
 
+#define OGS_ENABLE_TIMER
+
 // OCCA+gslib gather scatter
 typedef struct {
 
@@ -188,6 +191,10 @@ void ogsScatter    (void  *sv, void  *v, const char *type, const char *op, ogs_t
 void ogsScatterVec (void  *sv, void  *v, const int k, const char *type, const char *op, ogs_t *ogs);
 void ogsScatterMany(void  *sv, void  *v, const int k, const int stride, const char *type, const char *op, ogs_t *ogs);
 
+void ogsHostTic(MPI_Comm comm, int ifSync);
+void ogsHostToc();
+double ogsTime(bool reportHostTime);
+void ogsResetTime();
 
 // Synchronous device buffer versions
 void ogsGatherScatter    (occa::memory  o_v, const char *type, const char *op, ogs_t *ogs); //wrapper for gslib call
@@ -259,10 +266,10 @@ typedef struct {
 
 namespace oogs{
 
-void start(occa::memory o_v, const int k, const int stride, const char *type, const char *op, oogs_t *h);
-void finish(occa::memory o_v, const int k, const int stride, const char *type, const char *op, oogs_t *h);
+void start(occa::memory &o_v, const int k, const int stride, const char *type, const char *op, oogs_t *h);
+void finish(occa::memory &o_v, const int k, const int stride, const char *type, const char *op, oogs_t *h);
 void startFinish(void *v, const int k, const int stride, const char *type, const char *op, oogs_t *h);
-void startFinish(occa::memory o_v, const int k, const int stride, const char *type, const char *op, oogs_t *h);
+void startFinish(occa::memory &o_v, const int k, const int stride, const char *type, const char *op, oogs_t *h);
 oogs_t *setup(ogs_t *ogs, int nVec, int stride, const char *type, std::function<void()> callback, oogs_mode gsMode);
 oogs_t *setup(int N, long long int *ids, const int k, const int stride, const char *type, MPI_Comm &comm,
               int verbose, occa::device device, std::function<void()> callback, oogs_mode mode);

diff --git a/3rd_party/gslib/ogs/okl/gather.okl b/3rd_party/gslib/ogs/okl/gather.okl
@@ -31,7 +31,7 @@ SOFTWARE.
 		   @restrict const  float *  q,
 		   @restrict float *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -53,7 +53,7 @@ SOFTWARE.
        @restrict const  double *  q,
        @restrict double *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -75,7 +75,7 @@ SOFTWARE.
        @restrict const  int *  q,
        @restrict int *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -97,7 +97,7 @@ SOFTWARE.
        @restrict const  hlong *  q,
        @restrict hlong *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -119,7 +119,7 @@ SOFTWARE.
        @restrict const  float *  q,
        @restrict float *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -141,7 +141,7 @@ SOFTWARE.
        @restrict const  double *  q,
        @restrict double *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -163,7 +163,7 @@ SOFTWARE.
        @restrict const  int *  q,
        @restrict int *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -185,7 +185,7 @@ SOFTWARE.
        @restrict const  hlong *  q,
        @restrict hlong *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -207,7 +207,7 @@ SOFTWARE.
        @restrict const  float *  q,
        @restrict float *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -231,7 +231,7 @@ SOFTWARE.
        @restrict const  double *  q,
        @restrict double *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -255,7 +255,7 @@ SOFTWARE.
        @restrict const  int *  q,
        @restrict int *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -279,7 +279,7 @@ SOFTWARE.
        @restrict const  hlong *  q,
        @restrict hlong *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -303,7 +303,7 @@ SOFTWARE.
        @restrict const  float *  q,
        @restrict float *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -326,7 +326,7 @@ SOFTWARE.
        @restrict const  double *  q,
        @restrict double *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -349,7 +349,7 @@ SOFTWARE.
        @restrict const  int *  q,
        @restrict int *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];
@@ -372,7 +372,7 @@ SOFTWARE.
        @restrict const  hlong *  q,
        @restrict hlong *  gatherq){
 
-  for(dlong g=0;g<Ngather;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong start = gatherStarts[g];
     const dlong end = gatherStarts[g+1];

diff --git a/3rd_party/gslib/ogs/okl/gatherMany.okl b/3rd_party/gslib/ogs/okl/gatherMany.okl
@@ -34,7 +34,7 @@ SOFTWARE.
 		   @restrict const  float *  q,
 		   @restrict float *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -61,7 +61,7 @@ SOFTWARE.
        @restrict const  double *  q,
        @restrict double *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -88,7 +88,7 @@ SOFTWARE.
        @restrict const  int *  q,
        @restrict int *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -115,7 +115,7 @@ SOFTWARE.
        @restrict const  hlong *  q,
        @restrict hlong *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -142,7 +142,7 @@ SOFTWARE.
        @restrict const  float *  q,
        @restrict float *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -169,7 +169,7 @@ SOFTWARE.
        @restrict const  double *  q,
        @restrict double *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -196,7 +196,7 @@ SOFTWARE.
        @restrict const  int *  q,
        @restrict int *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -223,7 +223,7 @@ SOFTWARE.
        @restrict const  hlong *  q,
        @restrict hlong *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -250,7 +250,7 @@ SOFTWARE.
        @restrict const  float *  q,
        @restrict float *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -279,7 +279,7 @@ SOFTWARE.
        @restrict const  double *  q,
        @restrict double *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -308,7 +308,7 @@ SOFTWARE.
        @restrict const  int *  q,
        @restrict int *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -337,7 +337,7 @@ SOFTWARE.
        @restrict const  hlong *  q,
        @restrict hlong *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -366,7 +366,7 @@ SOFTWARE.
        @restrict const  float *  q,
        @restrict float *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -394,7 +394,7 @@ SOFTWARE.
        @restrict const  double *  q,
        @restrict double *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -422,7 +422,7 @@ SOFTWARE.
        @restrict const  int *  q,
        @restrict int *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
@@ -450,7 +450,7 @@ SOFTWARE.
        @restrict const  hlong *  q,
        @restrict hlong *  gatherq){
 
-  for(dlong g=0;g<Ngather*Nentries;++g;@tile(256,@outer,@inner)){
+  for(dlong g=0;g<Ngather*Nentries;++g;@tile(p_blockSize,@outer,@inner)){
 
     const dlong gid = g%Ngather;
     const int k = g/Ngather;
-Original file line number
+Diff line change
@@ Expand Up / @@ -45,6 +45,7 @@ extern "C" @@
       void ogsHostScatterMany(void *v, const int k, const char *type, const char *op, void *gsh);
       void ogsHostFree(void *gsh);
     }
     #endif