reedsol: Add recovery

src/ballet/reedsol/Local.mk

@@ -3,8 +3,14 @@ ifdef FD_HAS_GFNI
 $(call add-asms,fd_reedsol_gfni_32,fd_ballet)
 endif
 $(call add-objs,fd_reedsol,fd_ballet)
-$(call add-objs,fd_reedsol_internal_16,fd_ballet)
-$(call add-objs,fd_reedsol_internal_32,fd_ballet)
-$(call add-objs,fd_reedsol_internal_64,fd_ballet)
-$(call add-objs,fd_reedsol_internal_128,fd_ballet)
+$(call add-objs,fd_reedsol_encode_16,fd_ballet)
+$(call add-objs,fd_reedsol_encode_32,fd_ballet)
+$(call add-objs,fd_reedsol_encode_64,fd_ballet)
+$(call add-objs,fd_reedsol_encode_128,fd_ballet)
+$(call add-objs,fd_reedsol_recover_16,fd_ballet)
+$(call add-objs,fd_reedsol_recover_32,fd_ballet)
+$(call add-objs,fd_reedsol_recover_64,fd_ballet)
+$(call add-objs,fd_reedsol_recover_128,fd_ballet)
+$(call add-objs,fd_reedsol_recover_256,fd_ballet)
+$(call add-objs,fd_reedsol_pi,fd_ballet)
 $(call make-unit-test,test_reedsol,test_reedsol,fd_ballet fd_util)

src/ballet/reedsol/fd_reedsol.c

@@ -13,18 +13,58 @@
 void fd_reedsol_encode_fini( fd_reedsol_t * rs ) {
 #if FD_HAS_GFNI
   if( FD_LIKELY( (rs->data_shred_cnt==32UL) & (rs->parity_shred_cnt==32UL ) ) )
-    fd_reedsol_encode_32_32( rs->shred_sz, (uchar const * *)rs->data_shred, rs->parity_shred, rs->scratch );
+    fd_reedsol_encode_32_32( rs->shred_sz, rs->encode.data_shred, rs->encode.parity_shred, rs->scratch );
   else
 #endif
     if( FD_UNLIKELY( rs->data_shred_cnt<=16UL ) )
-      fd_reedsol_encode_16( rs->shred_sz, (uchar const * *)rs->data_shred, rs->data_shred_cnt, rs->parity_shred, rs->parity_shred_cnt );
+      fd_reedsol_encode_16( rs->shred_sz, rs->encode.data_shred, rs->data_shred_cnt, rs->encode.parity_shred, rs->parity_shred_cnt );
     else if( FD_LIKELY( rs->data_shred_cnt<=32UL ) )
-      fd_reedsol_encode_32( rs->shred_sz, (uchar const * *)rs->data_shred, rs->data_shred_cnt, rs->parity_shred, rs->parity_shred_cnt );
+      fd_reedsol_encode_32( rs->shred_sz, rs->encode.data_shred, rs->data_shred_cnt, rs->encode.parity_shred, rs->parity_shred_cnt );
     else if( FD_LIKELY( rs->data_shred_cnt<=64UL ) )
-      fd_reedsol_encode_64( rs->shred_sz, (uchar const * *)rs->data_shred, rs->data_shred_cnt, rs->parity_shred, rs->parity_shred_cnt );
+      fd_reedsol_encode_64( rs->shred_sz, rs->encode.data_shred, rs->data_shred_cnt, rs->encode.parity_shred, rs->parity_shred_cnt );
     else
-      fd_reedsol_encode_128( rs->shred_sz, (uchar const * *)rs->data_shred, rs->data_shred_cnt, rs->parity_shred, rs->parity_shred_cnt );
+      fd_reedsol_encode_128( rs->shred_sz, rs->encode.data_shred, rs->data_shred_cnt, rs->encode.parity_shred, rs->parity_shred_cnt );
 
-  rs->data_shred_cnt = 0UL;
+  rs->data_shred_cnt   = 0UL;
   rs->parity_shred_cnt = 0UL;
 }
+
+
+int fd_reedsol_recover_fini( fd_reedsol_t * rs ) {
+  /* How many shreds do we need to consider in order to find
+     rs->data_shred_cnt un-erased? */
+  ulong unerased = 0UL;
+  ulong i=0UL;
+
+  ulong data_shred_cnt   = rs->data_shred_cnt;
+  ulong parity_shred_cnt = rs->parity_shred_cnt;
+  rs->data_shred_cnt   = 0UL;
+  rs->parity_shred_cnt = 0UL;
+
+  for( ; i<data_shred_cnt + parity_shred_cnt; i++ ) {
+    unerased += !rs->recover.erased[ i ];
+    if( unerased==data_shred_cnt ) break;
+  }
+  if( FD_UNLIKELY( unerased != data_shred_cnt ) ) return FD_REEDSOL_ERR_INSUFFICIENT;
+
+  /* if( FD_LIKELY( i==data_shred_cnt ) ) {
+     // Common case: we have all of the data shreds
+       if( FD_UNLIKELY( i<=16UL ) )
+         return fd_reedsol_recover_first_16( rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt );
+       if( FD_LIKELY( i<=32UL ) )
+         return fd_reedsol_recover_first_32( rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt );
+       if( FD_LIKELY( i<=64UL ) )
+         return fd_reedsol_recover_first_64( rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt );
+       return fd_reedsol_recover_first_128(  rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt );
+     } */
+
+  if( FD_UNLIKELY( i<16UL ) )
+    return fd_reedsol_recover_var_16( rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt, rs->recover.erased );
+  if( FD_LIKELY(   i<32UL ) )
+    return fd_reedsol_recover_var_32( rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt, rs->recover.erased );
+  if( FD_LIKELY(   i<64UL ) )
+    return fd_reedsol_recover_var_64( rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt, rs->recover.erased );
+  if( FD_LIKELY(   i<128UL ) )
+    return fd_reedsol_recover_var_128( rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt, rs->recover.erased );
+  return fd_reedsol_recover_var_256( rs->shred_sz, rs->recover.shred, data_shred_cnt, parity_shred_cnt, rs->recover.erased );
+}

src/ballet/reedsol/fd_reedsol.h

@@ -16,13 +16,11 @@
    Solana also calls parity shreds "code" shreds, but due to the naming
    collision with executable code, we have opted for "parity."  This
    mathematical structure thus forces each shred to be of identical size
-   but doesn't otherwise impose any size restrictions. */
+   but doesn't otherwise impose any size restrictions.*/
+
 
 #include "../../util/fd_util.h"
 
-// TODO: Define decode API
-//#define SET_NAME reedsol_shred_set
-//#include "../../util/tmpl/fd_smallset.c"
 
 /* FD_REEDSOL_{DATA, PARITY}_SHREDS_MAX describe the inclusive maximum
    number of data and parity shreds that this implementation supports.
@@ -34,7 +32,14 @@
 
 
 #define FD_REEDSOL_ALIGN     (128UL)
-#define FD_REEDSOL_FOOTPRINT (2176UL)
+#define FD_REEDSOL_FOOTPRINT (2304UL)
+
+/* Return values for the recover operation, which is the only part that
+   can fail for non-bug reasons.  Their meaning is documented with
+   fd_reedsol_recover_fini. */
+#define FD_REEDSOL_OK (0)
+#define FD_REEDSOL_ERR_INCONSISTENT (-1)
+#define FD_REEDSOL_ERR_INSUFFICIENT (-2)
 
 struct __attribute__((aligned(FD_REEDSOL_ALIGN))) fd_reedsol_private {
   uchar scratch[ 1024 ]; /* Used for the ultra high performance implementation */
@@ -48,14 +53,23 @@ struct __attribute__((aligned(FD_REEDSOL_ALIGN))) fd_reedsol_private {
   ulong data_shred_cnt;
   ulong parity_shred_cnt;
 
-  /* {data, parity}_shred: pointers to the first byte of each shred */
-  uchar * data_shred[   FD_REEDSOL_DATA_SHREDS_MAX   ];
-  uchar * parity_shred[ FD_REEDSOL_PARITY_SHREDS_MAX ];
-
-  /* {data, parity}_shred_valid: whether the shred at the corresponding
-     index contains valid data.  Used only for decoding operations. */
-  //fd_reedsol_shred_set_t data_shred_valid;
-  //fd_reedsol_shred_set_t parity_shred_valid;
+  union {
+    struct {
+      /* {data, parity}_shred: pointers to the 1st byte of each shred */
+      uchar const * data_shred[   FD_REEDSOL_DATA_SHREDS_MAX   ];
+      uchar *     parity_shred[ FD_REEDSOL_PARITY_SHREDS_MAX ];
+    } encode;
+    struct {
+      uchar * shred[ FD_REEDSOL_DATA_SHREDS_MAX + FD_REEDSOL_PARITY_SHREDS_MAX ];
+
+      /* erased: whether the shred at the corresponding
+         index is an erasure (i.e. wasn't received or was corrupted).
+         Used only for decoding operations. */
+      /* TODO: Is this the right data type? Should it use a fd_smallset
+         instead? */
+      uchar erased[ FD_REEDSOL_DATA_SHREDS_MAX + FD_REEDSOL_PARITY_SHREDS_MAX ];
+    } recover;
+  };
 };
 
 typedef struct fd_reedsol_private fd_reedsol_t;
@@ -100,12 +114,7 @@ fd_reedsol_encode_init( void * mem, ulong shred_sz ) {
 
 static inline fd_reedsol_t *
 fd_reedsol_encode_add_data_shred( fd_reedsol_t * rs, void const * ptr ) {
-  /* The argument is const to make it clear that an encoding operation
-     won't write to the shred, but we store them in the struct as
-     non-const so that the same struct can be used for encoding and
-     decoding operations, in which the data shreds actually are
-     writeable. */
-  rs->data_shred[ rs->data_shred_cnt++ ] = (uchar *)ptr;
+  rs->encode.data_shred[ rs->data_shred_cnt++ ] = (uchar const*)ptr;
   return rs;
 }
 
@@ -122,7 +131,7 @@ fd_reedsol_encode_add_data_shred( fd_reedsol_t * rs, void const * ptr ) {
 
 static inline fd_reedsol_t *
 fd_reedsol_encode_add_parity_shred( fd_reedsol_t * rs, void * ptr ) {
-  rs->parity_shred[ rs->parity_shred_cnt++ ] = (uchar *)ptr;
+  rs->encode.parity_shred[ rs->parity_shred_cnt++ ] = (uchar *)ptr;
   return rs;
 }
 
@@ -144,8 +153,136 @@ fd_reedsol_encode_cancel( fd_reedsol_t * rs ) {
    have any read or write interest in any of the provided shreds. */
 void fd_reedsol_encode_fini( fd_reedsol_t * rs );
 
+/* fd_reedsol_recover_init: starts a Reed-Solomon recover/decode
+   operation that will recover shreds of size shred_sz.  mem is assumed
+   to be a piece of memory that meets the alignment and size constraints
+   specified above.  Takes a write interest in mem that persists until
+   the operation is canceled or finalized.  shred_sz must be >= 32.
+   Returns mem. */
+static inline fd_reedsol_t *
+fd_reedsol_recover_init( void * mem, ulong shred_sz ) {
+  /* TODO: This is the same as encode_init.  Should I merge them? */
+  fd_reedsol_t * rs = (fd_reedsol_t *)mem;
+
+  rs->shred_sz = shred_sz;
+  rs->data_shred_cnt   = 0UL;
+  rs->parity_shred_cnt = 0UL;
+
+  return rs;
+}
+
+/* fd_reedsol_recover_add_rcvd_shred adds the shred consisting of the of
+   memory [ptr, ptr+shred_sz) to the in-process Reed-Solomon recover
+   operation as a source of data.  Takes a read interest in the shred
+   that persists for the lifetime of the operation (i.e. until finalized
+   or cancelled).  Received shreds have no alignment restrictions and
+   can overlap with each other (if necessary, but there's no known use
+   case for doing so), but should not overlap with any erased shreds in
+   the same recovery operation.
+
+   The shred is treated as a data shred if is_data_shred is non-zero and
+   as a parity shred if not.  Data shreds and parity shreds are mostly
+   treated identically in the recover operation, but having the right
+   number of data shreds is important for validating the shreds are
+   correct.
+
+   Note: The order in which shreds are added (using this function and
+   fd_reedsol_recover_add_erased_shred) is very important for recovery.
+   Shreds must be added in the natural index order or the recover
+   operation will almost certainly fail.  In particular, all data shreds
+   must be added before any parity shreds are added. */
+static inline fd_reedsol_t *
+fd_reedsol_recover_add_rcvd_shred( fd_reedsol_t * rs, int is_data_shred, void const * ptr ) {
+#if FD_REEDSOL_HANDHOLDING
+  // assert is_data_shred==1 implies rs->parity_shred_cnt==0
+  // data_shred_cnt, parity_shred_cnt won't go over the max
+ #endif
+  /* For performance reasons, we need to store all the shred pointers in
+     one flat array, which means the array needs to be non-const.  The
+     const in the function signature signals that this operation won't
+     modify the shred. */
+  rs->recover.shred[  rs->data_shred_cnt + rs->parity_shred_cnt ] = (void *)ptr;
+  rs->recover.erased[ rs->data_shred_cnt + rs->parity_shred_cnt ] = (uchar)0;
+  rs->data_shred_cnt += !!is_data_shred;
+  rs->parity_shred_cnt += !is_data_shred;
+  return rs;
+}
+
+/* fd_reedsol_recover_add_erased_shred adds the block of memory [ptr,
+   ptr+shred_sz) to the in-process Reed-Solomon recover operation as the
+   destination for a shred that will be recovered.  Takes a write
+   interest in the shred that persists for the lifetime of the operation
+   (i.e. until finalized or cancelled).  Erased shreds have no alignment
+   restrictions but should not overlap with any other shreds in the same
+   recover operation.  The contents of the the block of memory are
+   ignored and will be overwritten by the time the operation is
+   finished.
+
+   The shred is treated as a data shred if is_data_shred is non-zero and
+   as a parity shred if not.  Data shreds and parity shreds are mostly
+   treated identically in the recover operation, but having the right
+   number of data shreds is important for validating the shreds are
+   correct.
+
+   Note: The order in which shreds are added (using this function and
+   fd_reedsol_recover_add_rcvd_shred) is very important for recovery.
+   Shreds must be added in the natural index order or the recover
+   operation will almost certainly fail.  In particular, all data shreds
+   must be added before any parity shreds are added. */
+static inline fd_reedsol_t *
+fd_reedsol_recover_add_erased_shred( fd_reedsol_t * rs, int is_data_shred, void * ptr ) {
+#if FD_REEDSOL_HANDHOLDING
+  // assert is_data_shred==1 implies rs->parity_shred_cnt==0
+  // data_shred_cnt, parity_shred_cnt won't go over the max
+ #endif
+  rs->recover.shred[  rs->data_shred_cnt + rs->parity_shred_cnt ] = ptr;
+  rs->recover.erased[ rs->data_shred_cnt + rs->parity_shred_cnt ] = (uchar)1;
+  rs->data_shred_cnt += !!is_data_shred;
+  rs->parity_shred_cnt += !is_data_shred;
+  return rs;
+}
+
+
+/* fd_reedsol_recover_cancel cancels an in-progress encoding operation.
+   Releases any read or write interests in any shreds that were added to
+   the operation.  Upon return, the contents of the erased shreds are
+   undefined. */
+static inline void
+fd_reedsol_recover_cancel( fd_reedsol_t * rs ) {
+  rs->data_shred_cnt   = 0UL;
+  rs->parity_shred_cnt = 0UL;
+}
+
+
+/* fd_reedsol_recover_fini finishes the in-progress recover operation.
+   If successful, upon return, any erased shreds will be filled with the
+   correct data as recovered by the Reed-Solomon recovery algorithm.
+
+   If the recover operation fails with FD_REEDSOL_ERR_{INCONSISTENT,
+   INSUFFICIENT} , the contents of any erased shreds are undefined.
+
+   Upon return, this will no longer have any read or write interest in
+   any of the provided shreds.
+
+   Returns one of:
+   FD_REEDSOL_OK if the recover operation was successful
+   FD_REEDSOL_ERR_INCONSISTENT if the shreds are not consistent with
+     having come from a Reed-Solomon encoding with the provided number
+     of data shreds
+   FD_REEDSOL_ERR_INSUFFICIENT if there's not enough un-erased data to
+     recover data_shred_cnt data shreds.  There must be at least one
+     un-erased shred (data or parity) for each data shred in the
+     operation.
+
 
-/* FIXME: Add decode API */
+   It's worth pointing out that the recovery process differs from
+   typical network coding theory by making no effort to correct data
+   corruption.  The shred signature verification process should detect
+   any data corruption, and any shred that fails signature verification
+   can be treated as an erasure.  This prevents the network from forking
+   if the leader (maliciously) creates data shreds from one version of
+   the block and parity shreds from another version of the block. */
+int fd_reedsol_recover_fini( fd_reedsol_t * rs );
 
 #endif /* HEADER_fd_src_ballet_reedsol_fd_reedsol_h */
 

src/ballet/reedsol/fd_reedsol_arith_avx2.h

@@ -26,7 +26,8 @@ static uchar const fd_reedsol_arith_scale4[ 256UL ] = {
   95, 111, 127,  15,  31,  47,  63, 156, 140, 188, 172, 220, 204, 252, 236,  28,  12,  60,  44,  92,  76, 124, 108, 129, 145, 161, 177, 193, 209, 225, 241,   1,  17,  33,  49,  65,  81,
   97, 113, 166, 182, 134, 150, 230, 246, 198, 214,  38,  54,   6,  22, 102, 118,  70,  86, 187, 171, 155, 139, 251, 235, 219, 203,  59,  43,  27,  11, 123, 107,  91,  75 }; /* Needs to be available at compile time, not link time, to allow the optimizer to use it */
 
-#define GF_ADD( a, b ) wb_xor( a, b )
+#define GF_ADD wb_xor
+#define GF_OR  wb_or
 #define GF_MUL( a, c ) (__extension__({                                                                 \
   wb_t lo = wb_and( a, wb_bcast( 0x0F ) );                                                              \
   wb_t hi = wb_shru( a, 4 );                                                                            \
@@ -35,6 +36,13 @@ static uchar const fd_reedsol_arith_scale4[ 256UL ] = {
   /* c is known at compile time, so this is not a runtime branch */                                     \
   (c==0) ? wb_zero() : ( (c==1) ? a : wb_xor( p0, p1 ) ); } ))
 
+#define GF_MUL_VAR( a, c ) (__extension__({                                                                 \
+  wb_t lo = wb_and( a, wb_bcast( 0x0F ) );                                                              \
+  wb_t hi = wb_shru( a, 4 );                                                                            \
+  wb_t p0 = _mm256_shuffle_epi8( wb_ld( fd_reedsol_arith_consts_avx_mul + 32*c ), lo );                            \
+  wb_t p1 = _mm256_shuffle_epi8( wb_ld( fd_reedsol_arith_consts_avx_mul + 32*fd_reedsol_arith_scale4[ c ] ), hi ); \
+  wb_xor( p0, p1 ); } ))
 
+#define GF_ANY( x ) (0 != _mm256_movemask_epi8( wb_ne( (x), wb_zero() ) ) )
 
 #endif /*HEADER_fd_src_ballet_reedsol_fd_reedsol_arith_avx2_h */

src/ballet/reedsol/fd_reedsol_arith_gfni.h

@@ -17,7 +17,8 @@ FD_IMPORT_BINARY( fd_reedsol_arith_consts_gfni_mul,    "src/ballet/reedsol/const
 extern uchar const fd_reedsol_arith_consts_gfni_mul[]  __attribute__((aligned(128)));
 #endif
 
-#define GF_ADD( a, b ) wb_xor( a, b )
+#define GF_ADD wb_xor
+#define GF_OR  wb_or
 
 /* Older versions of GCC have a bug that cause them to think
    _mm256_gf2p8affine_epi64_epi8 is a symmetric in the first two arguments
@@ -33,6 +34,7 @@ extern uchar const fd_reedsol_arith_consts_gfni_mul[]  __attribute__((aligned(12
 #if FD_USING_CLANG || (GCC_VERSION >= 100000)
 /* c is known at compile time, so this is not a runtime branch */
 #define GF_MUL( a, c ) ((c==0) ? wb_zero() : ( (c==1) ? (a) : _mm256_gf2p8affine_epi64_epi8( a, wb_ld( fd_reedsol_arith_consts_gfni_mul + 32*(c) ), 0 ) ))
+#define GF_MUL_VAR( a, c ) (_mm256_gf2p8affine_epi64_epi8( a, wb_ld( fd_reedsol_arith_consts_gfni_mul + 32*(c) ), 0 ) )
 
 #else
 
@@ -44,8 +46,18 @@ extern uchar const fd_reedsol_arith_consts_gfni_mul[]  __attribute__((aligned(12
             [vec]"x" (a) );                                                    \
       (c==0) ? wb_zero() : ( (c==1) ? (a) : product ); }))
 
+
+#define GF_MUL_VAR( a, c )  (__extension__({                                   \
+      wb_t product;                                                            \
+      __asm__( "vgf2p8affineqb $0x0, %[cons], %[vec], %[out]"                  \
+          : [out]"=x"(product)                                                 \
+          : [cons]"xm"( wb_ld( fd_reedsol_arith_consts_gfni_mul + 32*(c) ) ),  \
+            [vec]"x" (a) );                                                    \
+      (product); }))
+
 #endif
 
+#define GF_ANY( x ) (0 != _mm256_movemask_epi8( wb_ne( (x), wb_zero() ) ) )
 
 
 #endif /*HEADER_fd_src_ballet_reedsol_fd_reedsol_arith_gfni_h */

src/ballet/reedsol/fd_reedsol_arith_none.h

@@ -20,12 +20,16 @@ static short const * gf_arith_log_tbl     = (short const *)fd_reedsol_arith_cons
 static uchar const * gf_arith_invlog_tbl  = fd_reedsol_arith_consts_generic_mul + 256UL*sizeof(short) + 512UL*sizeof(uchar); /* Indexed [-512, 512) */
 
 #define GF_ADD( a, b ) ((a)^(b))
+#define GF_OR(  a, b ) ((a)|(b))
 
 /* c is known at compile time, so this is not a runtime branch.
    Exposing log_tbl at compile time would let the compiler remove a
    branch, but we don't care too much about performance in this case. */
 #define GF_MUL( a, c ) ((c==0) ? 0UL : ( (c==1) ? (a) : (ulong)gf_arith_invlog_tbl[ gf_arith_log_tbl[ a ] + gf_arith_log_tbl[ c ] ] ))
 
+#define GF_MUL_VAR( a, c ) ((ulong)gf_arith_invlog_tbl[ gf_arith_log_tbl[ a ] + gf_arith_log_tbl[ c ] ] )
+
+#define GF_ANY( x ) (!!(x))
 
 
 #endif /*HEADER_fd_src_ballet_reedsol_fd_reedsol_arith_none_h */