From dfc86566b8ecc2cbe221074cceae03aab776163e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Thu, 14 Apr 2022 20:05:55 +0800 Subject: [PATCH] Make in-progress backup/restore as failed when doing the reconcile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make in-progress backup/restore as failed when doing the reconcile to avoid hanging in in-progress status Signed-off-by: Wenkai Yin(尹文开) --- changelogs/unreleased/4833-ywk253100 | 1 + config/crd/v1/bases/velero.io_backups.yaml | 4 +++ config/crd/v1/crds/crds.go | 2 +- pkg/apis/velero/v1/backup.go | 4 +++ pkg/controller/backup_controller.go | 24 ++++++++++++++--- pkg/controller/backup_controller_test.go | 29 ++++++++++++++++---- pkg/controller/restore_controller.go | 21 ++++++++++++--- pkg/controller/restore_controller_test.go | 31 ++++++++++++++++++---- site/content/docs/main/api-types/backup.md | 2 ++ 9 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 changelogs/unreleased/4833-ywk253100 diff --git a/changelogs/unreleased/4833-ywk253100 b/changelogs/unreleased/4833-ywk253100 new file mode 100644 index 0000000000..188528745e --- /dev/null +++ b/changelogs/unreleased/4833-ywk253100 @@ -0,0 +1 @@ +Make in-progress backup/restore as failed when doing the reconcile to avoid hanging in in-progress status \ No newline at end of file diff --git a/config/crd/v1/bases/velero.io_backups.yaml b/config/crd/v1/bases/velero.io_backups.yaml index 259e3cd4fe..b0d3f0d81c 100644 --- a/config/crd/v1/bases/velero.io_backups.yaml +++ b/config/crd/v1/bases/velero.io_backups.yaml @@ -372,6 +372,10 @@ spec: format: date-time nullable: true type: string + failureReason: + description: FailureReason is an error that caused the entire backup + to fail. + type: string formatVersion: description: FormatVersion is the backup format version, including major, minor, and patch version. diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index 54838c3551..60e9847e21 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -29,7 +29,7 @@ import ( ) var rawCRDs = [][]byte{ - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec<]o#9r\xef\xfe\x15\x05\xe7a\xee\x00K\xbeE\x1e\x12\xf8m\xd63\x8b\b\xbb\x993֎\xf3\x10\xe4\x81\xea.Ib\x81F\xaf\xa5\xbe\xb2\x15f4\xd7\xde躺\x83\xf6\a?$\xe0\xe1\xd7\xf0#\x8f\xe6\x0f\x85\xb4\xee\xe7\xce\xc7_\xa4u\xfcCU\xd4F\x14\xcdL\xfc\xcdJ\xb5\xaf\va\xe2\xd7+\x00\x9b\xe9\n\xef\xe0\vMQ\x89\f\xf3+\x80\xb0\x1c\x9er\x15\x10>\xfe\xe0!d\a,\x85\xc7\x05@W\xa8>>l\x9e\xff\xf1\xb1\xf7\x19 G\x9b\x19Y9&\x8aG\f\xa4\x05\x01ϼ,0\x81\xfc\xe0\x0e\u0081\xc1ʠE\xe5,\xb8\x03B&*W\x1b\x04\xbd\x83\x9f\xeb-\x1a\x85\x0em\x03\x1a +j\xebЀu\xc2!\b\a\x02*-\x95\x03\xa9\xc0\xc9\x12\xe1\x0f\x1f\x1f6\xa0\xb7\u007f\xc1\xccY\x10*\aa\xadΤp\x98\xc3Q\x17u\x89~\xec\x1f\xd7\r\xd4\xca\xe8\n\x8d\x93\x91ξu\xa4\xaa\xf3u\xb0\xbc\x0fD\x01\xdf\vr\x12'\xf4\xcb\bT\xc4<\x10\x8d\xd6\xe3\x0eҶ\xcbe\t\xe9\x01\x06\xea$T@~\r\x8fh\b\f\u0603\xae\x8b\x9c\xa4\xf0\x88\x86\b\x96齒\xff\xdd\xc0\xb6\xe04OZ\b\x87A\x00\xda&\x95C\xa3D\x01GQ\xd4x\xc3$)\xc5\t\f\xd2,P\xab\x0e<\xeeb\xd7\xf0\xaf\xda H\xb5\xd3wpp\xae\xb2w\xb7\xb7{\xe9\xe2n\xcatY\xd6J\xba\xd3-o\f\xb9\xad\x9d6\xf66\xc7#\x16\xb7V\xeeW\xc2d\a\xe90#FފJ\xae\x18u\xc5;j]\xe6\xff\x10\x05\xc0~\xe8\xe1\xeaN$\x8c\xd6\x19\xa9\xf6\x9d\x1fX\xeag8@\x1b\xc0˗\x1f\xeaW\xd1\x12\x9a>\x11u~\xfd\xfc\xf8ԕ=i\x87\xd4g\xbaw\x04\xb2e\x01\x11L\xaa\x1d\x1a\xcfĝ\xd1%\xc3D\x95{\xe9c\xd1-$\xaa!\xf9m\xbd-\xa5#\xbe\xffW\x8d\x96\x84\\\xaf\xe1\x9eU\fl\x11\xea*'\xc9\\\xc3F\xc1\xbd(\xb1\xb8\x17\x16\xbf9\x03\x88\xd2vE\x84McAW;\x0e;{\xaau~\x88\xbal\x82_^!\xc3٫È9q\xa2\xa7\x1a\xb5B\xd0\x06J\xb2\a\xe7]\x87\x16\xacmS\xcb\xde\n\xd2Nڋ\xa8\xa9\v\xb4a\xaa\x9cun\xab\x03n&A7\x1c\xf1\xbeD!\xb6X\x80\xc5\x023\xa7\xcd89\x96\x98\xec[\x8a^\x9b\xa0∆ku7-\xb5]\xd8\fH \xb5\xfdz\x90\xd9\xc1\x9by\x92 \x86\x03\xb9F˻\\TUq\x9aZ$,q>L2\xb7\xd1۶\xb0\xe5\x87\xf0\xc66\u007f\xdb\x12tc\xdb\x16\xb4d\x9f\xb2\x8d8\x80ӳ\xcb\xfe\xffIب\xf6\xdf \xb4\x9b\xb3\xa1\xef+\xb4DRI\xee\xfcf\aXV\xeet\x03\xd2ůK\x10\xc9Yi\xe7\xff;f\xcc\xe5\x12\xbf\x19\x8e|W\x89\x9f\xe5\xca\x12D\xe2J3\xfd\xdf!S\xd8X<\x06[\x91̐_\xba\xa3n@\xee\x1a\x86\xe47\xb0\x93\x85C3\xe0\xcco\xda/\xefA\x8c\x14{G\xad\x14.;|\xfeJ\x9e\x97m\x13N\x89t\x19\x0e\xf6\xfek\xf4\xe7\xfb\x86y\x01.p\x80*\r\x96>\xf0}bj\xb6_أ\xfa\xf8\xe5\x13\xe6s\xe4\x814\xc9;[\xc8\xc7\x01\xb2ݩ\x83S\x9e\xba\x8c\xe0\xfa4\xf1\x8dOi܀\x80\x170!8\xb6N'\x1ep\xe2+\xea\xa2\xe5\xc5A\xba\"\x89-\xd2\xfe\r\xcbl\xd8\xd6I\x1a2c?X\xcf\"\xda\x05\aY%.\x94\xcc\x1cX\xe4\xdd\x12S_Ϣ\x90y3\x91\x97\xfb\x8d\x9a\xf6\x86\xfb\xed\x8bv\x1bu\x03\x9f\xbfJ\x1b\xb2\x8f\x9f4\xda/\xda\xf1\x97oBN\x8f\xf8\x1b\x88\xe9\a\xf2\xf6R^m\x13\x1d\xba9\xb4\x04\xe1\xf6m\xe3#\xbc\x86=\xd2\xc2FQ\xdc\x12\xe8\xc1\x19Q?ݼ}跲\xb6\x9c$SZ\xad\xd8T\xae\xc7f\xf2\xc4N\x04\xa9M\x8f#\xe7\xa85\x93\xfa\t\x13\xc1>\x91%\xf1\xe3}\x8e\xb7\x10\x19\xe6\x90\xd7LL\xceL\n\x87{\x99A\x89f?g8\xba\xad\"\xfd\x9e\x86B\xa2\xd6\xf5\xedB\tK3\xed\xb1\x05՝/#\xb3\xa2\x9d\x9b\xd0+2{\xb1\xebDBr\xba\xeb\xf2\x8a\xd8IJ\xff\xb1H]\x91\xe7|\x96$\x8a\x87\v4\xfe\x05\xbc8\xb7\xfd\x1e1o!K\xc1I\xc6\xff!3\xc7\x02\xfd\xbfP\ti\x12\xf6\xf0G>\x1a*\xb076d\xb1\xba\xd3\xd0\f\xd2\x02\xf1\xf7(\x8a\xf3T\xf7\xc8\xe24\xe9\x16,\xbc!\u05fb3\x8f\xe5\x06^\x0f\xdaz\x9b\xba\x938\x9aR\xed7i\xe1\xfa\x05O\xd77gz\xe0z\xa3\xae\xbd\x81\xbfX\xdd4ނV\xc5\t\xaey\xec\xf5oq\x82\x12%1\xa9\x1b\x1f\xc1\xa5\xba\xca\x14KFO\x80\x066\xe7N\xe4\xe6\xcea\x9d$\x87\x95\xb6.\x19\x95\am\x9d\xcf,\xf6\xdc\xd2K\xb2X\xe0e(d\xaf@\xec\xfcɟ6\xf1L\x87\xd4\xde \xe1J\\\xb3\xf3\x1a\x96\xd8\xd8d\xc4)\x19[\xbaCJD\xbaЕ\xff\xfc\xb5\x93\xbd\xa4\xcdO\u007f/\tߥx\x01\xefٲ\x14Ó\xc1$\x14\xef\xfdȸM\x02 \x1f\x1a\x98}\xcd[=݃\f\x82\xf4{0ӥT\x1b\x9e\x00~xw\xb3\xde(I|\x8b\xe3~\x1fǶDo>\xf0\xeeM\xf5\x884g\xee\r\xf68w\x9e\xe7&G1\x11\xa4Ү\x9bN \xb8\x95\xce?X\xd8Ic]\x17\xd1T\xa1\xa8\x17v\u007f\xdb.\x8d\x9c\xd4gc\xde\x148\xfdُ\xec$\xb2\x0e\xfa5\x9e\xafN\x1ef\x8e5>\x14B\x90;\x90\x0ePe\xbaV\x9c~\xa1\xad\xceSx\x16x\x05\x9dL\xb24\x05A\rU]\xa6\x11`\xc5R'\xd5l\x9e\xa6\xdb\xfd'!\x8bo\xc16'K\xd4\xf5\xac\xe1l[\x8fmO~d\uf83c\x14_eY\x97 J\"}jس\xf3\xc51=\x8eë\x90\x8e-\a\xc1e3\xe24m\xaa\xaa@\x97\xba#\xb7\xb8ӆ\xf7\xb3\x9596\x869H\x81V `'dQ\x9bD\ry\x11m/\x895\x82\xb2x\xbf \"m\xf2\x15\x93\"!\x11\x9b\xe8,\xcek\xebʤ\xbb\x8a\x0f\x06\xd3ܳ\xa5\xa4tt\xcf*#I\x96\xf4{{hAĄ:}w\xd1\xce\xdaw\x17m\xa1}w\xd1&\xdbw\x17m\xb9}w\xd1B\xfb\xee\xa2\xc5\xf6\xddE\xfb\xee\xa2\xcdu\x9b\xd3\xd6K\x18\xf9\x8a\xfb\x89\x1f\x17\xb1H8\x9e\x9eCq\x06~\xa8\xa6\xb8\xf7\xd5\xf7\xa9\x15\x96\x9b\xf1Q#u\xb5\xa1\xac\u007f\xc57\x12\xc6$\xa0-\xbahMISrI\x1b$\x8a\xb7/ ^(\xc2L*\xa7\x1c\xaf\xbeM)\xf8Y*\xf3\xe9י6e6\xb1\xd0T\xc7IF\xe8\x10o6\x90\xdbۭ!\xe9\xd7밟\x1b1\xfd\x9bנ&\x94\xe2,\x14\xe0\xcc\x17\xe6\xce\xd1k\x10z\xf4\tfz\x05\xa3\xbf\x1bz-T\xc9L\xd7Ƅ\x93 t\xe2\xf8ú\xff\x8bӡR\x06^\xa5;\x8c,\xe5\xf5\x80\x8aϰԾ[\xf6\x1a\xe5-\\1\x19\xd2\x11\xb4\x01%\v&猴\xf6\xc8\v\u007f\xae|\bw\xf1\xbe\x9c\x0f?\xd2ji\xde\\Aӯ\x90\x99Pї\x1e\x19\xa5\x17\n\xa7\xd7\xc8\xcc\x17\xb5\\R\x193\xac{\x99\x04\xba\\\x0f\x93\x129.Ծ\xbc\xa1\xe2%\xb1\xda\xf17\x1f\x8c\xa5Դ\xbc\xa9\x92e\xb1 0\xb1~\xa5_\x992\x0f\U00082a95$\xe2,W\xa8\\\\\x97\x12\xea@fב\\\x8d2Rg2\vx\xb2\x06e\xae\xbad!+u^y\x92^S2\v\x9a\xebM\x96+Iޯ^\xf4=|\xe0iU\xb3X\r\xb2\xe8#\xcf\xe3\xb7X\xefqI\x95\xc7\"\xc5\xdeX\xd1\xd1TlL\xcc{i\x1dG\xbfNc\x02hJ\xf5\xc6Du\xc6\x04\xc4ٚ\x8dԚ\x8c\t\xd8\vfwVJf~\x1c\xbf\b\t\x8b\xf6\xad\xf8kI\xd4[\x17\xa6M\x8ef\xd6COEs\x16\xc5~\xc6k0\xe7\xa0\xcc>^\x9c\xa4^]\xaf\u007f\x8c\xe5\xba)\t\xcf\xe0g\xa9r/'$\xe8\x1d?\x81/\nsQL㮴\xfe\xde8\xd0A\xa4a\xb1\x12\x86o\x92oO>[a\xd7\xf0Yd\x87~G8\bK1i9\xea\x86]7a\xdam\x1cE_\xae\xd7\x00?\xe9&\x12\xee^\xb3\xb2\xb2\xac\x8a\x13\xd4\x16\xe1\xba?\xe4m1Ǩ\x04X%*{\xd0\xf1\x0e\xecB\xd8\xf1\xd8\xef=\x12\xd1\xc7\x1b\xb0Y\xa1뼁>\xc1<\xa1N\xf0\xf0̾\x0f\xdf\x1d\xcc\xda{\x94\xc1\xbf\x89\x91\xc4\xf0\x9a\xe5\x8f\xef\x1f\xe1[\xa7\x8d\xd8\xe3/\xda_F^\xa2D\xbfw\xef&z\xd0a1\xe3\x16\v\xb2\xc4\b\x11µ\xe8\x01\xb06\x91\x1evC\x9b\xfc ,\xc7\xd4\xdb\xcc\xfes\xaeXX\xcc\xd3\xd3/~\x01N\x96\xb8\xfeT\xfblʪ\x12\xc6\"Q3.\xcc\x0f\xda\xd2\u007f\x0f\xfauL\xe1\xe9\xb0\xe6\x1f\x87x\x1b\xe4d='m.\xc2\xde_\x9b\x8e\x82\x17I\xb4$\xa8\xcf\xe3\xa3:\x81^\x87I~\x97\xeb\xb1s\x89)8\x9d\xd7%(\xb0\xf6\xc5v\xef{\xf7w\xcac\x99\xba\u007f\uf12b\xed\xf2\r|\xee\x16\xdf\xdb\bG>\xb5ዻ\x1e\x84\xbf\xe8\xfa\xa6K\xf8!C\xdd{\x03e\x9eO\xf7\xe7#\xf8\xa5\v\x93{\xd487\xdeܦ\u007f\x15\xb6ɂ\x8f\x9a\xf8\x16\x9c\x1f\xc9\x1e4A\xc3\x1c\xf0\x88\n\xb4\xe2\xa47_\x89\xf5\xaf\xb1\fnj%\x93:PBV\xbd\xae\n-\xf2\xb8ã\xcd\n/x<\xb1\xfe2G4\x1f\xec\fL~1`\xa7\xcd\x18\x11\xce\x15\xa67,w\x90\v\x87\xabQ\xa0I\xbaoT\xd82+\xfb\x82n?:G\xf1Ș\xb7\xde\xe7\xdf\xe3fjd\xb4\xbfN;Q\x80\xaa˭7\xe8\"v\x18\xe3\xdf\xe3f\xb0\xe5l8\x06\x99\xd9^~aR9\xdc\xe3\xd0\xe9<_\xd9}\x94\x9f\x8bW\u058c\x9cZ\x99\xad\xb3\f\xad\xdd\xd5E1\x16d4\x92\xfb\xfe\xcb\xe4\x03\xbeŇ\x0f\xb8\x93W\x81|:\x18_\xa7\xf0ǃ%Z+\xf6\xf1ŃW\xb2@{TȎ\xcf\xc8jB@\xda\x1e'\xf5\xef\xfb\xfb\x9c\x98\xc8\\-\xc2\x041\x1d\xd8\xe9\xf5a\xcc/(\xf4\x1ev\xb2\xe0\xae\xe1m\x96`\x9a/\xa4\xc9\xd7J\x9a\x14S\xfe\xb9\xe9H\xb4\xe1\x84(3\xa2}\xc3\b\v\xb9\x97d\a\x89I{a\xb6b\x8f\xabL\x17\x05r\xed\xc99^\xdfr\xb3zأo\x14\x9d-\xed\xa7n\xdf(\xb6A[y8\xf1ɢ\x9b\xe0b\x8dG\x18\xa5\xf8\x8b67PJE\xff\x90\x1fͩ\x858\xf8\"\x83\xce\xcfC,\xe0\xfd@}\x9a\x8a\x84\x8ey\xc2(fS\x0e\xe0\xf8)\xf4\n\xbe\u0e7f\xe2\x0f\x961\xe7\xe4\xd9\xd8\xc3L\xd4e\xa3\x1e\x8c\xdeS\xec;\xf2c\xa3\x12F~{\x10\xc6IQ\x14'?\xc9\xe4\xec#?|B2\a\x93>\xc18Y\x03\x96K\x94\r\xdd\xda@[*/\t|$\xbcյ\xebm\xd0v\x83\x8f\x88E\x9cs\r_\xb4Ø\x87\x95}\x98\xa4\xd2к\x15\xeev\xda8\x1fׯV w\xc1\xc7\x18\x81K&\x9a\xcf\x1e\xfc;I ]\x9b\xffj\xa5\x97\xc3\a\x83²\xf4:~\xad\x89\x8f\aE\x96\x91\v\x8b\xb7։bDk\xfc\xa6\xe3\x06v\xe6H\xfa0\xff\xb7\x11\xef\xe6\x8c\xe0\x9bn\xff\xe6\x8eFc3\x18\x9c\xa7\x1c\x97ox\x8d9j?\x80\x0f\xf5Q\xc1\xab\x91Α\x96\xea\x1e\u0380#\xbdT\x14`5\xec\xc4ċ\x1cs\xfa\x92\u007f'\x8b\xb6\x99N\n\xf6\xa3\x86\xa6\xf3\x94A\f\x8b\xd3Ė-\x93`bY\xbe\x86t\x8a\xaf\xe1\xbc\x1f\xbe\x9ax\x03V\xaa\xf8L\xa0O\xd7xQ\xb0\xe4\xbd\x1b\xe4\bx\xf4\xc0\xe9\xcc\xef\xeey\xd9}\xf4\xff\xba\x0e\xf6\xb1\xb10\x9fS<\xb5\xe7A\xf7\xc1A8\xed\xf2\x16b\xf0\xaeF\xe8\xf1\a\xb9\xf3'a\x19a\xfdǿ\xf9\x01\xf71\xc9g\xf90뮰'\xd2\xf8\x1d\xf0\t+\x83\x99\x18u\xe7\x01\x1e\n$?\xc2\"\xf6=\xa1\x0f\x179\x92Ƿ\x85F\xef\x19\x17\xc5\x17,\xdf'Z8\xbe-\"\xfaf\xe1\xd0\xfb\xae\xeeU\x18%\xd5~i\x8f\xfd{\xe86\x12\x0f\x05\b#\x11\xd1\xc82\x9a\x18i1\"\xea\x04D\x11lj\x87\xe5\x06A\xd2;\x85D\xa3v\xe0\xec#+м\xb3\xb7\xc3L\xe1K\x9bf\x12Y\x86$\xae_\x86/\xd5^_\xf3\x1f\xf11Z\xfe3\xd3ʛ[{\a\xff\xf1\x9fW\x10\xf2\x98\xcf\xf1\xd5Y\xfa\xf8\u007f\x01\x00\x00\xff\xff\x01\xf3\x19\x8b\xd5W\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec<]o#9r\xef\xfe\x15\x05\xe7a\xee\x00K\xbeE\x1e\x12\xf8m\xd63\x8b\b\xbb\x993֎\xf3\x10\xe4\x81\xea.Ib\x81F\xaf\xa5\xbe\xb2\x15f4\xd7\xde躺\x83\xf6\a?$\xe0\xe1\xd7\xf0#\x8f\xe6\x0f\x85\xb4\xee\xe7\xce\xc7_\xa4u\xfcCU\xd4F\x14\xcdL\xfc\xcdJ\xb5\xaf\va\xe2\xd7+\x00\x9b\xe9\n\xef\xe0\vMQ\x89\f\xf3+\x80\xb0\x1c\x9er\x15\x10>\xfe\xe0!d\a,\x85\xc7\x05@W\xa8>>l\x9e\xff\xf1\xb1\xf7\x19 G\x9b\x19Y9&\x8aG\f\xa4\x05\x01ϼ,0\x81\xfc\xe0\x0e\u0081\xc1ʠE\xe5,\xb8\x03B&*W\x1b\x04\xbd\x83\x9f\xeb-\x1a\x85\x0em\x03\x1a +j\xebЀu\xc2!\b\a\x02*-\x95\x03\xa9\xc0\xc9\x12\xe1\x0f\x1f\x1f6\xa0\xb7\u007f\xc1\xccY\x10*\aa\xadΤp\x98\xc3Q\x17u\x89~\xec\x1f\xd7\r\xd4\xca\xe8\n\x8d\x93\x91ξu\xa4\xaa\xf3u\xb0\xbc\x0fD\x01\xdf\vr\x12'\xf4\xcb\bT\xc4<\x10\x8d\xd6\xe3\x0eҶ\xcbe\t\xe9\x01\x06\xea$T@~\r\x8fh\b\f\u0603\xae\x8b\x9c\xa4\xf0\x88\x86\b\x96齒\xff\xdd\xc0\xb6\xe04OZ\b\x87A\x00\xda&\x95C\xa3D\x01GQ\xd4x\xc3$)\xc5\t\f\xd2,P\xab\x0e<\xeeb\xd7\xf0\xaf\xda H\xb5\xd3wpp\xae\xb2w\xb7\xb7{\xe9\xe2n\xcatY\xd6J\xba\xd3-o\f\xb9\xad\x9d6\xf66\xc7#\x16\xb7V\xeeW\xc2d\a\xe90#FފJ\xae\x18u\xc5;j]\xe6\xff\x10\x05\xc0~\xe8\xe1\xeaN$\x8c\xd6\x19\xa9\xf6\x9d\x1fX\xeag8@\x1b\xc0˗\x1f\xeaW\xd1\x12\x9a>\x11u~\xfd\xfc\xf8ԕ=i\x87\xd4g\xbaw\x04\xb2e\x01\x11L\xaa\x1d\x1a\xcfĝ\xd1%\xc3D\x95{\xe9c\xd1-$\xaa!\xf9m\xbd-\xa5#\xbe\xffW\x8d\x96\x84\\\xaf\xe1\x9eU\fl\x11\xea*'\xc9\\\xc3F\xc1\xbd(\xb1\xb8\x17\x16\xbf9\x03\x88\xd2vE\x84McAW;\x0e;{\xaau~\x88\xbal\x82_^!\xc3٫È9q\xa2\xa7\x1a\xb5B\xd0\x06J\xb2\a\xe7]\x87\x16\xacmS\xcb\xde\n\xd2Nڋ\xa8\xa9\v\xb4a\xaa\x9cun\xab\x03n&A7\x1c\xf1\xbeD!\xb6X\x80\xc5\x023\xa7\xcd89\x96\x98\xec[\x8a^\x9b\xa0∆ku7-\xb5]\xd8\fH \xb5\xfdz\x90\xd9\xc1\x9by\x92 \x86\x03\xb9F˻\\TUq\x9aZ$,q>L2\xb7\xd1۶\xb0\xe5\x87\xf0\xc66\u007f\xdb\x12tc\xdb\x16\xb4d\x9f\xb2\x8d8\x80ӳ\xcb\xfe\xffIب\xf6\xdf \xb4\x9b\xb3\xa1\xef+\xb4DRI\xee\xfcf\aXV\xeet\x03\xd2ůK\x10\xc9Yi\xe7\xff;f\xcc\xe5\x12\xbf\x19\x8e|W\x89\x9f\xe5\xca\x12D\xe2J3\xfd\xdf!S\xd8X<\x06[\x91̐_\xba\xa3n@\xee\x1a\x86\xe47\xb0\x93\x85C3\xe0\xcco\xda/\xefA\x8c\x14{G\xad\x14.;|\xfeJ\x9e\x97m\x13N\x89t\x19\x0e\xf6\xfek\xf4\xe7\xfb\x86y\x01.p\x80*\r\x96>\xf0}bj\xb6_أ\xfa\xf8\xe5\x13\xe6s\xe4\x814\xc9;[\xc8\xc7\x01\xb2ݩ\x83S\x9e\xba\x8c\xe0\xfa4\xf1\x8dOi܀\x80\x170!8\xb6N'\x1ep\xe2+\xea\xa2\xe5\xc5A\xba\"\x89-\xd2\xfe\r\xcbl\xd8\xd6I\x1a2c?X\xcf\"\xda\x05\aY%.\x94\xcc\x1cX\xe4\xdd\x12S_Ϣ\x90y3\x91\x97\xfb\x8d\x9a\xf6\x86\xfb\xed\x8bv\x1bu\x03\x9f\xbfJ\x1b\xb2\x8f\x9f4\xda/\xda\xf1\x97oBN\x8f\xf8\x1b\x88\xe9\a\xf2\xf6R^m\x13\x1d\xba9\xb4\x04\xe1\xf6m\xe3#\xbc\x86=\xd2\xc2FQ\xdc\x12\xe8\xc1\x19Q?ݼ}跲\xb6\x9c$SZ\xad\xd8T\xae\xc7f\xf2\xc4N\x04\xa9M\x8f#\xe7\xa85\x93\xfa\t\x13\xc1>\x91%\xf1\xe3}\x8e\xb7\x10\x19\xe6\x90\xd7LL\xceL\n\x87{\x99A\x89f?g8\xba\xad\"\xfd\x9e\x86B\xa2\xd6\xf5\xedB\tK3\xed\xb1\x05՝/#\xb3\xa2\x9d\x9b\xd0+2{\xb1\xebDBr\xba\xeb\xf2\x8a\xd8IJ\xff\xb1H]\x91\xe7|\x96$\x8a\x87\v4\xfe\x05\xbc8\xb7\xfd\x1e1o!K\xc1I\xc6\xff!3\xc7\x02\xfd\xbfP\ti\x12\xf6\xf0G>\x1a*\xb076d\xb1\xba\xd3\xd0\f\xd2\x02\xf1\xf7(\x8a\xf3T\xf7\xc8\xe24\xe9\x16,\xbc!\u05fb3\x8f\xe5\x06^\x0f\xdaz\x9b\xba\x938\x9aR\xed7i\xe1\xfa\x05O\xd77gz\xe0z\xa3\xae\xbd\x81\xbfX\xdd4ނV\xc5\t\xaey\xec\xf5oq\x82\x12%1\xa9\x1b\x1f\xc1\xa5\xba\xca\x14KFO\x80\x066\xe7N\xe4\xe6\xcea\x9d$\x87\x95\xb6.\x19\x95\am\x9d\xcf,\xf6\xdc\xd2K\xb2X\xe0e(d\xaf@\xec\xfcɟ6\xf1L\x87\xd4\xde \xe1J\\\xb3\xf3\x1a\x96\xd8\xd8d\xc4)\x19[\xbaCJD\xbaЕ\xff\xfc\xb5\x93\xbd\xa4\xcdO\u007f/\tߥx\x01\xefٲ\x14Ó\xc1$\x14\xef\xfdȸM\x02 \x1f\x1a\x98}\xcd[=݃\f\x82\xf4{0ӥT\x1b\x9e\x00~xw\xb3\xde(I|\x8b\xe3~\x1fǶDo>\xf0\xeeM\xf5\x884g\xee\r\xf68w\x9e\xe7&G1\x11\xa4Ү\x9bN \xb8\x95\xce?X\xd8Ic]\x17\xd1T\xa1\xa8\x17v\u007f\xdb.\x8d\x9c\xd4gc\xde\x148\xfdُ\xec$\xb2\x0e\xfa5\x9e\xafN\x1ef\x8e5>\x14B\x90;\x90\x0ePe\xbaV\x9c~\xa1\xad\xceSx\x16x\x05\x9dL\xb24\x05A\rU]\xa6\x11`\xc5R'\xd5l\x9e\xa6\xdb\xfd'!\x8bo\xc16'K\xd4\xf5\xac\xe1l[\x8fmO~d\uf83c\x14_eY\x97 J\"}jس\xf3\xc51=\x8eë\x90\x8e-\a\xc1e3\xe24m\xaa\xaa@\x97\xba#\xb7\xb8ӆ\xf7\xb3\x9596\x869H\x81V `'dQ\x9bD\ry\x11m/\x895\x82\xb2x\xbf \"m\xf2\x15\x93\"!\x11\x9b\xe8,\xcek\xebʤ\xbb\x8a\x0f\x06\xd3ܳ\xa5\xa4tt\xcf*#I\x96\xf4{{hAĄ:}w\xd1\xce\xdaw\x17m\xa1}w\xd1&\xdbw\x17m\xb9}w\xd1B\xfb\xee\xa2\xc5\xf6\xddE\xfb\xee\xa2\xcdu\x9b\xd3\xd6K\x18\xf9\x8a\xfb\x89\x1f\x17\xb1H8\x9e\x9eCq\x06~\xa8\xa6\xb8\xf7\xd5\xf7\xa9\x15\x96\x9b\xf1Q#u\xb5\xa1\xac\u007f\xc57\x12\xc6$\xa0-\xbahMISrI\x1b$\x8a\xb7/ ^(\xc2L*\xa7\x1c\xaf\xbeM)\xf8Y*\xf3\xe9י6e6\xb1\xd0T\xc7IF\xe8\x10o6\x90\xdbۭ!\xe9\xd7밟\x1b1\xfd\x9bנ&\x94\xe2,\x14\xe0\xcc\x17\xe6\xce\xd1k\x10z\xf4\tfz\x05\xa3\xbf\x1bz-T\xc9L\xd7Ƅ\x93 t\xe2\xf8ú\xff\x8bӡR\x06^\xa5;\x8c,\xe5\xf5\x80\x8aϰԾ[\xf6\x1a\xe5-\\1\x19\xd2\x11\xb4\x01%\v&猴\xf6\xc8\v\u007f\xae|\bw\xf1\xbe\x9c\x0f?\xd2ji\xde\\Aӯ\x90\x99Pї\x1e\x19\xa5\x17\n\xa7\xd7\xc8\xcc\x17\xb5\\R\x193\xac{\x99\x04\xba\\\x0f\x93\x129.Ծ\xbc\xa1\xe2%\xb1\xda\xf17\x1f\x8c\xa5Դ\xbc\xa9\x92e\xb1 0\xb1~\xa5_\x992\x0f\U00082a95$\xe2,W\xa8\\\\\x97\x12\xea@fב\\\x8d2Rg2\vx\xb2\x06e\xae\xbad!+u^y\x92^S2\v\x9a\xebM\x96+Iޯ^\xf4=|\xe0iU\xb3X\r\xb2\xe8#\xcf\xe3\xb7X\xefqI\x95\xc7\"\xc5\xdeX\xd1\xd1TlL\xcc{i\x1dG\xbfNc\x02hJ\xf5\xc6Du\xc6\x04\xc4ٚ\x8dԚ\x8c\t\xd8\vfwVJf~\x1c\xbf\b\t\x8b\xf6\xad\xf8kI\xd4[\x17\xa6M\x8ef\xd6COEs\x16\xc5~\xc6k0\xe7\xa0\xcc>^\x9c\xa4^]\xaf\u007f\x8c\xe5\xba)\t\xcf\xe0g\xa9r/'$\xe8\x1d?\x81/\nsQL㮴\xfe\xde8\xd0A\xa4a\xb1\x12\x86o\x92oO>[a\xd7\xf0Yd\x87~G8\bK1i9\xea\x86]7a\xdam\x1cE_\xae\xd7\x00?\xe9&\x12\xee^\xb3\xb2\xb2\xac\x8a\x13\xd4\x16\xe1\xba?\xe4m1Ǩ\x04X%*{\xd0\xf1\x0e\xecB\xd8\xf1\xd8\xef=\x12\xd1\xc7\x1b\xb0Y\xa1뼁>\xc1<\xa1N\xf0\xf0̾\x0f\xdf\x1d\xcc\xda{\x94\xc1\xbf\x89\x91\xc4\xf0\x9a\xe5\x8f\xef\x1f\xe1[\xa7\x8d\xd8\xe3/\xda_F^\xa2D\xbfw\xef&z\xd0a1\xe3\x16\v\xb2\xc4\b\x11µ\xe8\x01\xb06\x91\x1evC\x9b\xfc ,\xc7\xd4\xdb\xcc\xfes\xaeXX\xcc\xd3\xd3/~\x01N\x96\xb8\xfeT\xfblʪ\x12\xc6\"Q3.\xcc\x0f\xda\xd2\u007f\x0f\xfauL\xe1\xe9\xb0\xe6\x1f\x87x\x1b\xe4d='m.\xc2\xde_\x9b\x8e\x82\x17I\xb4$\xa8\xcf\xe3\xa3:\x81^\x87I~\x97\xeb\xb1s\x89)8\x9d\xd7%(\xb0\xf6\xc5v\xef{\xf7w\xcac\x99\xba\u007f\uf12b\xed\xf2\r|\xee\x16\xdf\xdb\bG>\xb5ዻ\x1e\x84\xbf\xe8\xfa\xa6K\xf8!C\xdd{\x03e\x9eO\xf7\xe7#\xf8\xa5\v\x93{\xd487\xdeܦ\u007f\x15\xb6ɂ\x8f\x9a\xf8\x16\x9c\x1f\xc9\x1e4A\xc3\x1c\xf0\x88\n\xb4\xe2\xa47_\x89\xf5\xaf\xb1\fnj%\x93:PBV\xbd\xae\n-\xf2\xb8ã\xcd\n/x<\xb1\xfe2G4\x1f\xec\fL~1`\xa7\xcd\x18\x11\xce\x15\xa67,w\x90\v\x87\xabQ\xa0I\xbaoT\xd82+\xfb\x82n?:G\xf1Ș\xb7\xde\xe7\xdf\xe3fjd\xb4\xbfN;Q\x80\xaa˭7\xe8\"v\x18\xe3\xdf\xe3f\xb0\xe5l8\x06\x99\xd9^~aR9\xdc\xe3\xd0\xe9<_\xd9}\x94\x9f\x8bW\u058c\x9cZ\x99\xad\xb3\f\xad\xdd\xd5E1\x16d4\x92\xfb\xfe\xcb\xe4\x03\xbeŇ\x0f\xb8\x93W\x81|:\x18_\xa7\xf0ǃ%Z+\xf6\xf1ŃW\xb2@{TȎ\xcf\xc8jB@\xda\x1e'\xf5\xef\xfb\xfb\x9c\x98\xc8\\-\xc2\x041\x1d\xd8\xe9\xf5a\xcc/(\xf4\x1ev\xb2\xe0\xae\xe1m\x96`\x9a/\xa4\xc9\xd7J\x9a\x14S\xfe\xb9\xe9H\xb4\xe1\x84(3\xa2}\xc3\b\v\xb9\x97d\a\x89I{a\xb6b\x8f\xabL\x17\x05r\xed\xc99^\xdfr\xb3\x86C\xbb_Q\xd8ť\xfd\xd4\xed\x1br+\x9e\xdb\xfe\xba\x9c\xf0\xaf\x96\xf0\x936N\x1al߈:CH\xf3\xc4\x17\x99nO\x85\xd1ה\xce1\xed\xf6\x8d\x1b,\xe8U\x0f'>\xaet\x13\x9c\xc1\xf1X\xa8\x14\u007f\xd1\xe6\x06J\xa9\xe8\x1f\xf2\xf89\t\x12\a_\x84??d\xb1\x80\xf7\x03\xf5ij':\x86\x14㆘rU\xc7\xcf\xcbW\xf0\x05\xcf=+\u007f\x04\x8e9\xa7\xf9ƞ\x90\xa2.\x1b\xf5`\xf4\x9e\xa2\xf4\x91\x1f\x1b\xe55\xf2ۃ0N\x8a\xa28\xf9I&g\x1f\xf9\xe1\x13\x92\xe1\x9a\xf4^\xc6\xc9\x1a\xb0\\\xa2l\xe8֦\x04\xa4\xf2\x92\xc0\x87\xd7[]\xbb\x9e*iUшX\xc49\xd7\xf0E;\x8c\x19cهI\xca\x17\xad[\xe1n\xa7\x8d\xf3\x19\x88\xd5\n\xe4.xC#piO\xf0)\x89\u007f\xd1\t\xa4k3u\xad\xf4r\xa0cx\x13\xf2\xb5\xefR\x9c\xfcA\xa6\xc82r\xb6\xf1\xd6:Q\x8c\xe8\xb7\xdft0\xc2n'I\x1f\xe6\xff6⇝\x11|\xd3\xed\xdf\xdc&i\xac\x1b\x83\xf3\x94\xe3B\x13\xaf\xdbG-\x1dp\xf9\x01*x5\xd29ҧ\xddc$p\xa4A\x8b\x02,锉\xb7C\xe64;\xffN\xb6w3\x9d\xbe\xec\xc77M\xe7)\xd3\x1d\x16\xa7\x89-[&\xc1IJ|!\xa3\xb4q,\xb12;\b\xb5'\xa12\xba\xde\x1f\xa2\\NX\xc6\t\xb8yMHAU\xd4{\x12\xf5p\f\xe3j\xa3:)\x98p0\x93w\xd0\x15\xd9\xcb$\xa6!\x11\x1d_\x15\xbc\r\xaf\x81\xacvF\x97\xab\xc0\vNQ݄Ԉ\x91\x9a\xfc\u007f\n\xe4'\x80\xb6\xd7\xeeY\f\xaa\n\x15\b\x1b\xf0I\xa8\xb2\x9cg\xeb\\\x9e\xc2\t\xe3R\xa3\x8a\xc7^煀\x82!\x8f\xe3\xfb\x18\x12?\xbe\xda\xf4~\xf8\xbe\xe3\rX\xa9\u20c6>\xb1\xe4E\xc1R\x9ca\x90c\xf5ѣ\xb1\xb3\b\xa1\x17\x0f\xf4\xd1\xff\xeb\x86\x02\xc7\xc6\xc2|N\xf1)\x9f\a\xdd\aG\xf6\xb4\xcb[\x88\xc1\x0f\x1c\xa1\xc7\x1f\xe4Ο\xd9e\x84\xf5\x1f\xff\xe6G\xf1\xc7$\x9f\xe5ì\xbb\u009eH\xe3w\xc0'\xac\ffb4\xf0\x00x(\x90\xfc\b\x8b\xd8\xf7\x84>\\\xe4\xf2\x1e\xdf\x16Ľg\x04\x17\xdf\xda|\x9f\xb8\xe6\xf8\xb6\xd8\xed\x9b\x05nﻺWa\x94T\xfb\xa5=\xf6\xef\xa1\xdbH\xe4\x16 \x8c\xc4n#\xcbh\xa2\xb9\xc5ح\x13\xbaE\x1c'\x9e\xc0\x1b\x84s\xef\x14\xbc\x8dځ\xb3\x8f\xac@\xf3\xce\xde\x0e3\x85/mBLd\x19\x92\xb8~\x19\xbe\xa9{}\xcd\u007f\xc4gs\xf9\xcfL+on\xed\x1d\xfc\xc7\u007f^Aȸ>\xc7\xf7q\xe9\xe3\xff\x05\x00\x00\xff\xffj\x17\xd2\xcb\u007fX\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4YKo#\xb9\x11\xbe\xebW\x14f\x0f\xbe\x8cZ\xb3\xc9!\x81.\x81F\x93\x00\x83x\xd6\xc6\xc8q\x0eI\x80\xa5Ȓ\xc45\x9b\xec\xf0!\xad\x12\xe4\xbf\aŇ\xba\xd5ݲ\xe4A\xb2ˋ->\x8aU_\xbdٓ\xe9t:a\x8d|F\xeb\xa4\xd1s`\x8dğ=j\xfa媗\u07fbJ\x9a\xd9\xfe\xfbɋ\xd4b\x0e\xcb༩\xbf\xa23\xc1r\xfc\x84\x1b\xa9\xa5\x97FOj\xf4L0\xcf\xe6\x13\x00\xa6\xb5\xf1\x8c\xa6\x1d\xfd\x04\xe0F{k\x94B;ݢ\xae^\xc2\x1a\xd7A*\x816\x12/W\xef?T\xbf\xab>L\x00\xb8\xc5x\xfcI\xd6\xe8<\xab\x9b9\xe8\xa0\xd4\x04@\xb3\x1a\xe7\xb0f\xfc%4\xce\x1b˶\xa8\fOwU{ThM%\xcd\xc45\xc8\xe9\xea\xad5\xa1\x99C\xbb\x90(d\xb6\x92H\x1f#\xb1U\"v\x9f\x89\xc5u%\x9d\xff\xf3\xe5=\xf7\xd2\xf9\xb8\xafQ\xc12u\x89\xad\xb8\xc5\xed\x8c\xf5?\xb4WOa\xedTZ\x91z\x1b\x14\xb3\x17\x8eO\x00\x1c7\r\xce!\x9en\x18G1\x01ȘEjS`BD-0\xf5h\xa5\xf6h\x97F\x85Z\x9f\xee\x12踕\x8d\x8f('Y \v\x03E\x1ap\x9e\xf9\xe0\xc0\x05\xbe\x03\xe6`\xb1gR\xb1\xb5\xc2\xd9_4+\xffGz\x00?9\xa3\x1f\x99\xdf͡J\xa7\xaaf\xc7\\YM:z\xec\xcc\xf8#\t༕z;\xc6\xd2=s\xfe\x99))NZ\a\xe9\xc0\xef\x10\x14s\x1e\x8c\xb6e,\x1e?\x97ț\x1c(\xfb[ƪ\x82E\xf6\\\xb3\x81\x0f \xa4\xa3\x02\xc0E\xa2C\xb0\xa8<\xa3\xf59x\x1b\xde$>7z#\xb7C\xa1\xbb5\xcd%\x8b\xb9B\xba\x87\xdc2\xdeD\xa1\x89\xac\xa3\xb1f/\x05\xda)\xf9\x87\xdcH\x9e9\t6e\xae\x8dD%\xdcP\xd2\v^\x16E\xb1(ȫ\x99\xba\xa2\xc3\xe5ic,\x8d\x99\xd4ɂ[\x021\xd8\xd8:\xa7T\xedQ\x8bS5rƍ\x89Qˡ\x80\x83\xf4\xbb\x14\x0e\u0558\xdf\xc1\xab\xbeG\xe3\x05\x8fc\xd3=ޟvH;S\x02Ep\xc8-\xfahm\xa8\xc8|Ȕ*\x80/\xc1ŀڏ\x13e\xc4B\xad\x9c~\xc1\xe3\x10h\xb8\xa6\xdc\\\xc2\\g\xf9\x8eJ\xe7°\xc5\rZ\xd4~4\xa8Sgb5z\x8cq]\x18\xee(\xa4sl\xbc\x9b\x99=ڽ\xc4\xc3\xec`\xec\x8b\xd4\xdb)\x01>\xcd\x1e4\x8bm\xc5\xec\xbb\xf8\xe7\x82\xc8O\x0f\x9f\x1e\xe6\xb0\x10\x02\x8cߡ%\xadm\x82*\x86֩o\xde\xc7\x1c\xfb\x1e\x82\x14\u007f\xb8\xfb\x16\\L\x93<\xe7\x06lV\xd1\xfa\x8fT\xa8E\xa6\b\xa2UҊ\xb1@\x99\x92\x94]gm\xa6X3f\x88c\x15fwP`\xa2\f2\x16Q_p\x18L_q\xb3\\\xec^\xf1\xb1RHK-$\xa7B\xec\xdc7J\x83!\xce\xea\xed\x11\xc1\xfa\x15\xf8\xa5\x880.x\x12 \xe7\xc3+\x1c?t\xf7\xb6mY\nO9\xc79\xf4T@9\xd0H9\x90\xd9!r1(p\xa35y\xa37\xc0N\xa1\xee\xce\xf5c\xfc\x1b#\xc4:\xf0\x17\x1c\x01~ \xcaǸ\xb1`\x9c\x8e\x11/\xc1a\f\xbe\xd7\u0600\xeb6\xce\xd9\x12\xed-\xbc,\x17\xb4\xf1\x94&\x19,\x17\xb0\x0eZ(,\x1c\x1dv\xa8\xa9C\x90\x9b\xe3\xf8]4\x9e\xeeW\x05\xd5Xa\xe4\x1a\xbf`;.C\x8a\xe1sX\x1fGj\x82\x1b\x84l,n\xe4\xcf7\b\xf9\x187\x16\xc0\x1b\xe6w \xb5\x93\x02\x81\x8d\xc0\x9f\x8a\xb5\v\x82\x9e\xf2\xffC\x8e\"ߠ\x9e\u05fc=\xb1\xf3\x16\x87/\x18_\xf1\x9fǼ\xed\x84B\xf9\x9d#\xffy-xɏG%ڟ\x1e\f\xfe\x94*,>\x92*Ϙy\x1e\x9ex\xa5R+\xcf\x16c\xceLu\x81\xb1\x16]c\xb4\xa0\xe6\xe9\xb6:\xade\xf9\u007fW\xad\x8d\xabuz\x1e\xe5zkE\v7\xb5*\xf1\x89\xe6\xcd\xcdJz\xb8\xea\xb6\x02f\xed\xa8Sl\xfb\x95\x9e\x8c\xbfH\x9b\xf2\xaeӧP?\xac!\xe8X\xa9Ō_\xc1\xdf5|\xa2ޖ\xb2\x93\x98\x13\xdfv\xcc\x00\xa4\x03m\x0et\xbcC/\x92\x00\xa3S\xbe\xa6n\x8di\x91\x9b\xe1\xb8t\x90JQƶX\x9b\xfdhƦBӢ:\x02sd:\xfb\xdfT\x1f\xaaw\xbfZ\x17\xa4\x98\xf3\xd4Ԡ\xf8\x8a{9|\xe5\x19\xa2{?8Q\x1c\xff\xe4\x0e\xf4\xe3\xc7\xd2,\xcfl\xde\xf6\xe3\b\x18\x1b\xa9\xa8\x16\x1c\x89\x13m\xc50|\x8f\xfc\xb8\xba\xbfs\xb1\x84G\xed\xc7ʾ\x03Z\x8c\x1d\x13\n\xaa\xe2M~\x97\bΣ\x1d1\x80\x93\xf6\xa2\xceA\x19\xbd\xed9N\x1a\xf9\x95\x82*\xb4dPƂ@O\xa9Io\x81\xef\x98\xdeb\xfb\n\x95\xf9\u007f\x9dS2\x9f\x9eʹ\x16\"\xf5%\xf3\xb8I\xa3Or\xacL\x1f\xbc\x00\xb7\x9b\xc7_\u007f\v\xf7E\xb3\x17ۜ+\xb8\x0f\xf6\x97,M\xa0N}\xfb\"\u070eooo\x87\xcf\xcd7 \xf1ַ\xf0W\xde5\xe0\xc0\\\xfb*\xfe\xeb\xe1PS\xb5z\xb5\x04\xfe\x92v\xa5\xe7\xc3|\x04\xd8\xda\x04\xff\x9agލ\x19t~\xee\u007f\v\x8f\xf1#Ƶ\"\x83\xf6\x14\x8d\xf0`\xa9\x95l_\xc5bP\x18\xcb-\xb7?/-z\xdfZ\xbak\xc3/17\xc85\x9ak\a\x93)_v\xf4\x9aA\xee΄\xf5饸p\x9e36\xfc\xfb?\x936yS\x86l<\x8a\x1f\xfa\x9f\xdaޥ\x00R\xbe\x97ş\x9c\xaa\x9a\xf4\xad\x10\xfe\xf6\x8fI\xba\x18\xc5s\xf9\xc0E\x93\xff\r\x00\x00\xff\xff\x04\x0e\x95\xf5\xa5\x1c\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4V=s\xe36\x10\xed\xf9+v.\xc55\x11u7)\x92Q\x97\xf8\xae\xf0$\xf1x\xec\x1b7\x99\x14\x10\xb0\x127&\x01dw!\xc7\xf9\xf5\x19\x00\xa4%Q\xf4\xc5)\u008e\xfb\x85\x87\xf7v\x97lV\xabUc\"= \v\x05\xbf\x01\x13\t\xffR\xf4\xf9M\xda\xc7\x1f\xa4\xa5\xb0>|l\x1eɻ\r\\%\xd10ܡ\x84\xc4\x16?\xe1\x8e<)\x05\xdf\f\xa8\xc6\x195\x9b\x06\xc0x\x1f\xd4d\xb3\xe4W\x00\x1b\xbcr\xe8{\xe4\xd5\x1e}\xfb\x98\xb6\xb8M\xd4;\xe4R|:\xfa\xf0\xa1\xfd\xbe\xfd\xd0\x00Xƒ\xfe\x85\x06\x145C܀O}\xdf\x00x3\xe0\x06\x1c\xf6\xa8\xb85\xf61E\xc6?\x13\x8aJ{\xc0\x1e9\xb4\x14\x1a\x89h\xf3\xc1{\x0e)n\xe0\xe8\xa8\xf9#\xa8z\xa1O\xa5\xd4O\xa5\xd4]-U\xbc=\x89\xfe\xfcZ\xc4/4F\xc5>\xb1\xe9\x97\x01\x95\x00!\xbfO\xbd\xe1Ő\x06@l\x88\xb8\x81\x9b\f+\x1a\x8b\xae\x01\x18\xf9(0W\xe3\x8d\x0f\x1fk9\xdb\xe1`*~\x80\x10\xd1\xffx{\xfd\xf0\xdd\xfd\x99\x19\xc0\xa1X\xa6\xa8\x85\xd5\x05\xfc@\x02\x06F\x14\xa0a\x04\a\xc1#\x04\x86!0BE*\xedK\xd1\xc8!\"+M\xfc\xd5\xe7\xa4uN\xac3\b\xef3\xca\x1a\x05.\xf7\f\nh\x87\xd3Mэ\x17\x83\xb0\x03\xedH\x8012\n\xfa\xdaEg\x85!\a\x19\x0fa\xfb\aZm\xe1\x1e9\x97\x01\xe9B\xea]n\xb5\x03\xb2\x02\xa3\r{O\u007f\xbfԖ|\xcf|hot\x12\xf9\xf8\x90Wdoz8\x98>\xe1\xb7`\xbc\x83\xc1<\x03c>\x05\x92?\xa9WB\xa4\x85_3M\xe4wa\x03\x9dj\x94\xcdz\xbd'\x9dFƆaH\x9e\xf4y]\xba\x9f\xb6I\x03\xcb\xda\xe1\x01\xfb\xb5\xd0~e\xd8v\xa4h51\xaeM\xa4U\x81\xee\xcbش\x83\xfb\x86\xc7!\x93\xf7gX\xf597\x8c(\x93ߟ8J7\u007fE\x81\xdc\xcbU\xf6\x9aZoq$:\x9b2;w\x9f\xef\xbf\xc0tt\x11c\xce~\xe1\xfd\x98(G\t2a\xe4w\xc8U\xc4\x1d\x87\xa1\xd4D\xefb \xaf\xe5\xc5\xf6\x84~N\xbf\xa4\xed@*SKf\xadZ\xb8*{\x04\xb6\b):\xa3\xe8Z\xb8\xf6pe\x06쯌\xe0\xff.@fZV\x99طIp\xba\x02\xe7\xc1\x95\xb5\x13Ǵ\xa3^\xd1kah\xef#ڬ`&1gӎl\x19\x0f\xd8\x05\x86\xa7\x8el7\r\xed\x8cݗ\x01o\xcf\x1c\xcb\x03\x9d\x9fZ&/\xa5\xb9\xe7\xd5\xcbCю\x18g]\xb8:)\xf6&^\xd4h\x92\xff\xc8Lə\xb8\xb1\x89\x19\xbd\x8e\x95ʶXJz+\x17\xc8\x1c\xf8\xc2:\x03\xf5\xb9\x04\x95\xef\x9c!/`\xfc\xf3\x98\b\xda\x19\x85'\xe4<\x066\xa4\xbcgЁK\x17\xfc\x8d\xb4tX\xc5\xca\xc2F\x0e\x16Eڋ8R\x1c\x160}E\x9d\xfc\xe4o\xa8\xd9\xf6\xb8\x01儯(k\x98\xcd\xf3\xcc\x17;#\v\xadpF\xc1m\x8eY\xd2\x00\xebV\xc7\u007f\x17\xa1\xd0\xed\xd3py\xd2\nn\xf0i\xc1z\xedo9\xec\x19e\xde\xf2\xd9y[\xd9+\xdf\xd47\xb2\xb4ؔ\x17F\xc9\xfbΝ\xb0(\x1a\xd8\xec'^\x8f-l\xacŨ\xe8n\xe6\u007f\x1d\xefޝ\xfd>\x94W\x1b\xbc\xa3\xfa\xd3\x04\xbf\xfd\xdeԪ\xe8\x1e\xa6\xbf\x81l\xfc'\x00\x00\xff\xff\x8c\xdb\x1fܮ\t\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WM\x8f\xdc6\f\xbd\xfbW\x10\xe9!-\x10{\x12\xf4\xd0bn\xed&\x87\xa0i\x10\xec\xa6{)z\xd0\xc8\x1c[]YREj6ۢ\xff\xbd\xa0dχdz\xbb9T7K\x14\xf5D>>\xd1U]ו\n\xe6\x16#\x19\xef֠\x82\xc1/\x8cN\xbe\xa8\xb9\xfb\x91\x1a\xe3W\xbb7՝q\xed\x1a\xae\x12\xb1\x1f\xae\x91|\x8a\x1a\xdf\xe2\xd68\xc3ƻj@V\xadb\xb5\xae\x00\x94s\x9e\x95L\x93|\x02h\xef8zk1\xd6\x1d\xba\xe6.mp\x93\x8cm1f\xe7\xd3ѻ\xd7\xcd\x0f\xcd\xeb\n@G\xcc\xdb?\x9b\x01\x89\xd5\x10\xd6\xe0\x92\xb5\x15\x80S\x03\xae\xa1\xf5\xf7\xcez\xd5F\xfc+!15;\xb4\x18}c|E\x01\xb5\x1c\xdaE\x9f\xc2\x1a\x0e\ve\xef\b\xa8\\\xe6\xed\xe8溸\xc9+\xd6\x10\xff\xb2\xb4\xfa\xc1\x8c\x16\xc1\xa6\xa8\xec9\x88\xbcH\xc6uɪx\xb6\\\x01\x90\xf6\x01\xd7\xf0Q`\x04\xa5\xb1\xad\x00ƻgX\xf5x\xbbݛ\xe2J\xf78\xa8\x82\x17\xc0\at?}z\u007f\xfb\xfd\xcd\xc94@\x8b\xa4\xa3\t\x9c#8\xc3\f\x86@\xc1\x88\x00\xd8\xefA\x81r\xa0\"\x9b\xad\xd2\f\xdb\xe8\a\xd8(}\x97\xc2\xde+\x80\xdf\xfc\x89\x9a\x81\xd8G\xd5\xe1+\xa0\xa4{P⯘\x82\xf5\x1dl\x8d\xc5f\xbf)D\x1f0\xb2\x99\xa2\\\xc6\x11\xb9\x8efg\xc0_\xca݊\x15\xb4\xc2*$\xe0\x1e\xa7\xf8`;\x86\x03\xfc\x16\xb87\x04\x11CDBWxv\xe2\x18\xc4H\xb9\xf1\x06\r\xdc`\x147@\xbdO\xb6\x152\xee02DԾs\xe6\xef\xbdo\x92\bɡV\xf1D\x87\xc30\x8e1:ea\xa7l\xc2W\xa0\\\v\x83z\x80\x889N\xc9\x1d\xf9\xcb&\xd4\xc0\xaf>\"\x18\xb7\xf5k\xe8\x99\x03\xadW\xab\xce\xf0TT\xda\x0fCr\x86\x1fV\xb9>\xcc&\xb1\x8f\xb4jq\x87vE\xa6\xabUԽaԜ\"\xaeT0u\x86\xeera5C\xfbM\x1cː^\x9e`\xe5\a\xa1\x19q4\xae;ZȜ\u007f$\x03\xc2\xfaB\x98\xb2\xb5\xdc\xe2\x10h\x99\x92\xe8\\\xbf\xbb\xf9\f\xd3\xd19\x19\xf3\xe8\x17\xe6\xec7\xd2!\x05\x120\xe3\xb6\x18K\x123\xf3\xc4'\xba6x\xe38\u007fhk\xd0\xcd\xc3Oi3\x18\xa6\x89̒\xab\x06\xae\xb2\xd2\xc0\x06!\x85V1\xb6\r\xbcwp\xa5\x06\xb4W\x8a\xf0\u007fO\x80D\x9aj\t\xec\xf3Rp,\x92s\xe3\x12\xb5\xa3\x85I\xc9.\xe4kV\xea7\x01\xb5dO\x02(;\xcd\xd6\xe8\\\x1a\xb0\xf5\x11ԡ\xf2\xc7\x006'\x9e\x97+7\x83S\xb1C\x9e\xcfΰ|\xceFr\xfc}\xafN\x85\xe6[l\xbaF\xb4\x82F E=\xbek\xce<^\xc6\x00\x8b\xec]D2\x91X\xc2 q\x15)\x10\x91:\xc6t~\xb4\ftiX>\xa0\x86\x9f3\xe6\x0f\xbe{t\xfd\xca;\x16\xba?jt\xebm\x1a\xf0Ʃ@\xbd\u007f\xc2\xf6=\xe3\xf0<\xcb\xe9A\xde?R\xe7\x86\xd7(R\x8e\x97/1\x1a\\#%{\xe1\xb8\v\xb4\x9eF~\xbe\x9eΑ<\x80S\x8edK\xd1t\x04i\v\xa2CF:\xc8˽\xe1~\xd1#\xc0}ot\x9f7\xe6\x04\x8br\x11ym\xb2\x0e|=|\xa9\v\x13q\x81du&\xdf´\x80?\x9b\xbeP͗\x0e\xa8\xc7\n{\x96\"\xb0\xe2D_\xa1\t\xd9~\n\xb5N1\xa2\xe3\xd1K~#\xe7\x1b\x9e+\nS%\xfdv\xfd\xe1\tex{\xb0\xcc]\xa02\xae\xa0\t\x11k2\x9d\xbc\xec\xb2&ڐk\xf6<\x18e\x9cv\x1a\xa7\x81Z\xcc(~\t&f\x05|\x02\u2efda\x110t\xe5q\x9a\xf7R\xd9!R~\xf8\xb5\x9a\xb7\x1c26\b-Zdla\xf3P\x94\xf8\x81\x18\x87s\xdc[\x1f\a\xc5k\x90G\xabf\xb3@#\xe9w\xd5\xc6\xe2\x1a8\xa6K,[\xbcx\xe8\x15-\x94\xe1ɝ?\x89\xcd\x121\xf6\xc5\xf8(3\xe0\xa2^\xd6\xf0\x11\xef\x17f?E\xaf\x91\b\xcf\xcb\xe8\xe2M\x16\x8b\xe0l\x92\xa4\xb3h\x8f\xa246\xac\xc73i\xb3\xef\x94&\xc4c)\xc1?\xffV\x87\xaaRZc`l?\xce\u007f\x14^\xbc8\xe9\xfc\xf3\xa7\xf6\xae5\xe5\x1f\a~\xff\xa3*\ac{;5\xf42\xf9_\x00\x00\x00\xff\xff\xcbT\xc3P]\r\x00\x00"), diff --git a/pkg/apis/velero/v1/backup.go b/pkg/apis/velero/v1/backup.go index 15c9eddfc7..51f5f6ea3a 100644 --- a/pkg/apis/velero/v1/backup.go +++ b/pkg/apis/velero/v1/backup.go @@ -292,6 +292,10 @@ type BackupStatus struct { // +optional VolumeSnapshotsCompleted int `json:"volumeSnapshotsCompleted,omitempty"` + // FailureReason is an error that caused the entire backup to fail. + // +optional + FailureReason string `json:"failureReason,omitempty"` + // Warnings is a count of all warning messages that were generated during // execution of the backup. The actual warnings are in the backup's log // file in object storage. diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 7402f7e455..0a4310e116 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -147,13 +147,12 @@ func NewBackupController( backup := obj.(*velerov1api.Backup) switch backup.Status.Phase { - case "", velerov1api.BackupPhaseNew: - // only process new backups + case "", velerov1api.BackupPhaseNew, velerov1api.BackupPhaseInProgress: default: c.logger.WithFields(logrus.Fields{ "backup": kubeutil.NamespaceAndName(backup), "phase": backup.Status.Phase, - }).Debug("Backup is not new, skipping") + }).Debug("Backup is not new or in-progress, skipping") return } @@ -241,7 +240,22 @@ func (c *backupController) processBackup(key string) error { // this key (even though it was a no-op). switch original.Status.Phase { case "", velerov1api.BackupPhaseNew: - // only process new backups + case velerov1api.BackupPhaseInProgress: + // A backup may stay in-progress forever because of + // 1) the controller restarts during the processing of a backup + // 2) the backup with in-progress status isn't updated to completed or failed status successfully + // So we try to mark such Backups as failed to avoid it + updated := original.DeepCopy() + updated.Status.Phase = velerov1api.BackupPhaseFailed + updated.Status.FailureReason = fmt.Sprintf("got a Backup with unexpected status %q, this may be due to a restart of the controller during the backing up, mark it as %q", + velerov1api.BackupPhaseInProgress, updated.Status.Phase) + updated.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} + _, err = patchBackup(original, updated, c.client) + if err != nil { + return errors.Wrapf(err, "error updating Backup status to %s", updated.Status.Phase) + } + log.Warn(updated.Status.FailureReason) + return nil default: return nil } @@ -261,6 +275,7 @@ func (c *backupController) processBackup(key string) error { if err != nil { return errors.Wrapf(err, "error updating Backup status to %s", request.Status.Phase) } + // store ref to just-updated item for creating patch original = updatedBackup request.Backup = updatedBackup.DeepCopy() @@ -287,6 +302,7 @@ func (c *backupController) processBackup(key string) error { // result in the backup being Failed. log.WithError(err).Error("backup failed") request.Status.Phase = velerov1api.BackupPhaseFailed + request.Status.FailureReason = err.Error() } switch request.Status.Phase { diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 85c581a81a..fffa831486 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -94,11 +94,6 @@ func TestProcessBackupNonProcessedItems(t *testing.T) { key: "velero/backup-1", backup: defaultBackup().Phase(velerov1api.BackupPhaseFailedValidation).Result(), }, - { - name: "InProgress backup is not processed", - key: "velero/backup-1", - backup: defaultBackup().Phase(velerov1api.BackupPhaseInProgress).Result(), - }, { name: "Completed backup is not processed", key: "velero/backup-1", @@ -140,6 +135,28 @@ func TestProcessBackupNonProcessedItems(t *testing.T) { } } +func TestMarkInProgressBackupAsFailed(t *testing.T) { + backup := defaultBackup().Phase(velerov1api.BackupPhaseInProgress).Result() + clientset := fake.NewSimpleClientset(backup) + sharedInformers := informers.NewSharedInformerFactory(clientset, 0) + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatText) + + c := &backupController{ + genericController: newGenericController("backup-test", logger), + client: clientset.VeleroV1(), + lister: sharedInformers.Velero().V1().Backups().Lister(), + clock: &clock.RealClock{}, + } + require.NoError(t, sharedInformers.Velero().V1().Backups().Informer().GetStore().Add(backup)) + + err := c.processBackup(fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)) + require.Nil(t, err) + + res, err := clientset.VeleroV1().Backups(backup.Namespace).Get(context.TODO(), backup.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, velerov1api.BackupPhaseFailed, res.Status.Phase) +} + func TestProcessBackupValidationFailures(t *testing.T) { defaultBackupLocation := builder.ForBackupStorageLocation("velero", "loc-1").Result() @@ -729,6 +746,7 @@ func TestProcessBackupCompletions(t *testing.T) { }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, + FailureReason: "backup already exists in object storage", Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, @@ -766,6 +784,7 @@ func TestProcessBackupCompletions(t *testing.T) { }, Status: velerov1api.BackupStatus{ Phase: velerov1api.BackupPhaseFailed, + FailureReason: "error checking if backup already exists in object storage: Backup already exists in object storage", Version: 1, FormatVersion: "1.1.0", StartTimestamp: ×tamp, diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 6a79ead15c..4d01b36dfa 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -144,13 +144,12 @@ func NewRestoreController( restore := obj.(*api.Restore) switch restore.Status.Phase { - case "", api.RestorePhaseNew: - // only process new restores + case "", api.RestorePhaseNew, api.RestorePhaseInProgress: default: c.logger.WithFields(logrus.Fields{ "restore": kubeutil.NamespaceAndName(restore), "phase": restore.Status.Phase, - }).Debug("Restore is not new, skipping") + }).Debug("Restore is not new or in-progress, skipping") return } @@ -202,7 +201,21 @@ func (c *restoreController) processQueueItem(key string) error { // is ("" | New) switch restore.Status.Phase { case "", api.RestorePhaseNew: - // only process new restores + case api.RestorePhaseInProgress: + // A restore may stay in-progress forever because of + // 1) the controller restarts during the processing of a restore + // 2) the restore with in-progress status isn't updated to completed or failed status successfully + // So we try to mark such restores as failed to avoid it + updated := restore.DeepCopy() + updated.Status.Phase = api.RestorePhaseFailed + updated.Status.FailureReason = fmt.Sprintf("got a Restore with unexpected status %q, this may be due to a restart of the controller during the restore, mark it as %q", + api.RestorePhaseInProgress, updated.Status.Phase) + _, err = patchRestore(restore, updated, c.restoreClient) + if err != nil { + return errors.Wrapf(err, "error updating Restore status to %s", updated.Status.Phase) + } + log.Warn(updated.Status.FailureReason) + return nil default: return nil } diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index 1641b4f850..1e37f2f567 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io/ioutil" "testing" "time" @@ -170,11 +171,6 @@ func TestProcessQueueItemSkips(t *testing.T) { restoreKey: "foo/bar", expectError: true, }, - { - name: "restore with phase InProgress does not get processed", - restoreKey: "foo/bar", - restore: builder.ForRestore("foo", "bar").Phase(velerov1api.RestorePhaseInProgress).Result(), - }, { name: "restore with phase Completed does not get processed", restoreKey: "foo/bar", @@ -226,6 +222,31 @@ func TestProcessQueueItemSkips(t *testing.T) { } } +func TestMarkInProgressRestoreAsFailed(t *testing.T) { + var ( + restore = builder.ForRestore("velero", "bar").Phase(velerov1api.RestorePhaseInProgress).Result() + client = fake.NewSimpleClientset(restore) + sharedInformers = informers.NewSharedInformerFactory(client, 0) + logger = velerotest.NewLogger() + ) + + c := restoreController{ + genericController: newGenericController("restore-test", logger), + restoreClient: client.VeleroV1(), + restoreLister: sharedInformers.Velero().V1().Restores().Lister(), + } + + err := sharedInformers.Velero().V1().Restores().Informer().GetStore().Add(restore) + require.Nil(t, err) + + err = c.processQueueItem(fmt.Sprintf("%s/%s", restore.Namespace, restore.Name)) + require.Nil(t, err) + + res, err := c.restoreClient.Restores(restore.Namespace).Get(context.Background(), restore.Name, metav1.GetOptions{}) + require.Nil(t, err) + assert.Equal(t, velerov1api.RestorePhaseFailed, res.Status.Phase) +} + func TestProcessQueueItem(t *testing.T) { defaultStorageLocation := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result() diff --git a/site/content/docs/main/api-types/backup.md b/site/content/docs/main/api-types/backup.md index 5620c9ff69..3955bba8cd 100644 --- a/site/content/docs/main/api-types/backup.md +++ b/site/content/docs/main/api-types/backup.md @@ -144,5 +144,7 @@ status: warnings: 2 # Number of errors that were logged by the backup. errors: 0 + # An error that caused the entire backup to fail. + failureReason: "" ```