restore: truncate range ts (pingcap#29)

* truncate range ts Signed-off-by: 5kbpers <[email protected]> * fix downlaod request range Signed-off-by: 5kbpers <[email protected]>
YuJuncen · Nov 4, 2019 · 11db863 · 11db863
1 parent dd5b880
commit 11db863
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 8 deletions.
diff --git a/pkg/restore/import.go b/pkg/restore/import.go
@@ -94,8 +94,8 @@ func (importer *FileImporter) Import(file *backup.File, rewriteRules *restore_ut
 				continue
 			}
 			if isEmpty {
-				log.Warn(
-					"file don't have key in this region, skip it",
+				log.Info(
+					"file don't have any key in this region, skip it",
 					zap.Reflect("file", file),
 					zap.Reflect("region", regionInfo.Region),
 					zap.Reflect("tableRewriteRules", rules(rewriteRules.Table)),
@@ -108,9 +108,8 @@ func (importer *FileImporter) Import(file *backup.File, rewriteRules *restore_ut
 				if err != nil {
 					log.Warn("ingest file failed",
 						zap.Reflect("file", file),
+						zap.Reflect("range", fileMeta.GetRange()),
 						zap.Reflect("region", regionInfo.Region),
-						zap.Reflect("tableRewriteRules", rules(rewriteRules.Table)),
-						zap.Reflect("dataRewriteRules", rules(rewriteRules.Data)),
 						zap.Error(err),
 					)
 					return err
@@ -162,7 +161,7 @@ func (importer *FileImporter) downloadSST(regionInfo *restore_util.RegionInfo, f
 	if regionRule == nil {
 		return nil, true, errRewriteRuleNotFound
 	}
-	sstMeta := getSSTMetaFromFile(id, file, regionRule)
+	sstMeta := getSSTMetaFromFile(id, file, regionInfo.Region, regionRule)
 	sstMeta.RegionId = regionInfo.Region.GetId()
 	sstMeta.RegionEpoch = regionInfo.Region.GetRegionEpoch()
 	req := &import_sstpb.DownloadRequest{
@@ -182,10 +181,11 @@ func (importer *FileImporter) downloadSST(regionInfo *restore_util.RegionInfo, f
 			return nil, true, errors.Trace(err)
 		}
 		if resp.GetIsEmpty() {
-			return nil, true, nil
+			return &sstMeta, true, nil
 		}
-		sstMeta.Range = &resp.Range
 	}
+	sstMeta.Range.Start = truncateTS(resp.Range.GetStart())
+	sstMeta.Range.End = truncateTS(resp.Range.GetEnd())
 	return &sstMeta, false, nil
 }
 

diff --git a/pkg/restore/util.go b/pkg/restore/util.go
@@ -77,7 +77,7 @@ func GetRewriteRules(newTable *model.TableInfo, oldTable *model.TableInfo) *rest
 
 // getSSTMetaFromFile compares the keys in file, region and rewrite rules, then returns a sst meta.
 // The range of the returned sst meta is [regionRule.NewKeyPrefix, append(regionRule.NewKeyPrefix, 0xff)]
-func getSSTMetaFromFile(id []byte, file *backup.File, regionRule *import_sstpb.RewriteRule) import_sstpb.SSTMeta {
+func getSSTMetaFromFile(id []byte, file *backup.File, region *metapb.Region, regionRule *import_sstpb.RewriteRule) import_sstpb.SSTMeta {
 	// Get the column family of the file by the file name.
 	var cfName string
 	if strings.Contains(file.GetName(), "default") {
@@ -88,7 +88,15 @@ func getSSTMetaFromFile(id []byte, file *backup.File, regionRule *import_sstpb.R
 	// Find the overlapped part between the file and the region.
 	// Here we rewrites the keys to compare with the keys of the region.
 	rangeStart := regionRule.GetNewKeyPrefix()
+	//  rangeStart = max(rangeStart, region.StartKey)
+	if bytes.Compare(rangeStart, region.GetStartKey()) < 0 {
+		rangeStart = region.GetStartKey()
+	}
 	rangeEnd := append(append([]byte{}, regionRule.GetNewKeyPrefix()...), 0xff)
+	// rangeEnd = min(rangeEnd, region.EndKey)
+	if bytes.Compare(rangeEnd, region.GetEndKey()) > 0 {
+		rangeEnd = region.GetEndKey()
+	}
 	return import_sstpb.SSTMeta{
 		Uuid:   id,
 		CfName: cfName,
@@ -202,6 +210,10 @@ func rewriteRawKeyWithNewPrefix(key []byte, rewriteRules *restore_util.RewriteRu
 	return []byte("")
 }
 
+func truncateTS(key []byte) []byte {
+	return key[:len(key)-8]
+}
+
 // SplitRegion splits region by
 // 1. data range after rewrite
 // 2. rewrite rules