Skip to content

Commit

Permalink
Save selfie to backend (#1209)
Browse files Browse the repository at this point in the history
Saves the selfie uploaded file IDs and metadata
  • Loading branch information
mludowise-stripe authored Jun 15, 2022
1 parent eb524bc commit 12af6ba
Show file tree
Hide file tree
Showing 18 changed files with 251 additions and 78 deletions.
35 changes: 14 additions & 21 deletions StripeCameraCore/StripeCameraCore/Source/CameraExifMetadata.swift
Original file line number Diff line number Diff line change
Expand Up @@ -10,33 +10,26 @@ import ImageIO
import CoreMedia

/// A helper to extract properties from an EXIF metadata dictionary
@_spi(STP) public struct CameraExifMetadata {
public let exifDictionary: [CFString: Any]

// MARK: - Init
@_spi(STP) public struct CameraExifMetadata: Equatable {
public let brightnessValue: Double?
public let focalLength: Double?
public let lensModel: String?
}

public init?(exifDictionary: [CFString: Any]?) {
public extension CameraExifMetadata {
init?(exifDictionary: [CFString: Any]?) {
guard let exifDictionary = exifDictionary else {
return nil
}
self.exifDictionary = exifDictionary
}

public init?(sampleBuffer: CMSampleBuffer) {
self.init(exifDictionary: CMGetAttachment(sampleBuffer, key: kCGImagePropertyExifDictionary, attachmentModeOut: nil) as? [CFString: Any])
}

// MARK: - Computed Properties

public var brightnessValue: Double? {
return exifDictionary[kCGImagePropertyExifBrightnessValue] as? Double
self.init(
brightnessValue: exifDictionary[kCGImagePropertyExifBrightnessValue] as? Double,
focalLength: exifDictionary[kCGImagePropertyExifFocalLength] as? Double,
lensModel: exifDictionary[kCGImagePropertyExifLensModel] as? String
)
}

public var lensModel: String? {
return exifDictionary[kCGImagePropertyExifLensModel] as? String
}

public var focalLength: Double? {
return exifDictionary[kCGImagePropertyExifFocalLength] as? Double
init?(sampleBuffer: CMSampleBuffer) {
self.init(exifDictionary: CMGetAttachment(sampleBuffer, key: kCGImagePropertyExifDictionary, attachmentModeOut: nil) as? [CFString: Any])
}
}
6 changes: 5 additions & 1 deletion StripeIdentity/StripeIdentity.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
E61676FF2850023100C9E44A /* IdentityAnalytic.swift in Sources */ = {isa = PBXBuildFile; fileRef = E61676FE2850023100C9E44A /* IdentityAnalytic.swift */; };
E616770228500F3400C9E44A /* VerificationSheetController+Analytics.swift in Sources */ = {isa = PBXBuildFile; fileRef = E616770128500F3400C9E44A /* VerificationSheetController+Analytics.swift */; };
E61676F9284FFBB500C9E44A /* TimeInterval+StripeIdentity.swift in Sources */ = {isa = PBXBuildFile; fileRef = E61676F8284FFBB500C9E44A /* TimeInterval+StripeIdentity.swift */; };
E616770A2853F6A700C9E44A /* VerificationPageDataFace.swift in Sources */ = {isa = PBXBuildFile; fileRef = E61677092853F6A700C9E44A /* VerificationPageDataFace.swift */; };
E616770228500F3400C9E44A /* IdentityAnalyticsClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = E616770128500F3400C9E44A /* IdentityAnalyticsClient.swift */; };
E61ADAD3270F6293004ED998 /* VerificationSheetController.swift in Sources */ = {isa = PBXBuildFile; fileRef = E61ADAD2270F6293004ED998 /* VerificationSheetController.swift */; };
E61C32462797AEA2008A30D4 /* DocumentFileUploadViewControllerTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = E61C32452797AEA2008A30D4 /* DocumentFileUploadViewControllerTest.swift */; };
Expand Down Expand Up @@ -240,6 +241,7 @@
E616770128500F3400C9E44A /* VerificationSheetController+Analytics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "VerificationSheetController+Analytics.swift"; sourceTree = "<group>"; };
E61676F8284FFBB500C9E44A /* TimeInterval+StripeIdentity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "TimeInterval+StripeIdentity.swift"; sourceTree = "<group>"; };
E616770128500F3400C9E44A /* IdentityAnalyticsClient.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IdentityAnalyticsClient.swift; sourceTree = "<group>"; };
E61677092853F6A700C9E44A /* VerificationPageDataFace.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VerificationPageDataFace.swift; sourceTree = "<group>"; };
E61ADAD2270F6293004ED998 /* VerificationSheetController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VerificationSheetController.swift; sourceTree = "<group>"; };
E61C32452797AEA2008A30D4 /* DocumentFileUploadViewControllerTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DocumentFileUploadViewControllerTest.swift; sourceTree = "<group>"; };
E61EA92926A0D89900CAEE52 /* FBSnapshotTestCase.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = FBSnapshotTestCase.xcframework; path = ../Carthage/Build/FBSnapshotTestCase.xcframework; sourceTree = "<group>"; };
Expand Down Expand Up @@ -739,9 +741,10 @@
E6548F5D2731E4D500F399B2 /* VerificationPageDataUpdate */ = {
isa = PBXGroup;
children = (
E6B906ED27CAB4F200D0A703 /* VerificationPageCollectedData.swift */,
E6B9071127D1C95B00D0A703 /* VerificationPageClearData.swift */,
E6B906ED27CAB4F200D0A703 /* VerificationPageCollectedData.swift */,
E657B57F276416FD00134033 /* VerificationPageDataDocumentFileData.swift */,
E61677092853F6A700C9E44A /* VerificationPageDataFace.swift */,
E6548F5E2731E4E500F399B2 /* VerificationPageDataUpdate.swift */,
);
path = VerificationPageDataUpdate;
Expand Down Expand Up @@ -1366,6 +1369,7 @@
E6A50DC027B785B800D7BDED /* HTMLViewWithIconLabels.swift in Sources */,
E6548F522731D9B400F399B2 /* VerificationPageDataRequirements.swift in Sources */,
E61676F9284FFBB500C9E44A /* TimeInterval+StripeIdentity.swift in Sources */,
E616770A2853F6A700C9E44A /* VerificationPageDataFace.swift in Sources */,
E657B580276416FD00134033 /* VerificationPageDataDocumentFileData.swift in Sources */,
E61676E62849894600C9E44A /* VerificationPageStaticContentSelfieModels.swift in Sources */,
E6C7BB2527A8BE2E000807A6 /* MLModelUnexpectedOutputError.swift in Sources */,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,28 @@ final class IdentityAPIClientImpl: IdentityAPIClient {
func updateIdentityVerificationPageData(
updating verificationData: StripeAPI.VerificationPageDataUpdate
) -> Promise<StripeAPI.VerificationPageData> {
// TODO(mludowise|IDPROD-4030): Remove API v1 check when selfie is production ready
guard apiVersion > 1 else {
// Translate into v1 API models to avoid API error
return apiClient.post(
resource: APIEndpointVerificationPageData(id: verificationSessionId),
object: StripeAPI.VerificationPageDataUpdateV1(
clearData: .init(
biometricConsent: verificationData.clearData?.biometricConsent,
idDocumentBack: verificationData.clearData?.idDocumentBack,
idDocumentFront: verificationData.clearData?.idDocumentFront,
idDocumentType: verificationData.clearData?.idDocumentType
),
collectedData: .init(
biometricConsent: verificationData.collectedData?.biometricConsent,
idDocumentBack: verificationData.collectedData?.idDocumentBack,
idDocumentFront: verificationData.collectedData?.idDocumentFront,
idDocumentType: verificationData.collectedData?.idDocumentType
)
)
)
}

return apiClient.post(
resource: APIEndpointVerificationPageData(id: verificationSessionId),
object: verificationData
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,16 @@ import Foundation

extension StripeAPI {
struct VerificationPageClearData: Encodable, Equatable {
let biometricConsent: Bool?
let face: Bool?
let idDocumentBack: Bool?
let idDocumentFront: Bool?
let idDocumentType: Bool?
}

// TODO(mludowise|IDPROD-4030): Remove v1 API models when selfie is production ready
/// API model compatible with V1 Identity endpoints that won't encode a `face` property
struct VerificationPageClearDataV1: Encodable, Equatable {
let biometricConsent: Bool?
let idDocumentBack: Bool?
let idDocumentFront: Bool?
Expand All @@ -21,6 +31,7 @@ extension StripeAPI.VerificationPageClearData {
init(clearFields fields: Set<StripeAPI.VerificationPageFieldType>) {
self.init(
biometricConsent: fields.contains(.biometricConsent),
face: fields.contains(.face),
idDocumentBack: fields.contains(.idDocumentBack),
idDocumentFront: fields.contains(.idDocumentFront),
idDocumentType: fields.contains(.idDocumentType)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,34 @@ extension StripeAPI {
struct VerificationPageCollectedData: Encodable, Equatable {

let biometricConsent: Bool?
let face: VerificationPageDataFace?
let idDocumentBack: VerificationPageDataDocumentFileData?
let idDocumentFront: VerificationPageDataDocumentFileData?
let idDocumentType: DocumentType?

init(
biometricConsent: Bool? = nil,
face: VerificationPageDataFace? = nil,
idDocumentBack: VerificationPageDataDocumentFileData? = nil,
idDocumentFront: VerificationPageDataDocumentFileData? = nil,
idDocumentType: DocumentType? = nil
) {
self.biometricConsent = biometricConsent
self.face = face
self.idDocumentBack = idDocumentBack
self.idDocumentFront = idDocumentFront
self.idDocumentType = idDocumentType
}
}

// TODO(mludowise|IDPROD-4030): Remove v1 API models when selfie is production ready
/// API model compatible with V1 Identity endpoints that won't encode a `face` property
struct VerificationPageCollectedDataV1: Encodable, Equatable {
let biometricConsent: Bool?
let idDocumentBack: VerificationPageDataDocumentFileData?
let idDocumentFront: VerificationPageDataDocumentFileData?
let idDocumentType: DocumentType?
}
}

extension StripeAPI.VerificationPageCollectedData {
Expand All @@ -38,6 +50,7 @@ extension StripeAPI.VerificationPageCollectedData {
func merging(_ otherData: StripeAPI.VerificationPageCollectedData) -> StripeAPI.VerificationPageCollectedData {
return StripeAPI.VerificationPageCollectedData(
biometricConsent: otherData.biometricConsent ?? self.biometricConsent,
face: otherData.face ?? self.face,
idDocumentBack: otherData.idDocumentBack ?? self.idDocumentBack,
idDocumentFront: otherData.idDocumentFront ?? self.idDocumentFront,
idDocumentType: otherData.idDocumentType ?? self.idDocumentType
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
//
// VerificationPageDataFace.swift
// StripeIdentity
//
// Created by Mel Ludowise on 6/10/22.
//

import Foundation
@_spi(STP) import StripeCore

extension StripeAPI {
struct VerificationPageDataFace: Encodable, Equatable {

/// File ID of uploaded image for best selfie frame. This will be cropped to the bounds of the face in the image.
let bestHighResImage: String
/// File ID of uploaded image for best selfie frame. This will be un-cropped.
let bestLowResImage: String
/// File ID of uploaded image for first selfie frame. This will be cropped to the bounds of the face in the image.
let firstHighResImage: String
/// File ID of uploaded image for first selfie frame. This will be un-cropped.
let firstLowResImage: String
/// File ID of uploaded image for last selfie frame. This will be cropped to the bounds of the face in the image.
let lastHighResImage: String
/// File ID of uploaded image for last selfie frame. This will be un-cropped.
let lastLowResImage: String
/// FaceDetector score for the best selfie frame.
let bestFaceScore: TwoDecimalFloat
/// Variance of the FaceDetector scores over all selfie frames.
let faceScoreVariance: TwoDecimalFloat
/// The total number of selfie frames taken.
let numFrames: Int
/// Camera brightness value for the best selfie frame.
let bestBrightnessValue: TwoDecimalFloat?
/// Camera lens model for the best selfie frame.
let bestCameraLensModel: String?
/// Camera exposure duration for the best selfie frame.
let bestExposureDuration: Int?
/// Camera exposure ISO for the best selfie frame
let bestExposureIso: TwoDecimalFloat?
/// Camera focal length for the best selfie frame.
let bestFocalLength: TwoDecimalFloat?
/// If the best selfie frame was taken by a virtual camera.
let bestIsVirtualCamera: Bool?
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,13 @@ extension StripeAPI {
let clearData: VerificationPageClearData?
let collectedData: VerificationPageCollectedData?
}

// TODO(mludowise|IDPROD-4030): Remove v1 API models when selfie is production ready
/// API model compatible with V1 Identity endpoints that won't encode a `face` property
struct VerificationPageDataUpdateV1: Encodable, Equatable {

let clearData: VerificationPageClearDataV1?
let collectedData: VerificationPageCollectedDataV1?
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import Foundation
import UIKit
@_spi(STP) import StripeCore
@_spi(STP) import StripeCameraCore

extension IdentityImageUploader.Configuration {
Expand All @@ -21,3 +22,39 @@ extension IdentityImageUploader.Configuration {
)
}
}

extension StripeAPI.VerificationPageDataFace {
init(
uploadedFiles: SelfieUploader.FileData,
capturedImages: FaceCaptureData,
bestFrameExifMetadata: CameraExifMetadata?,
trainingConsent: Bool?
) {
// TODO(mludowise|IDPROD-4088): Save training consent when API is updated
self.init(
bestHighResImage: uploadedFiles.bestHighResFile.id,
bestLowResImage: uploadedFiles.bestLowResFile.id,
firstHighResImage: uploadedFiles.firstHighResFile.id,
firstLowResImage: uploadedFiles.firstLowResFile.id,
lastHighResImage: uploadedFiles.lastHighResFile.id,
lastLowResImage: uploadedFiles.lastLowResFile.id,
bestFaceScore: .init(capturedImages.bestMiddle.scannerOutput.faceScore),
faceScoreVariance: .init(capturedImages.faceScoreVariance),
numFrames: capturedImages.numSamples,
bestBrightnessValue: bestFrameExifMetadata?.brightnessValue.map {
TwoDecimalFloat(double: $0)
},
bestCameraLensModel: bestFrameExifMetadata?.lensModel,
bestExposureDuration: capturedImages.bestMiddle.scannerOutput.cameraProperties.map {
Int($0.exposureDuration.seconds * 1000)
},
bestExposureIso: capturedImages.bestMiddle.scannerOutput.cameraProperties.map {
TwoDecimalFloat($0.exposureISO)
},
bestFocalLength: bestFrameExifMetadata?.focalLength.map {
TwoDecimalFloat(double: $0)
},
bestIsVirtualCamera: capturedImages.bestMiddle.scannerOutput.cameraProperties?.isVirtualDevice
)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,20 @@

import Foundation
import CoreGraphics
@_spi(STP) import StripeCameraCore

struct FaceScannerInputOutput: Equatable {
let image: CGImage
let scannerOutput: FaceScannerOutput
let cameraExifMetadata: CameraExifMetadata?
}

struct FaceCaptureData: Equatable {
let first: FaceScannerInputOutput
let last: FaceScannerInputOutput
let bestMiddle: FaceScannerInputOutput

let numSamples: Int
let faceScoreVariance: Float

var toArray: [FaceScannerInputOutput] {
Expand All @@ -38,6 +41,7 @@ extension FaceCaptureData {
first: first,
last: last,
bestMiddle: bestMiddle,
numSamples: samples.count,
faceScoreVariance: samples.standardDeviation(with: { $0.scannerOutput.faceScore })
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -343,4 +343,8 @@ extension ImageScanningSession where ExpectedClassificationType == EmptyClassifi
func startTimeoutTimer() {
startTimeoutTimer(expectedClassification: .empty)
}

func reset() {
reset(to: .empty)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -175,20 +175,39 @@ final class DocumentUploader: DocumentUploaderProtocol {
method: StripeAPI.VerificationPageDataDocumentFileData.FileUploadMethod,
fileNamePrefix: String
) -> Future<StripeAPI.VerificationPageDataDocumentFileData> {
return imageUploader.uploadLowAndHighResImages(
originalImage,
highResRegionOfInterest: documentScannerOutput?.idDetectorOutput.documentBounds,
cropPaddingComputationMethod: .maxImageWidthOrHeight,
lowResFileName: "\(fileNamePrefix)_full_frame",
highResFileName: fileNamePrefix
).chained { (lowResFile, highResFile) in
return Promise(value: StripeAPI.VerificationPageDataDocumentFileData(
documentScannerOutput: documentScannerOutput,
highResImage: highResFile.id,
lowResImage: lowResFile?.id,
exifMetadata: exifMetadata,
uploadMethod: method
))

// Only upload a low res image if the high res image will be cropped
if let documentBounds = documentScannerOutput?.idDetectorOutput.documentBounds {
return imageUploader.uploadLowAndHighResImages(
originalImage,
highResRegionOfInterest: documentBounds,
cropPaddingComputationMethod: .maxImageWidthOrHeight,
lowResFileName: "\(fileNamePrefix)_full_frame",
highResFileName: fileNamePrefix
).chained { (lowResFile, highResFile) in
return Promise(value: StripeAPI.VerificationPageDataDocumentFileData(
documentScannerOutput: documentScannerOutput,
highResImage: highResFile.id,
lowResImage: lowResFile.id,
exifMetadata: exifMetadata,
uploadMethod: method
))
}
} else {
return imageUploader.uploadHighResImage(
originalImage,
regionOfInterest: nil,
cropPaddingComputationMethod: .maxImageWidthOrHeight,
fileName: fileNamePrefix
).chained { highResFile in
return Promise(value: StripeAPI.VerificationPageDataDocumentFileData(
documentScannerOutput: documentScannerOutput,
highResImage: highResFile.id,
lowResImage: nil,
exifMetadata: exifMetadata,
uploadMethod: method
))
}
}
}

Expand Down
Loading

0 comments on commit 12af6ba

Please sign in to comment.