diff --git a/.helm/templates/crd-microsoft-synapse.yaml b/.helm/templates/crd-microsoft-synapse.yaml new file mode 100644 index 0000000..c64964c --- /dev/null +++ b/.helm/templates/crd-microsoft-synapse.yaml @@ -0,0 +1,284 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: microsoft-synapse.streaming.sneaksanddata.com +spec: + group: streaming.sneaksanddata.com + scope: Namespaced + names: + plural: microsoft-synapse-streams + singular: microsoft-synapse-streams + kind: MicrosoftSynapseStream + shortNames: + - mssynapsestream + versions: + - name: v1beta1 + served: true + storage: true + additionalPrinterColumns: + - name: Source location + type: string + jsonPath: .spec.baseLocation + - name: Entity + type: string + jsonPath: .spec.entityName + - name: Refresh Interval + type: string + jsonPath: .spec.changeCaptureIntervalSeconds + - name: Sink location + type: string + jsonPath: .spec.sinkLocation + - name: Phase + type: string + jsonPath: .status.phase + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + connectionStringRef: + description: | + Name of the secret containing the connection string. + The secret should have a key named 'ARCANE_CONNECTIONSTRING'. + type: object + properties: + name: + type: string + catalogAuthSecretRef: + description: | + Name of the secret containing the catalog authentication. + The secret should have the following mandatory keys: + - ARCANE_FRAMEWORK__S3_CATALOG_AUTH_CLIENT_URI + - ARCANE_FRAMEWORK__S3_CATALOG_AUTH_SCOPE + And the authentication information in the following keys: + - ARCANE_FRAMEWORK__S3_CATALOG_AUTH_CLIENT_ID + - ARCANE_FRAMEWORK__S3_CATALOG_AUTH_CLIENT_SECRET + OR + - ARCANE_FRAMEWORK__S3_CATALOG_AUTH_INIT_TOKEN + + The authentication information is mutually exclusive. + If init token is provided, client id and secret are not used. + type: object + properties: + name: + type: string + catalog: + type: object + properties: + namespace: + type: string + warehouse: + type: string + authSecretRef: + description: | + Not used in the operator version 0.0.x + In operator version 0.1.x and above this field will replace the `catalogAuthSecretRef` field. + type: object + properties: + name: + type: string + optional: true + default: null # for backward compatibility + mergeService: + type: object + properties: + uri: + type: string + authSecretRef: + description: | + Not used in the operator version 0.0.x + In operator version 0.1.x and above this field will replace the `mergeAuthSecretRef` field. + type: object + properties: + name: + type: string + optional: true + default: null # for backward compatibility + mergeAuthSecretRef: + description: | + Name of the secret containing the data storage authentication for the merge service. + The secret should have a key named 'ARCANE_FRAMEWORK__MERGE_SERVICE__JDBC_URL'. + type: object + properties: + name: + type: string + optional: true + default: null # for backward compatibility + stagingStorageAuthSecretRef: + description: | + Name of the secret containing the data storage authentication for the S3 bucket containing staging data. + The secret should have the following mandatory keys: + - ARCANE_FRAMEWORK__S3_CATALOG_SECRET_ACCESS_KEY + - ARCANE_FRAMEWORK__S3_CATALOG_ACCESS_KEY_ID + - ARCANE_FRAMEWORK__S3_CATALOG_ENDPOINT + - ARCANE_FRAMEWORK__S3_CATALOG_REGION + - AWS_REGION + The AWS_REGION key is used to set the region for the AWS SDK and it's value should be + the same as ARCANE_FRAMEWORK__S3_STAGING_CATALOG_REGION. + type: object + properties: + name: + type: string + optional: true + default: null # for backward compatibility + stagingStorage: + authSecretRef: + description: | + Not used in the operator version 0.0.x + In operator version 0.1.x and above this field will replace the `stagingStorageAuthSecretRef` field. + type: object + properties: + name: + type: string + optional: true + default: null # for backward compatibility + jobTemplateRef: + description: | + Name of the job template to be used for the streaming job if stream is running in normal mode. + type: object + properties: + name: + type: string + kind: + type: string + apiGroup: + type: string + default: + apiGroup: streaming.sneaksanddata.com + kind: StreamingJobTemplate + name: standard-job + backfillJobTemplateRef: + description: | + Name of the job template to be used for the streaming job if stream is running in backfill mode. + type: object + properties: + name: + type: string + kind: + type: string + apiGroup: + type: string + default: + apiGroup: streaming.sneaksanddata.com + kind: StreamingJobTemplate + name: large-job + baseLocation: + type: string + description: Location root for CDM entities, in Proteus format. + entityName: + type: string + description: Name of a CDM entity to stream. + rowsPerGroup: + type: integer + description: Number of rows per parquet row group. + groupingIntervalSeconds: + type: integer + description: Max time to wait for rowsPerGroup to accumulate. Can be from 1 to 60 seconds. + minimum: 1 + maximum: 60 + groupsPerFile: + type: integer + description: Number of row groups per file. + sinkLocation: + type: string + description: Data location for parquet files. + lookBackInterval: + type: integer + description: Number of seconds to look back when determining first set of changes to extract. + changeCaptureIntervalSeconds: + type: integer + description: How long to wait before polling for next result set. Can be from 1 to 60 seconds. + minimum: 1 + maximum: 60 + streamMetadata: + type: object + optional: true + default: null + properties: + datePartition: + type: object + properties: + description: + type: string + fieldName: + type: string + fieldFormat: + type: string + fieldExpression: + type: string + partitions: + type: array + items: + type: object + properties: + description: + type: string + fieldName: + type: string + fieldFormat: + type: string + stagingLocation: + type: string + description: Data location for staging files. + optional: true + default: null # for backward compatibility + retention: + type: object + properties: + retentionPeriod: + type: integer + description: Number of days to retain data. + retentionUnit: + optional: true + default: hours + type: string + enum: + - days + - hours + - minutes + location: + type: string + description: Data location for retention table. + optional: true + default: null # for backward compatibility + status: + type: object + properties: + phase: + type: string + enum: + - RESTARTING + - RUNNING + - RELOADING + - TERMINATING + - STOPPED + - SUSPENDED + - FAILED + conditions: + type: array + items: + type: object + required: + - status + - type + properties: + message: + type: string + phase: + type: string + type: + type: string + enum: + - WARNING + - ERROR + - INFO + - READY + status: + type: string + enum: + - "True" + - "False" diff --git a/.helm/templates/streamclass-synapse.yaml b/.helm/templates/streamclass-synapse.yaml new file mode 100644 index 0000000..cd0df9c --- /dev/null +++ b/.helm/templates/streamclass-synapse.yaml @@ -0,0 +1,20 @@ +apiVersion: streaming.sneaksanddata.com/v1beta1 +kind: StreamClass +metadata: + name: {{ template "app.name" . }}-synapse + namespace: {{ .Release.Namespace }} + labels: + {{- include "streamclass.labels" $ | nindent 4 }} + {{- if .Values.additionalAnnotations }} + annotations: + {{- with .Values.additionalAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +spec: + apiGroupRef: streaming.sneaksanddata.com + kindRef: MicrosoftSynapseStream + apiVersion: v1beta1 + pluralName: microsoft-synapse-streams + secretRefs: + - connectionStringRef