Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New Unity Catalog resource connector #8248

Merged
merged 2 commits into from
Jun 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/merge-v5.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ jobs:
uses: docker/build-push-action@v5
with:
push: true
tags: odpi/egeria-platform:${{ env.VERSION }}, odpi/egeria-platform:latest, quay.io/odpi/egeria-platform:${{ env.VERSION }}, quay.io/odpi/egeria-platform:latest
tags: odpi/egeria-platform:${{ env.VERSION }}, odpi/egeria-platform:latest, quay.io/odpi/egeria:${{ env.VERSION }}, quay.io/odpi/egeria:latest
context: ./open-metadata-distribution/omag-server-platform/build/unpacked/egeria-platform-${{ env.VERSION }}-distribution.tar.gz
platforms: linux/amd64,linux/arm64
- name: Build and push(egeria-platform) to quay.io and docker.io (no tag latest)
Expand Down
2 changes: 1 addition & 1 deletion CoreContentPackGUIDMap.json

Large diffs are not rendered by default.

31 changes: 16 additions & 15 deletions MAINTAINERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,22 @@ new pull requests into main.
The current Egeria maintainers are:


| Name | Email | GitHub id
| --------------------- | ------------------------------ | --------------
| Mandy Chessell | [email protected] | mandy-chessell
| Nigel Jones | [email protected] | planetf1
| David Radley | [email protected] | davidradl
| Maryna Strelchuk | [email protected] | mstrelchuk
| Bogdan Sava | [email protected] | bogdan-sava
| Chris Grote | [email protected] | cmgrote
| Raluca Popa | [email protected] | popa-raluca
| Chris Replogle | [email protected] | CDARip2U
| Ljupcho Palashevski | [email protected] | lpalashevski
| Alexandra Bucur | [email protected] | alexandra-bucur
| Cezar Sirbu | [email protected] | sarbull
| Liviu Constantin Popa | [email protected] | lcpopa
| Cristian-Mihai Iliescu | [email protected] | MihaiIliescu
| Name | Email | GitHub id
|------------------------|--------------------------------| --------------
| Mandy Chessell | [email protected] | mandy-chessell
| Dan Wolfson | [email protected] | dwolfson
| Nigel Jones | [email protected] | planetf1
| David Radley | [email protected] | davidradl
| Maryna Strelchuk | [email protected] | mstrelchuk
| Bogdan Sava | [email protected] | bogdan-sava
| Chris Grote | [email protected] | cmgrote
| Raluca Popa | [email protected] | popa-raluca
| Chris Replogle | [email protected] | CDARip2U
| Ljupcho Palashevski | [email protected] | lpalashevski
| Alexandra Bucur | [email protected] | alexandra-bucur
| Cezar Sirbu | [email protected] | sarbull
| Liviu Constantin Popa | [email protected] | lcpopa
| Cristian-Mihai Iliescu | [email protected] | MihaiIliescu

To understand how to become an Egeria maintainer
see the [Egeria Operations Guide](https://egeria-project.org/guides/project-operations/).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ public enum PostgresPlaceholderProperty
"string",
"myDatabase"),

/**
* The name of the database being catalogued.
*/
DATABASE_DESCRIPTION ("databaseDescription",
"The description of the database being catalogued.",
"string",
null),

/**
* The userId to store in the userId attribute of the connection.
*/
Expand All @@ -52,13 +60,23 @@ public enum PostgresPlaceholderProperty
/**
* The name of the server being catalogued.
*/
SERVER_NAME ("serverName", "The name of the database server being catalogued.", "string", "myServer"),
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myServer"),

/**
* The description of the server being catalogued.
*/
SERVER_DESCRIPTION ("serverDescription", "The description of the server being catalogued.", "string", null),

/**
* The name of the schema being catalogued.
*/
SCHEMA_NAME ("schemaName", "The name of the database schema being catalogued.", "string", "MyServer.schema"),

/**
* The description of the schema being catalogued.
*/
SCHEMA_DESCRIPTION ("schemaDescription", "The description of the database schema being catalogued.", "string", null),

/**
* The name of the database table being catalogued.
*/
Expand Down Expand Up @@ -158,6 +176,7 @@ public static List<PlaceholderPropertyType> getPostgresServerPlaceholderProperty
placeholderPropertyTypes.add(PostgresPlaceholderProperty.HOST_IDENTIFIER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.PORT_NUMBER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

Expand All @@ -178,6 +197,7 @@ public static List<PlaceholderPropertyType> getPostgresDatabasePlaceholderProper
placeholderPropertyTypes.add(PostgresPlaceholderProperty.PORT_NUMBER.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

Expand All @@ -199,6 +219,7 @@ public static List<PlaceholderPropertyType> getPostgresSchemaPlaceholderProperty
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SERVER_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SCHEMA_NAME.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.SCHEMA_DESCRIPTION.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_USER_ID.getPlaceholderType());
placeholderPropertyTypes.add(PostgresPlaceholderProperty.DATABASE_PASSWORD.getPlaceholderType());

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@

@ucURL=http://localhost:8080

###
# @name list catalogs
GET {{ucURL}}/api/2.1/unity-catalog/catalogs?max_results=1

###
# @name create catalog
POST {{ucURL}}/api/2.1/unity-catalog/catalogs
Content-Type: application/json

{
"name": "test3",
"comment": "Test catalog3",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name get catalog
GET {{ucURL}}/api/2.1/unity-catalog/catalogs/test3
Content-Type: application/json


###
# @name patch catalog
PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test4
Content-Type: application/json

{
"comment" : "This is a comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name rename catalog
PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test3_temp
Content-Type: application/json

{
"new_name" : "test4",
"comment" : "This is a new comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name delete catalog
DELETE {{ucURL}}/api/2.1/unity-catalog/catalogs/test3
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

@ucURL=http://localhost:8080

###
# @name list functions
GET {{ucURL}}/api/2.1/unity-catalog/functions?catalog_name=test&schema_name=test4schema1&max_results=10

###
# @name create function
POST {{ucURL}}/api/2.1/unity-catalog/functions
Content-Type: application/json

{
"function_info" :
{
"name": "createWisdom",
"catalog_name" : "test",
"schema_name" : "test4schema1",
"comment": "Test catalog4 function 1",
"parameter_style" : "S",
"routine_body" : "EXTERNAL",
"external_language" : "Java",
"security_type" : "DEFINER"
}
}

###
# @name get function
GET {{ucURL}}/api/2.1/unity-catalog/functions/test.test4schema1.createWisdom
Content-Type: application/json


###
# @name delete function
DELETE {{ucURL}}/api/2.1/unity-catalog/functions/test.test4schema1.createWisdom
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@

@ucURL=http://localhost:8080

###
# @name list schemas
GET {{ucURL}}/api/2.1/unity-catalog/schemas?catalog_name=unity&max_results=10

###
# @name create schema
POST {{ucURL}}/api/2.1/unity-catalog/schemas
Content-Type: application/json

{
"name": "test4schema1",
"catalog_name" : "test",
"comment": "Test catalog4 schema 1",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name get schema
GET {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema1
Content-Type: application/json


###
# @name patch schema
PATCH {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema2
Content-Type: application/json

{
"new_name" : "test4schema1",
"comment" : "This is a comment",
"properties": {
"egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d"
}
}

###
# @name delete schema
DELETE {{ucURL}}/api/2.1/unity-catalog/schemas/test.test4schema1
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

@ucURL=http://localhost:8080

###
# @name list tables
GET {{ucURL}}/api/2.1/unity-catalog/tables?catalog_name=test&schema_name=test4schema1&max_results=10

###
# @name create table
POST {{ucURL}}/api/2.1/unity-catalog/tables
Content-Type: application/json

{
"table_info" :
{
"name": "bookOfKnowledge",
"catalog_name" : "test",
"schema_name" : "test4schema1",
"comment": "Useful data",
"parameter_style" : "S",
"routine_body" : "EXTERNAL",
"external_language" : "Java",
"security_type" : "DEFINER"
}
}

###
# @name get table
GET {{ucURL}}/api/2.1/unity-catalog/tables/test.test4schema1.createWisdom
Content-Type: application/json


###
# @name delete table
DELETE {{ucURL}}/api/2.1/unity-catalog/tables/test.test4schema1.createWisdom
Content-Type: application/json

Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@

@ucURL=http://localhost:8080

###
# @name list volumes
GET {{ucURL}}/api/2.1/unity-catalog/volumes?catalog_name=unity&schema_name=default&max_results=10

###
# @name create volume
POST {{ucURL}}/api/2.1/unity-catalog/volumes
Content-Type: application/json

{
"name": "test4volume1",
"catalog_name" : "test",
"schema_name": "test4schema1",
"comment": "Test catalog4 volume 1",
"volume_type": "EXTERNAL",
"storage_location": "file:///Users/me/Code/uc/unitycatalog/etc/data/external/unity/default/volumes/json_files/"
}

###
# @name get volume
GET {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.volume1
Content-Type: application/json


###
# @name patch volume
PATCH {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.test4volume1
Content-Type: application/json

{
"new_name" : "volume1",
"comment" : "This is an updated comment"
}

###
# @name delete volume
DELETE {{ucURL}}/api/2.1/unity-catalog/volumes/test.test4schema1.test4volume1
Content-Type: application/json

Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,16 @@ dependencies {
compileOnly project(':open-metadata-implementation:access-services:asset-manager:asset-manager-api')
compileOnly project(':open-metadata-implementation:integration-services:catalog-integrator:catalog-integrator-api')
compileOnly project(':open-metadata-implementation:repository-services:repository-services-apis')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:rest-client-connectors-api')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:rest-client-factory')
compileOnly project(':open-metadata-implementation:adapters:open-connectors:rest-client-connectors:spring-rest-client-connector')
testCompileOnly project(':open-metadata-implementation:frameworks:audit-log-framework')
testCompileOnly project(':open-metadata-implementation:repository-services:repository-services-apis')
testImplementation project(':open-metadata-test:open-metadata-ut')
testImplementation 'org.testng:testng'
compileOnly 'org.springframework:spring-core'
compileOnly 'com.fasterxml.jackson.core:jackson-databind'
compileOnly 'com.fasterxml.jackson.core:jackson-annotations'
}

shadowJar {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,13 @@ public enum UnityCatalogPlaceholderProperty
/**
* The name of the server being catalogued.
*/
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myOMAGServer"),
SERVER_NAME ("serverName", "The name of the server being catalogued.", "string", "myServer"),

/**
* The description of the server being catalogued.
*/
SERVER_DESCRIPTION ("serverDescription", "The description of the server being catalogued.", "string", null),

;

public final String name;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@ public enum UCAuditCode implements AuditLogMessageSet
"The integration connector continues, looking for another database.",
"This is an information message showing that the integration connector is working, but does not need to do any processing on this database."),



/**
* UNITY-CATALOG-CONNECTOR-0005 - A client-side exception was received from API call {0} to server {1} at {2}. The error message was {3}
*/
CLIENT_SIDE_REST_API_ERROR( "UNITY-CATALOG-CONNECTOR-0005",
AuditLogRecordSeverityLevel.EXCEPTION,
"A client-side exception was received from API call {0} to server {1} at {2}. The error message was {3}",
"The server has issued a call to the open metadata access service REST API in a remote server and has received an exception from the local client libraries.",
"Look for errors in the local server's console to understand and correct the source of the error."),

;

private final String logMessageId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,15 @@ public enum UCErrorCode implements ExceptionMessageSet
"The connector is unable to process the current request.",
"Use the details from the error message to determine the cause of the error and retry the request once it is resolved."),


/**
* UNITY-CATALOG-CONNECTOR-503-001 - A client-side exception {0} was received from API call {1} to URL {2}. The error message was {3}
*/
CLIENT_SIDE_REST_API_ERROR(503, "UNITY-CATALOG-CONNECTOR-503-001",
"A client-side exception of {0} was received from API call {1} to URL {2}. The error message was {3}",
"The connector has issued a call to the open metadata access service REST API in a remote server and has received an exception from the local client libraries.",
"Look for errors in the local server's console to understand and correct the source of the error."),

;

private final int httpErrorCode;
Expand Down
Loading
Loading