diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 23a178ea3ac63..97d9ffb03838f 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -75,7 +75,9 @@ jobs: - name: Install dependencies run: ./metadata-ingestion/scripts/install_deps.sh - name: Gradle build - run: ./gradlew build -x check -x docs-website:build -x test -x yarnTest -x lint -x yarnLint -x testQuick -x :metadata-integration:java:spark-lineage:test + run: | + ./gradlew build -x check -x docs-website:build -x test -x yarnTest -x lint -x yarnLint -x testQuick -x :metadata-integration:java:spark-lineage:test + ./gradlew :datahub-frontend:dist - name: Smoke test run: ./smoke-test/smoke.sh env: diff --git a/.gitignore b/.gitignore index 6c340de3e64db..4e1bb91170814 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,7 @@ metadata-ingestion/generated/** # docs docs/generated/ tmp* -temp* \ No newline at end of file +temp* + +# frontend assets +datahub-frontend/public/** \ No newline at end of file diff --git a/build.gradle b/build.gradle index 29bd632d7b126..d7cf84bd3defe 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,7 @@ buildscript { } classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0" classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3" + classpath "org.gradle.playframework:gradle-playframework:0.12" classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.18.1" } } @@ -110,14 +111,14 @@ project.ext.externalDependency = [ 'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15', 'parquet': 'org.apache.parquet:parquet-avro:1.12.2', 'picocli': 'info.picocli:picocli:4.5.0', - 'playCache': 'com.typesafe.play:play-cache_2.11:2.6.18', - 'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.11:2.0.8', - 'playDocs': 'com.typesafe.play:play-docs_2.11:2.6.18', - 'playGuice': 'com.typesafe.play:play-guice_2.11:2.6.18', - 'playJavaJdbc': 'com.typesafe.play:play-java-jdbc_2.11:2.6.18', - 'playTest': 'com.typesafe.play:play-test_2.11:2.6.18', + 'playCache': 'com.typesafe.play:play-cache_2.12:2.7.6', + 'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.0.8', + 'playDocs': 'com.typesafe.play:play-docs_2.12:2.7.6', + 'playGuice': 'com.typesafe.play:play-guice_2.12:2.7.6', + 'playJavaJdbc': 'com.typesafe.play:play-java-jdbc_2.12:2.7.6', + 'playTest': 'com.typesafe.play:play-test_2.12:2.7.6', 'pac4j': 'org.pac4j:pac4j-oidc:3.6.0', - 'playPac4j': 'org.pac4j:play-pac4j_2.11:7.0.1', + 'playPac4j': 'org.pac4j:play-pac4j_2.12:8.0.2', 'postgresql': 'org.postgresql:postgresql:42.3.3', 'protobuf': 'com.google.protobuf:protobuf-java:3.19.3', 'reflections': 'org.reflections:reflections:0.9.9', diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java index 4ee3b16502b15..3358998acde2e 100644 --- a/datahub-frontend/app/auth/AuthModule.java +++ b/datahub-frontend/app/auth/AuthModule.java @@ -12,7 +12,6 @@ import com.linkedin.util.Configuration; import com.datahub.authentication.Authentication; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; import java.util.Collections; import org.apache.commons.codec.digest.DigestUtils; import org.pac4j.core.client.Client; diff --git a/datahub-frontend/app/auth/Authenticator.java b/datahub-frontend/app/auth/Authenticator.java index 13e8cbace7547..0add06c297bdc 100644 --- a/datahub-frontend/app/auth/Authenticator.java +++ b/datahub-frontend/app/auth/Authenticator.java @@ -1,6 +1,7 @@ package auth; import com.typesafe.config.Config; +import java.util.Optional; import javax.inject.Inject; import play.mvc.Http; import play.mvc.Result; @@ -22,7 +23,8 @@ public class Authenticator extends Security.Authenticator { @Inject public Authenticator(@Nonnull Config config) { - this.metadataServiceAuthEnabled = config.hasPath(METADATA_SERVICE_AUTH_ENABLED_CONFIG_PATH) && config.getBoolean(METADATA_SERVICE_AUTH_ENABLED_CONFIG_PATH); + this.metadataServiceAuthEnabled = config.hasPath(METADATA_SERVICE_AUTH_ENABLED_CONFIG_PATH) + && config.getBoolean(METADATA_SERVICE_AUTH_ENABLED_CONFIG_PATH); } @Override @@ -38,9 +40,28 @@ public String getUsername(@Nonnull Http.Context ctx) { } } + @Override + public Optional getUsername(@Nonnull Http.Request request) { + Http.Context ctx = Http.Context.current(); + if (this.metadataServiceAuthEnabled) { + // If Metadata Service auth is enabled, we only want to verify presence of the + // "Authorization" header OR the presence of a frontend generated session cookie. + // At this time, the actor is still considered to be unauthenicated. + return Optional.ofNullable(AuthUtils.isEligibleForForwarding(ctx) ? "urn:li:corpuser:UNKNOWN" : null); + } else { + // If Metadata Service auth is not enabled, verify the presence of a valid session cookie. + return Optional.ofNullable(AuthUtils.hasValidSessionCookie(ctx) ? ctx.session().get(ACTOR) : null); + } + } + @Override @Nonnull public Result onUnauthorized(@Nullable Http.Context ctx) { return unauthorized(); } + + @Override + public Result onUnauthorized(Http.Request req) { + return unauthorized(); + } } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java index 50c40d24e8c8b..f350f1b001dc5 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java @@ -10,6 +10,11 @@ public class OidcResponseErrorHandler { + + private OidcResponseErrorHandler() { + + } + private static final Logger _logger = LoggerFactory.getLogger("OidcResponseErrorHandler"); private static final String ERROR_FIELD_NAME = "error"; @@ -22,20 +27,18 @@ public static Result handleError(final PlayWebContext context) { getErrorDescription(context)); if (getError(context).equals("access_denied")) { - return unauthorized(String.format("Access denied. " + - "The OIDC service responded with 'Access denied'. " + - "It seems that you don't have access to this application yet. Please apply for access. \n\n" + - "If you already have been assigned this application, it may be so that your OIDC request is still in action. " + - "Error details: '%s':'%s'", + return unauthorized(String.format("Access denied. " + + "The OIDC service responded with 'Access denied'. " + + "It seems that you don't have access to this application yet. Please apply for access. \n\n" + + "If you already have been assigned this application, it may be so that your OIDC request is still in action. " + + "Error details: '%s':'%s'", context.getRequestParameter("error"), context.getRequestParameter("error_description"))); } return internalServerError( - String.format("Internal server error. The OIDC service responded with an error: '%s'.\n" + - "Error description: '%s'", - getError(context), - getErrorDescription(context))); + String.format("Internal server error. The OIDC service responded with an error: '%s'.\n" + + "Error description: '%s'", getError(context), getErrorDescription(context))); } public static boolean isError(final PlayWebContext context) { diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java index 2ff34f496242e..d780f692f3c40 100644 --- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java +++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java @@ -71,8 +71,12 @@ public CustomOidcAuthenticator(final OidcConfiguration configuration, final Oidc chosenMethod = preferredMethod; } else { throw new TechnicalException( - "Preferred authentication method (" + preferredMethod + ") not supported " + - "by provider according to provider metadata (" + metadataMethods + ")."); + "Preferred authentication method (" + + preferredMethod + + ") not supported " + + "by provider according to provider metadata (" + + metadataMethods + + ")."); } } else { chosenMethod = firstSupportedMethod(metadataMethods); @@ -83,13 +87,13 @@ public CustomOidcAuthenticator(final OidcConfiguration configuration, final Oidc chosenMethod); } - final ClientID _clientID = new ClientID(configuration.getClientId()); + final ClientID clientID = new ClientID(configuration.getClientId()); if (ClientAuthenticationMethod.CLIENT_SECRET_POST.equals(chosenMethod)) { - final Secret _secret = new Secret(configuration.getSecret()); - clientAuthentication = new ClientSecretPost(_clientID, _secret); + final Secret secret = new Secret(configuration.getSecret()); + clientAuthentication = new ClientSecretPost(clientID, secret); } else if (ClientAuthenticationMethod.CLIENT_SECRET_BASIC.equals(chosenMethod)) { - final Secret _secret = new Secret(configuration.getSecret()); - clientAuthentication = new ClientSecretBasic(_clientID, _secret); + final Secret secret = new Secret(configuration.getSecret()); + clientAuthentication = new ClientSecretBasic(clientID, secret); } else if (ClientAuthenticationMethod.NONE.equals(chosenMethod)) { clientAuthentication = null; // No client authentication in none mode } else { @@ -128,8 +132,8 @@ private static ClientAuthenticationMethod firstSupportedMethod(final List proxy(String path) throws ExecutionException, I .toMap() .entrySet() .stream() - .filter(entry -> !AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER.equals(entry.getKey())) // Remove X-DataHub-Actor to prevent malicious delegation. + // Remove X-DataHub-Actor to prevent malicious delegation. + .filter(entry -> !AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER.equals(entry.getKey())) .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey())) .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equals(entry.getKey())) .filter(entry -> !Http.HeaderNames.AUTHORIZATION.equals(entry.getKey())) diff --git a/datahub-frontend/app/controllers/CentralLogoutController.java b/datahub-frontend/app/controllers/CentralLogoutController.java index 86c1dbf253aae..9c8470df4a0a1 100644 --- a/datahub-frontend/app/controllers/CentralLogoutController.java +++ b/datahub-frontend/app/controllers/CentralLogoutController.java @@ -19,14 +19,14 @@ public class CentralLogoutController extends LogoutController { @Inject public CentralLogoutController(Config config) { - String _authBaseUrl = config.hasPath(AUTH_BASE_URL_CONFIG_PATH) + String authBaseUrl = config.hasPath(AUTH_BASE_URL_CONFIG_PATH) ? config.getString(AUTH_BASE_URL_CONFIG_PATH) : DEFAULT_BASE_URL_PATH; _isOidcEnabled = config.hasPath("auth.oidc.enabled") && config.getBoolean("auth.oidc.enabled"); - setDefaultUrl(_authBaseUrl); - setLogoutUrlPattern(_authBaseUrl + ".*"); + setDefaultUrl(authBaseUrl); + setLogoutUrlPattern(authBaseUrl + ".*"); setLocalLogout(true); setCentralLogout(true); diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java index c3a7251f63436..27a4c53724339 100644 --- a/datahub-frontend/app/controllers/SsoCallbackController.java +++ b/datahub-frontend/app/controllers/SsoCallbackController.java @@ -58,7 +58,8 @@ public class SsoCallbackLogic implements CallbackLogic { private final OidcCallbackLogic _oidcCallbackLogic; - SsoCallbackLogic(final SsoManager ssoManager, final Authentication systemAuthentication, final EntityClient entityClient, final AuthServiceClient authClient) { + SsoCallbackLogic(final SsoManager ssoManager, final Authentication systemAuthentication, + final EntityClient entityClient, final AuthServiceClient authClient) { _oidcCallbackLogic = new OidcCallbackLogic(ssoManager, systemAuthentication, entityClient, authClient); } diff --git a/datahub-frontend/app/controllers/TrackingController.java b/datahub-frontend/app/controllers/TrackingController.java index f0c35132cb560..6263550c7588d 100644 --- a/datahub-frontend/app/controllers/TrackingController.java +++ b/datahub-frontend/app/controllers/TrackingController.java @@ -35,7 +35,7 @@ public class TrackingController extends Controller { private final Logger _logger = LoggerFactory.getLogger(TrackingController.class.getName()); private static final List KAFKA_SSL_PROTOCOLS = Collections.unmodifiableList( - Arrays.asList(SecurityProtocol.SSL.name(),SecurityProtocol.SASL_SSL.name(), + Arrays.asList(SecurityProtocol.SSL.name(), SecurityProtocol.SASL_SSL.name(), SecurityProtocol.SASL_PLAINTEXT.name())); private final Boolean _isEnabled; @@ -81,7 +81,7 @@ public Result track() throws Exception { _producer.send(record); _producer.flush(); return ok(); - } catch(Exception e) { + } catch (Exception e) { _logger.error(String.format("Failed to emit product analytics event. actor: %s, event: %s", actor, event)); return internalServerError(e.getMessage()); } diff --git a/datahub-frontend/app/security/AuthenticationManager.java b/datahub-frontend/app/security/AuthenticationManager.java index 1fb755564999d..3b877ae86e275 100644 --- a/datahub-frontend/app/security/AuthenticationManager.java +++ b/datahub-frontend/app/security/AuthenticationManager.java @@ -44,6 +44,7 @@ public void handle(@Nonnull Callback[] callbacks) { NameCallback nc = null; PasswordCallback pc = null; for (Callback callback : callbacks) { + Logger.error("The submitted callback is of type: " + callback.getClass() + " : " + callback); if (callback instanceof NameCallback) { nc = (NameCallback) callback; nc.setName(this.username); diff --git a/datahub-frontend/app/utils/ConfigUtil.java b/datahub-frontend/app/utils/ConfigUtil.java index 89f0444a18b39..b99a5e123b9eb 100644 --- a/datahub-frontend/app/utils/ConfigUtil.java +++ b/datahub-frontend/app/utils/ConfigUtil.java @@ -6,6 +6,10 @@ public class ConfigUtil { + private ConfigUtil() { + + } + // New configurations, provided via application.conf file. public static final String METADATA_SERVICE_HOST_CONFIG_PATH = "metadataService.host"; public static final String METADATA_SERVICE_PORT_CONFIG_PATH = "metadataService.port"; diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle index 68f1103efa0f8..c63e4ef9d0e0d 100644 --- a/datahub-frontend/build.gradle +++ b/datahub-frontend/build.gradle @@ -46,4 +46,21 @@ graphqlCodegen { tasks.withType(Checkstyle) { exclude "**/generated/**" +} + +checkstyleMain.source = "app/" + + +/* +PLAY UPGRADE NOTE +Generates the distribution jars under the expected names. The playFramework plugin only accepts certain name values +for the resulting folders and files, so some changes were made to accommodate. Default distribution is main if these are excluded + */ +distributions { + create("datahub-frontend") { + distributionBaseName = project.ext.playBinaryBaseName + } + playBinary { + distributionBaseName = project.ext.playBinaryBaseName + } } \ No newline at end of file diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf index 042987a47923b..17772b55a0471 100644 --- a/datahub-frontend/conf/application.conf +++ b/datahub-frontend/conf/application.conf @@ -26,6 +26,10 @@ play.modules.disabled += "play.api.mvc.CookiesModule" play.modules.enabled += "play.api.mvc.LegacyCookiesModule" play.modules.enabled += "auth.AuthModule" +# Legacy Configuration to avoid code changes, update to modern approaches eventually +play.allowHttpContext = true +play.allowGlobalApplication = true + # Database configuration # ~~~~~ # You can declare as many datasources as you want. diff --git a/datahub-frontend/conf/routes b/datahub-frontend/conf/routes index 200341baec9aa..32af2d54f4a26 100644 --- a/datahub-frontend/conf/routes +++ b/datahub-frontend/conf/routes @@ -19,22 +19,22 @@ POST /callback/:protocol co GET /logOut controllers.CentralLogoutController.executeLogout() # Proxies API requests to the metadata service api -GET /api/*path controllers.Application.proxy(path) -POST /api/*path controllers.Application.proxy(path) -DELETE /api/*path controllers.Application.proxy(path) -PUT /api/*path controllers.Application.proxy(path) +GET /api/*path controllers.Application.proxy(path) +POST /api/*path controllers.Application.proxy(path) +DELETE /api/*path controllers.Application.proxy(path) +PUT /api/*path controllers.Application.proxy(path) # Proxies API requests to the metadata service api -GET /openapi/*path controllers.Application.proxy(path) -POST /openapi/*path controllers.Application.proxy(path) -DELETE /openapi/*path controllers.Application.proxy(path) -PUT /openapi/*path controllers.Application.proxy(path) +GET /openapi/*path controllers.Application.proxy(path) +POST /openapi/*path controllers.Application.proxy(path) +DELETE /openapi/*path controllers.Application.proxy(path) +PUT /openapi/*path controllers.Application.proxy(path) # Map static resources from the /public folder to the /assets URL path -GET /assets/*file controllers.Assets.at(path="/public", file) +GET /assets/*file controllers.Assets.at(path="/public", file) # Analytics route -POST /track controllers.TrackingController.track() +POST /track controllers.TrackingController.track() # Wildcard route accepts any routes and delegates to serveAsset which in turn serves the React Bundle -GET /*path controllers.Application.index(path) +GET /*path controllers.Application.index(path) diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index 6b8517ca62a21..ad1d7f59e8fa8 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -1,4 +1,4 @@ -apply plugin: 'play' +apply plugin: "org.gradle.playframework" // Change this to listen on a different port project.ext.httpPort = 9001 @@ -23,42 +23,56 @@ dependencies { play('org.springframework:spring-core:5.2.3.RELEASE') play('com.fasterxml.jackson.core:jackson-databind:2.9.10.4') play('com.nimbusds:nimbus-jose-jwt:7.9') - play('com.typesafe.akka:akka-actor_2.11:2.5.16') + play('com.typesafe.akka:akka-actor_2.12:2.5.16') play('net.minidev:json-smart:2.4.1') play('io.netty:netty-all:4.1.68.Final') } - play project(":datahub-graphql-core") - play project(":metadata-service:auth-api") - - play externalDependency.jettyJaas - play externalDependency.graphqlJava - play externalDependency.antlr4Runtime - play externalDependency.antlr4 - - play externalDependency.jerseyCore - play externalDependency.jerseyGuava - - play externalDependency.pac4j - play externalDependency.playPac4j - play externalDependency.shiroCore - play externalDependency.playCache - play externalDependency.playWs - play externalDependency.kafkaClients - - playTest externalDependency.mockito - playTest externalDependency.playTest - - playRun externalDependency.lombok - playRun externalDependency.guice - playRun externalDependency.playDocs - playRun externalDependency.playGuice - playRun externalDependency.logbackClassic + compile project(":datahub-graphql-core") + compile project(":metadata-service:auth-api") + + implementation externalDependency.jettyJaas + implementation externalDependency.graphqlJava + implementation externalDependency.antlr4Runtime + implementation externalDependency.antlr4 + + implementation externalDependency.jerseyCore + implementation externalDependency.jerseyGuava + + implementation externalDependency.pac4j + implementation externalDependency.playPac4j + implementation externalDependency.shiroCore + implementation externalDependency.playCache + implementation externalDependency.playWs + implementation externalDependency.kafkaClients + + testImplementation externalDependency.mockito + testImplementation externalDependency.playTest + + compileOnly externalDependency.lombok + runtime externalDependency.guice + runtime externalDependency.playDocs + runtime externalDependency.playGuice + runtime externalDependency.logbackClassic + + annotationProcessor externalDependency.lombok +} + +dist.dependsOn(':datahub-web-react:copyAssets') + +play { + platform { + playVersion = '2.7.6' + scalaVersion = '2.12' + javaVersion = JavaVersion.VERSION_1_8 + } + + injectedRoutesGenerator = true } model { components { play { - platform play: '2.6.18', scala: '2.11', java: '1.8' + platform play: '2.7.6', scala: '2.12', java: '1.8' injectedRoutesGenerator = true binaries.all { @@ -73,11 +87,6 @@ model { } } } - distributions { - playBinary { - baseName = project.ext.playBinaryBaseName - } - } } task unzipAssets(type: Copy, dependsOn: [configurations.assets, ':datahub-web-react:yarnBuild']) { @@ -92,3 +101,14 @@ task moveAssets(type: Copy, dependsOn: unzipAssets) { into "${buildDir}/assets" from ("${buildDir}/assets/assets") } + +clean { + delete 'public/platforms' + delete 'public/static' + delete 'public/asset-manifest.json' + delete 'public/manifest.json' + delete 'public/robots.txt' + delete 'public/logo.png' + delete 'public/index.html' + delete 'public/favicon.ico' +} diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle index 28714fc4c9ecb..6e58e4aec13e9 100644 --- a/datahub-web-react/build.gradle +++ b/datahub-web-react/build.gradle @@ -87,6 +87,14 @@ distZip { from 'dist' } +task copyAssets { + dependsOn distZip + copy { + from 'dist' + into '../datahub-frontend/public/' + } +} + if (!gradle.startParameter.taskNames.any { it in ["idea"] }) { artifacts { assets distZip diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile index ee9ba4283cfb7..723e20a237cd8 100644 --- a/docker/datahub-frontend/Dockerfile +++ b/docker/datahub-frontend/Dockerfile @@ -52,4 +52,4 @@ ENV JAVA_OPTS=" \ -Dlogback.configurationFile=datahub-frontend/conf/logback.xml \ -Dlogback.debug=false \ -Dpidfile.path=/dev/null" -CMD ["datahub-frontend/bin/playBinary"] +CMD ["datahub-frontend/bin/datahub-frontend"] diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index fa50944e080be..94b06cba59e91 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -55,4 +55,4 @@ services: args: APP_ENV: dev volumes: - - ../datahub-frontend/build/stage/datahub-frontend:/datahub-frontend + - ../datahub-frontend/build/stage/playBinary:/datahub-frontend