diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 97d9ffb03838f..23a178ea3ac63 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -75,9 +75,7 @@ jobs: - name: Install dependencies run: ./metadata-ingestion/scripts/install_deps.sh - name: Gradle build - run: | - ./gradlew build -x check -x docs-website:build -x test -x yarnTest -x lint -x yarnLint -x testQuick -x :metadata-integration:java:spark-lineage:test - ./gradlew :datahub-frontend:dist + run: ./gradlew build -x check -x docs-website:build -x test -x yarnTest -x lint -x yarnLint -x testQuick -x :metadata-integration:java:spark-lineage:test - name: Smoke test run: ./smoke-test/smoke.sh env: diff --git a/.gitignore b/.gitignore index 4e1bb91170814..6c340de3e64db 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,4 @@ metadata-ingestion/generated/** # docs docs/generated/ tmp* -temp* - -# frontend assets -datahub-frontend/public/** \ No newline at end of file +temp* \ No newline at end of file diff --git a/build.gradle b/build.gradle index a3fe594582855..c34905d5aa183 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,6 @@ buildscript { } classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0" classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3" - classpath "org.gradle.playframework:gradle-playframework:0.12" classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.18.1" } } @@ -111,14 +110,14 @@ project.ext.externalDependency = [ 'opentracingJdbc':'io.opentracing.contrib:opentracing-jdbc:0.2.15', 'parquet': 'org.apache.parquet:parquet-avro:1.12.2', 'picocli': 'info.picocli:picocli:4.5.0', - 'playCache': 'com.typesafe.play:play-cache_2.12:2.7.6', - 'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.0.8', - 'playDocs': 'com.typesafe.play:play-docs_2.12:2.7.6', - 'playGuice': 'com.typesafe.play:play-guice_2.12:2.7.6', - 'playJavaJdbc': 'com.typesafe.play:play-java-jdbc_2.12:2.7.6', - 'playTest': 'com.typesafe.play:play-test_2.12:2.7.6', + 'playCache': 'com.typesafe.play:play-cache_2.11:2.6.18', + 'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.11:2.0.8', + 'playDocs': 'com.typesafe.play:play-docs_2.11:2.6.18', + 'playGuice': 'com.typesafe.play:play-guice_2.11:2.6.18', + 'playJavaJdbc': 'com.typesafe.play:play-java-jdbc_2.11:2.6.18', + 'playTest': 'com.typesafe.play:play-test_2.11:2.6.18', 'pac4j': 'org.pac4j:pac4j-oidc:3.6.0', - 'playPac4j': 'org.pac4j:play-pac4j_2.12:8.0.2', + 'playPac4j': 'org.pac4j:play-pac4j_2.11:7.0.1', 'postgresql': 'org.postgresql:postgresql:42.3.3', 'protobuf': 'com.google.protobuf:protobuf-java:3.19.3', 'reflections': 'org.reflections:reflections:0.9.9', diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java index 3358998acde2e..4ee3b16502b15 100644 --- a/datahub-frontend/app/auth/AuthModule.java +++ b/datahub-frontend/app/auth/AuthModule.java @@ -12,6 +12,7 @@ import com.linkedin.util.Configuration; import com.datahub.authentication.Authentication; import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; import java.util.Collections; import org.apache.commons.codec.digest.DigestUtils; import org.pac4j.core.client.Client; diff --git a/datahub-frontend/app/auth/Authenticator.java b/datahub-frontend/app/auth/Authenticator.java index 0add06c297bdc..13e8cbace7547 100644 --- a/datahub-frontend/app/auth/Authenticator.java +++ b/datahub-frontend/app/auth/Authenticator.java @@ -1,7 +1,6 @@ package auth; import com.typesafe.config.Config; -import java.util.Optional; import javax.inject.Inject; import play.mvc.Http; import play.mvc.Result; @@ -23,8 +22,7 @@ public class Authenticator extends Security.Authenticator { @Inject public Authenticator(@Nonnull Config config) { - this.metadataServiceAuthEnabled = config.hasPath(METADATA_SERVICE_AUTH_ENABLED_CONFIG_PATH) - && config.getBoolean(METADATA_SERVICE_AUTH_ENABLED_CONFIG_PATH); + this.metadataServiceAuthEnabled = config.hasPath(METADATA_SERVICE_AUTH_ENABLED_CONFIG_PATH) && config.getBoolean(METADATA_SERVICE_AUTH_ENABLED_CONFIG_PATH); } @Override @@ -40,28 +38,9 @@ public String getUsername(@Nonnull Http.Context ctx) { } } - @Override - public Optional getUsername(@Nonnull Http.Request request) { - Http.Context ctx = Http.Context.current(); - if (this.metadataServiceAuthEnabled) { - // If Metadata Service auth is enabled, we only want to verify presence of the - // "Authorization" header OR the presence of a frontend generated session cookie. - // At this time, the actor is still considered to be unauthenicated. - return Optional.ofNullable(AuthUtils.isEligibleForForwarding(ctx) ? "urn:li:corpuser:UNKNOWN" : null); - } else { - // If Metadata Service auth is not enabled, verify the presence of a valid session cookie. - return Optional.ofNullable(AuthUtils.hasValidSessionCookie(ctx) ? ctx.session().get(ACTOR) : null); - } - } - @Override @Nonnull public Result onUnauthorized(@Nullable Http.Context ctx) { return unauthorized(); } - - @Override - public Result onUnauthorized(Http.Request req) { - return unauthorized(); - } } diff --git a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java index f350f1b001dc5..50c40d24e8c8b 100644 --- a/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java +++ b/datahub-frontend/app/auth/sso/oidc/OidcResponseErrorHandler.java @@ -10,11 +10,6 @@ public class OidcResponseErrorHandler { - - private OidcResponseErrorHandler() { - - } - private static final Logger _logger = LoggerFactory.getLogger("OidcResponseErrorHandler"); private static final String ERROR_FIELD_NAME = "error"; @@ -27,18 +22,20 @@ public static Result handleError(final PlayWebContext context) { getErrorDescription(context)); if (getError(context).equals("access_denied")) { - return unauthorized(String.format("Access denied. " - + "The OIDC service responded with 'Access denied'. " - + "It seems that you don't have access to this application yet. Please apply for access. \n\n" - + "If you already have been assigned this application, it may be so that your OIDC request is still in action. " - + "Error details: '%s':'%s'", + return unauthorized(String.format("Access denied. " + + "The OIDC service responded with 'Access denied'. " + + "It seems that you don't have access to this application yet. Please apply for access. \n\n" + + "If you already have been assigned this application, it may be so that your OIDC request is still in action. " + + "Error details: '%s':'%s'", context.getRequestParameter("error"), context.getRequestParameter("error_description"))); } return internalServerError( - String.format("Internal server error. The OIDC service responded with an error: '%s'.\n" - + "Error description: '%s'", getError(context), getErrorDescription(context))); + String.format("Internal server error. The OIDC service responded with an error: '%s'.\n" + + "Error description: '%s'", + getError(context), + getErrorDescription(context))); } public static boolean isError(final PlayWebContext context) { diff --git a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java index d780f692f3c40..2ff34f496242e 100644 --- a/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java +++ b/datahub-frontend/app/auth/sso/oidc/custom/CustomOidcAuthenticator.java @@ -71,12 +71,8 @@ public CustomOidcAuthenticator(final OidcConfiguration configuration, final Oidc chosenMethod = preferredMethod; } else { throw new TechnicalException( - "Preferred authentication method (" - + preferredMethod - + ") not supported " - + "by provider according to provider metadata (" - + metadataMethods - + ")."); + "Preferred authentication method (" + preferredMethod + ") not supported " + + "by provider according to provider metadata (" + metadataMethods + ")."); } } else { chosenMethod = firstSupportedMethod(metadataMethods); @@ -87,13 +83,13 @@ public CustomOidcAuthenticator(final OidcConfiguration configuration, final Oidc chosenMethod); } - final ClientID clientID = new ClientID(configuration.getClientId()); + final ClientID _clientID = new ClientID(configuration.getClientId()); if (ClientAuthenticationMethod.CLIENT_SECRET_POST.equals(chosenMethod)) { - final Secret secret = new Secret(configuration.getSecret()); - clientAuthentication = new ClientSecretPost(clientID, secret); + final Secret _secret = new Secret(configuration.getSecret()); + clientAuthentication = new ClientSecretPost(_clientID, _secret); } else if (ClientAuthenticationMethod.CLIENT_SECRET_BASIC.equals(chosenMethod)) { - final Secret secret = new Secret(configuration.getSecret()); - clientAuthentication = new ClientSecretBasic(clientID, secret); + final Secret _secret = new Secret(configuration.getSecret()); + clientAuthentication = new ClientSecretBasic(_clientID, _secret); } else if (ClientAuthenticationMethod.NONE.equals(chosenMethod)) { clientAuthentication = null; // No client authentication in none mode } else { @@ -132,8 +128,8 @@ private static ClientAuthenticationMethod firstSupportedMethod(final List proxy(String path) throws ExecutionException, I .toMap() .entrySet() .stream() - // Remove X-DataHub-Actor to prevent malicious delegation. - .filter(entry -> !AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER.equals(entry.getKey())) + .filter(entry -> !AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER.equals(entry.getKey())) // Remove X-DataHub-Actor to prevent malicious delegation. .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey())) .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equals(entry.getKey())) .filter(entry -> !Http.HeaderNames.AUTHORIZATION.equals(entry.getKey())) diff --git a/datahub-frontend/app/controllers/CentralLogoutController.java b/datahub-frontend/app/controllers/CentralLogoutController.java index 9c8470df4a0a1..86c1dbf253aae 100644 --- a/datahub-frontend/app/controllers/CentralLogoutController.java +++ b/datahub-frontend/app/controllers/CentralLogoutController.java @@ -19,14 +19,14 @@ public class CentralLogoutController extends LogoutController { @Inject public CentralLogoutController(Config config) { - String authBaseUrl = config.hasPath(AUTH_BASE_URL_CONFIG_PATH) + String _authBaseUrl = config.hasPath(AUTH_BASE_URL_CONFIG_PATH) ? config.getString(AUTH_BASE_URL_CONFIG_PATH) : DEFAULT_BASE_URL_PATH; _isOidcEnabled = config.hasPath("auth.oidc.enabled") && config.getBoolean("auth.oidc.enabled"); - setDefaultUrl(authBaseUrl); - setLogoutUrlPattern(authBaseUrl + ".*"); + setDefaultUrl(_authBaseUrl); + setLogoutUrlPattern(_authBaseUrl + ".*"); setLocalLogout(true); setCentralLogout(true); diff --git a/datahub-frontend/app/controllers/SsoCallbackController.java b/datahub-frontend/app/controllers/SsoCallbackController.java index 27a4c53724339..c3a7251f63436 100644 --- a/datahub-frontend/app/controllers/SsoCallbackController.java +++ b/datahub-frontend/app/controllers/SsoCallbackController.java @@ -58,8 +58,7 @@ public class SsoCallbackLogic implements CallbackLogic { private final OidcCallbackLogic _oidcCallbackLogic; - SsoCallbackLogic(final SsoManager ssoManager, final Authentication systemAuthentication, - final EntityClient entityClient, final AuthServiceClient authClient) { + SsoCallbackLogic(final SsoManager ssoManager, final Authentication systemAuthentication, final EntityClient entityClient, final AuthServiceClient authClient) { _oidcCallbackLogic = new OidcCallbackLogic(ssoManager, systemAuthentication, entityClient, authClient); } diff --git a/datahub-frontend/app/controllers/TrackingController.java b/datahub-frontend/app/controllers/TrackingController.java index 6263550c7588d..f0c35132cb560 100644 --- a/datahub-frontend/app/controllers/TrackingController.java +++ b/datahub-frontend/app/controllers/TrackingController.java @@ -35,7 +35,7 @@ public class TrackingController extends Controller { private final Logger _logger = LoggerFactory.getLogger(TrackingController.class.getName()); private static final List KAFKA_SSL_PROTOCOLS = Collections.unmodifiableList( - Arrays.asList(SecurityProtocol.SSL.name(), SecurityProtocol.SASL_SSL.name(), + Arrays.asList(SecurityProtocol.SSL.name(),SecurityProtocol.SASL_SSL.name(), SecurityProtocol.SASL_PLAINTEXT.name())); private final Boolean _isEnabled; @@ -81,7 +81,7 @@ public Result track() throws Exception { _producer.send(record); _producer.flush(); return ok(); - } catch (Exception e) { + } catch(Exception e) { _logger.error(String.format("Failed to emit product analytics event. actor: %s, event: %s", actor, event)); return internalServerError(e.getMessage()); } diff --git a/datahub-frontend/app/security/AuthenticationManager.java b/datahub-frontend/app/security/AuthenticationManager.java index 8103d4a8c2d15..14f95709d625f 100644 --- a/datahub-frontend/app/security/AuthenticationManager.java +++ b/datahub-frontend/app/security/AuthenticationManager.java @@ -55,7 +55,6 @@ public void handle(@Nonnull Callback[] callbacks) { NameCallback nc = null; PasswordCallback pc = null; for (Callback callback : callbacks) { - Logger.error("The submitted callback is of type: " + callback.getClass() + " : " + callback); if (callback instanceof NameCallback) { nc = (NameCallback) callback; nc.setName(this.username); diff --git a/datahub-frontend/app/utils/ConfigUtil.java b/datahub-frontend/app/utils/ConfigUtil.java index b99a5e123b9eb..89f0444a18b39 100644 --- a/datahub-frontend/app/utils/ConfigUtil.java +++ b/datahub-frontend/app/utils/ConfigUtil.java @@ -6,10 +6,6 @@ public class ConfigUtil { - private ConfigUtil() { - - } - // New configurations, provided via application.conf file. public static final String METADATA_SERVICE_HOST_CONFIG_PATH = "metadataService.host"; public static final String METADATA_SERVICE_PORT_CONFIG_PATH = "metadataService.port"; diff --git a/datahub-frontend/build.gradle b/datahub-frontend/build.gradle index c63e4ef9d0e0d..68f1103efa0f8 100644 --- a/datahub-frontend/build.gradle +++ b/datahub-frontend/build.gradle @@ -46,21 +46,4 @@ graphqlCodegen { tasks.withType(Checkstyle) { exclude "**/generated/**" -} - -checkstyleMain.source = "app/" - - -/* -PLAY UPGRADE NOTE -Generates the distribution jars under the expected names. The playFramework plugin only accepts certain name values -for the resulting folders and files, so some changes were made to accommodate. Default distribution is main if these are excluded - */ -distributions { - create("datahub-frontend") { - distributionBaseName = project.ext.playBinaryBaseName - } - playBinary { - distributionBaseName = project.ext.playBinaryBaseName - } } \ No newline at end of file diff --git a/datahub-frontend/conf/application.conf b/datahub-frontend/conf/application.conf index 17772b55a0471..042987a47923b 100644 --- a/datahub-frontend/conf/application.conf +++ b/datahub-frontend/conf/application.conf @@ -26,10 +26,6 @@ play.modules.disabled += "play.api.mvc.CookiesModule" play.modules.enabled += "play.api.mvc.LegacyCookiesModule" play.modules.enabled += "auth.AuthModule" -# Legacy Configuration to avoid code changes, update to modern approaches eventually -play.allowHttpContext = true -play.allowGlobalApplication = true - # Database configuration # ~~~~~ # You can declare as many datasources as you want. diff --git a/datahub-frontend/conf/routes b/datahub-frontend/conf/routes index 32af2d54f4a26..200341baec9aa 100644 --- a/datahub-frontend/conf/routes +++ b/datahub-frontend/conf/routes @@ -19,22 +19,22 @@ POST /callback/:protocol co GET /logOut controllers.CentralLogoutController.executeLogout() # Proxies API requests to the metadata service api -GET /api/*path controllers.Application.proxy(path) -POST /api/*path controllers.Application.proxy(path) -DELETE /api/*path controllers.Application.proxy(path) -PUT /api/*path controllers.Application.proxy(path) +GET /api/*path controllers.Application.proxy(path) +POST /api/*path controllers.Application.proxy(path) +DELETE /api/*path controllers.Application.proxy(path) +PUT /api/*path controllers.Application.proxy(path) # Proxies API requests to the metadata service api -GET /openapi/*path controllers.Application.proxy(path) -POST /openapi/*path controllers.Application.proxy(path) -DELETE /openapi/*path controllers.Application.proxy(path) -PUT /openapi/*path controllers.Application.proxy(path) +GET /openapi/*path controllers.Application.proxy(path) +POST /openapi/*path controllers.Application.proxy(path) +DELETE /openapi/*path controllers.Application.proxy(path) +PUT /openapi/*path controllers.Application.proxy(path) # Map static resources from the /public folder to the /assets URL path -GET /assets/*file controllers.Assets.at(path="/public", file) +GET /assets/*file controllers.Assets.at(path="/public", file) # Analytics route -POST /track controllers.TrackingController.track() +POST /track controllers.TrackingController.track() # Wildcard route accepts any routes and delegates to serveAsset which in turn serves the React Bundle -GET /*path controllers.Application.index(path) +GET /*path controllers.Application.index(path) diff --git a/datahub-frontend/play.gradle b/datahub-frontend/play.gradle index ad1d7f59e8fa8..6b8517ca62a21 100644 --- a/datahub-frontend/play.gradle +++ b/datahub-frontend/play.gradle @@ -1,4 +1,4 @@ -apply plugin: "org.gradle.playframework" +apply plugin: 'play' // Change this to listen on a different port project.ext.httpPort = 9001 @@ -23,56 +23,42 @@ dependencies { play('org.springframework:spring-core:5.2.3.RELEASE') play('com.fasterxml.jackson.core:jackson-databind:2.9.10.4') play('com.nimbusds:nimbus-jose-jwt:7.9') - play('com.typesafe.akka:akka-actor_2.12:2.5.16') + play('com.typesafe.akka:akka-actor_2.11:2.5.16') play('net.minidev:json-smart:2.4.1') play('io.netty:netty-all:4.1.68.Final') } - compile project(":datahub-graphql-core") - compile project(":metadata-service:auth-api") - - implementation externalDependency.jettyJaas - implementation externalDependency.graphqlJava - implementation externalDependency.antlr4Runtime - implementation externalDependency.antlr4 - - implementation externalDependency.jerseyCore - implementation externalDependency.jerseyGuava - - implementation externalDependency.pac4j - implementation externalDependency.playPac4j - implementation externalDependency.shiroCore - implementation externalDependency.playCache - implementation externalDependency.playWs - implementation externalDependency.kafkaClients - - testImplementation externalDependency.mockito - testImplementation externalDependency.playTest - - compileOnly externalDependency.lombok - runtime externalDependency.guice - runtime externalDependency.playDocs - runtime externalDependency.playGuice - runtime externalDependency.logbackClassic - - annotationProcessor externalDependency.lombok -} - -dist.dependsOn(':datahub-web-react:copyAssets') - -play { - platform { - playVersion = '2.7.6' - scalaVersion = '2.12' - javaVersion = JavaVersion.VERSION_1_8 - } - - injectedRoutesGenerator = true + play project(":datahub-graphql-core") + play project(":metadata-service:auth-api") + + play externalDependency.jettyJaas + play externalDependency.graphqlJava + play externalDependency.antlr4Runtime + play externalDependency.antlr4 + + play externalDependency.jerseyCore + play externalDependency.jerseyGuava + + play externalDependency.pac4j + play externalDependency.playPac4j + play externalDependency.shiroCore + play externalDependency.playCache + play externalDependency.playWs + play externalDependency.kafkaClients + + playTest externalDependency.mockito + playTest externalDependency.playTest + + playRun externalDependency.lombok + playRun externalDependency.guice + playRun externalDependency.playDocs + playRun externalDependency.playGuice + playRun externalDependency.logbackClassic } model { components { play { - platform play: '2.7.6', scala: '2.12', java: '1.8' + platform play: '2.6.18', scala: '2.11', java: '1.8' injectedRoutesGenerator = true binaries.all { @@ -87,6 +73,11 @@ model { } } } + distributions { + playBinary { + baseName = project.ext.playBinaryBaseName + } + } } task unzipAssets(type: Copy, dependsOn: [configurations.assets, ':datahub-web-react:yarnBuild']) { @@ -101,14 +92,3 @@ task moveAssets(type: Copy, dependsOn: unzipAssets) { into "${buildDir}/assets" from ("${buildDir}/assets/assets") } - -clean { - delete 'public/platforms' - delete 'public/static' - delete 'public/asset-manifest.json' - delete 'public/manifest.json' - delete 'public/robots.txt' - delete 'public/logo.png' - delete 'public/index.html' - delete 'public/favicon.ico' -} diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle index 6e58e4aec13e9..28714fc4c9ecb 100644 --- a/datahub-web-react/build.gradle +++ b/datahub-web-react/build.gradle @@ -87,14 +87,6 @@ distZip { from 'dist' } -task copyAssets { - dependsOn distZip - copy { - from 'dist' - into '../datahub-frontend/public/' - } -} - if (!gradle.startParameter.taskNames.any { it in ["idea"] }) { artifacts { assets distZip diff --git a/docker/datahub-frontend/Dockerfile b/docker/datahub-frontend/Dockerfile index 723e20a237cd8..ee9ba4283cfb7 100644 --- a/docker/datahub-frontend/Dockerfile +++ b/docker/datahub-frontend/Dockerfile @@ -52,4 +52,4 @@ ENV JAVA_OPTS=" \ -Dlogback.configurationFile=datahub-frontend/conf/logback.xml \ -Dlogback.debug=false \ -Dpidfile.path=/dev/null" -CMD ["datahub-frontend/bin/datahub-frontend"] +CMD ["datahub-frontend/bin/playBinary"] diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 94b06cba59e91..fa50944e080be 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -55,4 +55,4 @@ services: args: APP_ENV: dev volumes: - - ../datahub-frontend/build/stage/playBinary:/datahub-frontend + - ../datahub-frontend/build/stage/datahub-frontend:/datahub-frontend