-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor(restli): set threads based on cpu cores
feat(mce-consumers): hit local restli endpoint
- Loading branch information
1 parent
2734fd3
commit 549453d
Showing
31 changed files
with
543 additions
and
153 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -32,4 +32,5 @@ services: | |
| - "9090:9090" | ||
| depends_on: | ||
| - kafka-setup | ||
| - datahub-gms | ||
| - elasticsearch-setup | ||
| - mysql-setup | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -33,4 +33,5 @@ services: | |
| - "9090:9090" | ||
| depends_on: | ||
| - kafka-setup | ||
| - datahub-gms | ||
| - elasticsearch-setup | ||
| - mysql-setup | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
74 changes: 74 additions & 0 deletions
74
...a-jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/EbeanServerConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package com.linkedin.metadata.restli; | ||
|
|
||
| import io.ebean.datasource.DataSourceConfig; | ||
| import org.springframework.beans.factory.annotation.Qualifier; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Configuration; | ||
| import org.springframework.context.annotation.Primary; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
|
|
||
| import static com.linkedin.gms.factory.common.LocalEbeanServerConfigFactory.getListenerToTrackCounts; | ||
|
|
||
| @Configuration | ||
| public class EbeanServerConfig { | ||
| @Value("${ebean.username}") | ||
| private String ebeanDatasourceUsername; | ||
|
|
||
| @Value("${ebean.password}") | ||
| private String ebeanDatasourcePassword; | ||
|
|
||
| @Value("${ebean.driver}") | ||
| private String ebeanDatasourceDriver; | ||
|
|
||
| @Value("${ebean.minConnections:1}") | ||
| private Integer ebeanMinConnections; | ||
|
|
||
| @Value("${ebean.maxInactiveTimeSeconds:120}") | ||
| private Integer ebeanMaxInactiveTimeSecs; | ||
|
|
||
| @Value("${ebean.maxAgeMinutes:120}") | ||
| private Integer ebeanMaxAgeMinutes; | ||
|
|
||
| @Value("${ebean.leakTimeMinutes:15}") | ||
| private Integer ebeanLeakTimeMinutes; | ||
|
|
||
| @Value("${ebean.waitTimeoutMillis:1000}") | ||
| private Integer ebeanWaitTimeoutMillis; | ||
|
|
||
| @Value("${ebean.autoCreateDdl:false}") | ||
| private Boolean ebeanAutoCreate; | ||
|
|
||
| @Value("${ebean.postgresUseIamAuth:false}") | ||
| private Boolean postgresUseIamAuth; | ||
|
|
||
|
|
||
| @Bean("ebeanDataSourceConfig") | ||
| @Primary | ||
| public DataSourceConfig buildDataSourceConfig( | ||
| @Value("${ebean.url}") String dataSourceUrl, | ||
| @Qualifier("parseqEngineThreads") int ebeanMaxConnections | ||
| ) { | ||
| DataSourceConfig dataSourceConfig = new DataSourceConfig(); | ||
| dataSourceConfig.setUsername(ebeanDatasourceUsername); | ||
| dataSourceConfig.setPassword(ebeanDatasourcePassword); | ||
| dataSourceConfig.setUrl(dataSourceUrl); | ||
| dataSourceConfig.setDriver(ebeanDatasourceDriver); | ||
| dataSourceConfig.setMinConnections(ebeanMinConnections); | ||
| dataSourceConfig.setMaxConnections(ebeanMaxConnections); | ||
| dataSourceConfig.setMaxInactiveTimeSecs(ebeanMaxInactiveTimeSecs); | ||
| dataSourceConfig.setMaxAgeMinutes(ebeanMaxAgeMinutes); | ||
| dataSourceConfig.setLeakTimeMinutes(ebeanLeakTimeMinutes); | ||
| dataSourceConfig.setWaitTimeoutMillis(ebeanWaitTimeoutMillis); | ||
| dataSourceConfig.setListener(getListenerToTrackCounts("mce-consumer")); | ||
| // Adding IAM auth access for AWS Postgres | ||
| if (postgresUseIamAuth) { | ||
| Map<String, String> custom = new HashMap<>(); | ||
| custom.put("wrapperPlugins", "iam"); | ||
| dataSourceConfig.setCustomProperties(custom); | ||
| } | ||
| return dataSourceConfig; | ||
| } | ||
| } |
38 changes: 38 additions & 0 deletions
38
...jobs/mce-consumer-job/src/main/java/com/linkedin/metadata/restli/RestliServletConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package com.linkedin.metadata.restli; | ||
|
|
||
| import com.linkedin.entity.client.RestliEntityClient; | ||
| import com.linkedin.restli.client.Client; | ||
| import com.linkedin.restli.server.RestliHandlerServlet; | ||
| import org.springframework.beans.factory.annotation.Qualifier; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.boot.web.servlet.ServletRegistrationBean; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Configuration; | ||
| import org.springframework.context.annotation.Primary; | ||
|
|
||
| import java.net.URI; | ||
|
|
||
| @Configuration | ||
| public class RestliServletConfig { | ||
| @Value("${server.port}") | ||
| private int configuredPort; | ||
|
|
||
| @Bean("restliEntityClient") | ||
| @Primary | ||
| public RestliEntityClient restliEntityClient() { | ||
| String selfUri = String.format("http://localhost:%s/gms/", configuredPort); | ||
| final Client restClient = DefaultRestliClientFactory.getRestLiClient(URI.create(selfUri), null); | ||
| return new RestliEntityClient(restClient); | ||
| } | ||
|
|
||
| @Bean | ||
| public ServletRegistrationBean<RestliHandlerServlet> servletRegistrationBean( | ||
| @Qualifier("restliHandlerServlet") RestliHandlerServlet servlet) { | ||
| return new ServletRegistrationBean<>(servlet, "/gms/*"); | ||
| } | ||
|
|
||
| @Bean | ||
| public RestliHandlerServlet restliHandlerServlet() { | ||
| return new RestliHandlerServlet(); | ||
| } | ||
| } |
Oops, something went wrong.