Skip to content

Commit

Permalink
ODP-2648: TEZ-4383: upgrade to mockito 4.3.1 (apache#190) (Laszlo Att…
Browse files Browse the repository at this point in the history 
…ila Toth reviewed by Laszlo Bodor) (#19)

(cherry picked from commit e5a5578)
(cherry picked from commit 5a05d43)
  • Loading branch information
@prabhjyotsingh @shubhluck
prabhjyotsingh authored and shubhluck committed Nov 21, 2024
1 parent 136dad5 commit 8e7a9ee
Show file tree
Hide file tree
Showing 79 changed files with 728 additions and 644 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -847,8 +847,8 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.10.8</version>
<artifactId>mockito-core</artifactId>
<version>4.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
Expand Down
2 changes: 1 addition & 1 deletion tez-api/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
68 changes: 33 additions & 35 deletions tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,14 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.isNull;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

Expand Down Expand Up @@ -89,9 +89,7 @@
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAMStatusRequestProto;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetAMStatusResponseProto;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusRequestProto;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.GetDAGStatusResponseProto;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.ShutdownSessionRequestProto;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.SubmitDAGRequestProto;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolRPC.TezAppMasterStatusProto;
import org.apache.tez.dag.api.records.DAGProtos.DAGStatusProto;
Expand Down Expand Up @@ -188,10 +186,11 @@ TezClientForTest configureAndCreateTezClient(Map<String, LocalResource> lrs, boo
YarnClient yarnClient = mock(YarnClient.class, RETURNS_DEEP_STUBS);
when(yarnClient.createApplication().getNewApplicationResponse().getApplicationId()).thenReturn(appId1);
when(yarnClient.getApplicationReport(appId1).getYarnApplicationState()).thenReturn(YarnApplicationState.NEW);
when(yarnClient.submitApplication(any(ApplicationSubmissionContext.class))).thenReturn(appId1);
when(yarnClient.submitApplication(any())).thenReturn(appId1);

DAGClientAMProtocolBlockingPB sessionAmProxy = mock(DAGClientAMProtocolBlockingPB.class, RETURNS_DEEP_STUBS);
when(sessionAmProxy.getAMStatus(any(RpcController.class), any(GetAMStatusRequestProto.class)))
when(sessionAmProxy.getAMStatus(any(), any()))
.thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.RUNNING).build())
.thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.RUNNING).build());

client.sessionAmProxy = sessionAmProxy;
Expand Down Expand Up @@ -255,7 +254,7 @@ private void _testTezClientSessionLargeDAGPlan(int maxIPCMsgSize, int payloadSiz
client.stop();

ArgumentCaptor<SubmitDAGRequestProto> captor = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
verify(client.sessionAmProxy).submitDAG((RpcController)any(), captor.capture());
verify(client.sessionAmProxy).submitDAG(any(), captor.capture());
SubmitDAGRequestProto request = captor.getValue();

if (shouldSerialize) {
Expand Down Expand Up @@ -308,12 +307,12 @@ public void testGetClient() throws Exception {
verify(client2.mockYarnClient, times(0)).submitApplication(captor.capture());

// Validate dag submission from second TezClient as normal */
verify(client2.sessionAmProxy, times(1)).submitDAG((RpcController)any(), (SubmitDAGRequestProto) any());
verify(client2.sessionAmProxy, times(1)).submitDAG(any(), any());

// Validate stop from new TezClient as normal */
client2.stop();
verify(client2.sessionAmProxy, times(1)).shutdownSession((RpcController) any(),
(ShutdownSessionRequestProto) any());
verify(client2.sessionAmProxy, times(1)).shutdownSession(any(),
any());
verify(client2.mockYarnClient, times(1)).stop();
/* END reuse of AM from new TezClient */
}
Expand All @@ -330,7 +329,7 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) thr
when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
.thenReturn(YarnApplicationState.RUNNING);
client.start();
verify(client.mockYarnClient, times(1)).init((Configuration)any());
verify(client.mockYarnClient, times(1)).init(any());
verify(client.mockYarnClient, times(1)).start();
if (isSession) {
verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
Expand All @@ -353,14 +352,18 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) thr
Vertex vertex = Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1,
Resource.newInstance(1, 1));
DAG dag = DAG.create("DAG").addVertex(vertex).addTaskLocalFiles(lrDAG);
if (!isSession) {
when(client.sessionAmProxy.getAMStatus(any(), any()))
.thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.SHUTDOWN).build());
}
DAGClient dagClient = client.submitDAG(dag);

assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString());

if (isSession) {
verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
verify(client.sessionAmProxy, times(1)).submitDAG((RpcController)any(), (SubmitDAGRequestProto) any());
verify(client.sessionAmProxy, times(1)).submitDAG(any(), any());
} else {
verify(client.mockYarnClient, times(1)).submitApplication(captor.capture());
ApplicationSubmissionContext context = captor.getValue();
Expand Down Expand Up @@ -399,7 +402,7 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) thr
assertEquals(dagClient.getSessionIdentifierString(), client.mockAppId.toString());
// additional resource is sent
ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
verify(client.sessionAmProxy, times(2)).submitDAG((RpcController)any(), captor1.capture());
verify(client.sessionAmProxy, times(2)).submitDAG(any(), captor1.capture());
SubmitDAGRequestProto proto = captor1.getValue();
Assert.assertEquals(1, proto.getAdditionalAmResources().getLocalResourcesCount());
Assert.assertEquals(lrName2, proto.getAdditionalAmResources().getLocalResources(0).getName());
Expand All @@ -426,8 +429,8 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop) thr
if(shouldStop) {
client.stop();
if (isSession) {
verify(client.sessionAmProxy, times(1)).shutdownSession((RpcController) any(),
(ShutdownSessionRequestProto) any());
verify(client.sessionAmProxy, times(1)).shutdownSession(any(),
any());
}
verify(client.mockYarnClient, times(1)).stop();
}
Expand All @@ -443,14 +446,14 @@ public void testPreWarm() throws Exception {
.thenReturn(YarnApplicationState.RUNNING);

when(
client.sessionAmProxy.getAMStatus((RpcController) any(), (GetAMStatusRequestProto) any()))
client.sessionAmProxy.getAMStatus(any(), any()))
.thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.READY).build());

PreWarmVertex vertex = PreWarmVertex.create("PreWarm", 1, Resource.newInstance(1, 1));
client.preWarm(vertex);

ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
verify(client.sessionAmProxy, times(1)).submitDAG((RpcController)any(), captor1.capture());
verify(client.sessionAmProxy, times(1)).submitDAG(any(), captor1.capture());
SubmitDAGRequestProto proto = captor1.getValue();
assertTrue(proto.getDAGPlan().getName().startsWith(TezConstants.TEZ_PREWARM_DAG_NAME_PREFIX));

Expand All @@ -467,7 +470,7 @@ public void testPreWarmCloseStuck() throws Exception {

when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
.thenReturn(YarnApplicationState.RUNNING);
when(client.sessionAmProxy.getAMStatus((RpcController) any(), (GetAMStatusRequestProto) any()))
when(client.sessionAmProxy.getAMStatus(any(), any()))
.thenReturn(GetAMStatusResponseProto.newBuilder().setStatus(TezAppMasterStatusProto.READY).build());

PreWarmVertex vertex = PreWarmVertex.create("PreWarm", 1, Resource.newInstance(1, 1));
Expand All @@ -480,7 +483,7 @@ public void testPreWarmCloseStuck() throws Exception {
private void setClientToReportStoppedDags(TezClientForTest client) throws Exception {
when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
.thenReturn(YarnApplicationState.FINISHED);
when(client.sessionAmProxy.getDAGStatus(isNull(RpcController.class), any(GetDAGStatusRequestProto.class)))
when(client.sessionAmProxy.getDAGStatus(isNull(), any()))
.thenReturn(GetDAGStatusResponseProto.newBuilder().setDagStatus(DAGStatusProto.newBuilder()
.addDiagnostics("Diagnostics_0").setState(DAGStatusStateProto.DAG_SUCCEEDED)
.setDAGProgress(ProgressProto.newBuilder()
Expand All @@ -502,8 +505,8 @@ public void testPreWarmWithTimeout() throws Exception {
spyClient.mockAppId).getYarnApplicationState())
.thenReturn(YarnApplicationState.RUNNING);
when(
spyClient.sessionAmProxy.getAMStatus((RpcController) any(),
(GetAMStatusRequestProto) any()))
spyClient.sessionAmProxy.getAMStatus(any(),
any()))
.thenReturn(
GetAMStatusResponseProto.newBuilder().setStatus(
TezAppMasterStatusProto.INITIALIZING).build());
Expand All @@ -518,15 +521,15 @@ public void testPreWarmWithTimeout() throws Exception {
endTime = Time.monotonicNow();
assertTrue("Time taken is not as expected",
(endTime - startTime) > timeout);
verify(spyClient, times(0)).submitDAG(any(DAG.class));
verify(spyClient, times(0)).submitDAG(any());
Assert.assertTrue("Unexpected Exception message",
te.getMessage().contains("Tez AM not ready"));

}

when(
spyClient.sessionAmProxy.getAMStatus((RpcController) any(),
(GetAMStatusRequestProto) any()))
spyClient.sessionAmProxy.getAMStatus(any(),
any()))
.thenReturn(
GetAMStatusResponseProto.newBuilder().setStatus(
TezAppMasterStatusProto.READY).build());
Expand All @@ -536,7 +539,7 @@ public void testPreWarmWithTimeout() throws Exception {
endTime = Time.monotonicNow();
assertTrue("Time taken is not as expected",
(endTime - startTime) <= timeout);
verify(spyClient, times(1)).submitDAG(any(DAG.class));
verify(spyClient, times(1)).submitDAG(any());
} catch (TezException te) {
fail("PreWarm should have succeeded!");
}
Expand Down Expand Up @@ -571,7 +574,7 @@ public void run() {
endTime = Time.monotonicNow();
assertTrue("Time taken is not as expected",
(endTime - startTime) <= timeout);
verify(spyClient, times(2)).submitDAG(any(DAG.class));
verify(spyClient, times(2)).submitDAG(any());
setClientToReportStoppedDags(client);
spyClient.stop();
client.stop();
Expand Down Expand Up @@ -926,8 +929,7 @@ public void testAMClientHeartbeat() throws Exception {
Thread.sleep(1000);
}
client.stop();
verify(client.sessionAmProxy, atLeast(3)).getAMStatus(any(RpcController.class),
any(GetAMStatusRequestProto.class));
verify(client.sessionAmProxy, atLeast(3)).getAMStatus(any(), any());

conf.setInt(TezConfiguration.TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS, -1);
final TezClientForTest client2 = configureAndCreateTezClient(conf);
Expand All @@ -940,10 +942,7 @@ public void testAMClientHeartbeat() throws Exception {
Thread.sleep(1000);
}
client2.stop();
verify(client2.sessionAmProxy, times(0)).getAMStatus(any(RpcController.class),
any(GetAMStatusRequestProto.class));


verify(client2.sessionAmProxy, times(0)).getAMStatus(any(), any());
}

@Test(timeout = 20000)
Expand Down Expand Up @@ -987,8 +986,7 @@ public void testAMHeartbeatFailOnGetAMStatus() throws Exception {
final TezClientForTest client = configureAndCreateTezClient(conf);
client.start();

when(client.sessionAmProxy.getAMStatus(any(RpcController.class),
any(GetAMStatusRequestProto.class))).thenThrow(new ServiceException("error"));
when(client.sessionAmProxy.getAMStatus(any(), any())).thenThrow(new ServiceException("error"));
client.callRealGetSessionAMProxy = true;
when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
.thenReturn(YarnApplicationState.FAILED);
Expand Down
4 changes: 2 additions & 2 deletions tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

package org.apache.tez.common.security;

import static org.mockito.Matchers.any;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
Expand Down Expand Up @@ -153,7 +153,7 @@ private MockFileSystem createFileSystemForServiceName(final String service)
throws IOException {
MockFileSystem mockFs = new MockFileSystem();
when(mockFs.getCanonicalServiceName()).thenReturn(service);
when(mockFs.getDelegationToken(any(String.class))).thenAnswer(
when(mockFs.getDelegationToken(any())).thenAnswer(
new Answer<Token<?>>() {
int unique = 0;
@Override
Expand Down
13 changes: 6 additions & 7 deletions tez-api/src/test/java/org/apache/tez/dag/api/TestEntityDescriptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,18 @@
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;

import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.tez.common.TezUtils;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;

import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;

public class TestEntityDescriptor {

Expand All @@ -57,7 +56,7 @@ public void testSingularWrite(InputDescriptor entityDescriptor, InputDescriptor
ByteArrayOutputStream bos = new ByteArrayOutputStream(out.getData().length);
bos.write(out.getData());

Mockito.verify(entityDescriptor).writeSingular(eq(out), any(ByteBuffer.class));
verify(entityDescriptor).writeSingular(eq(out), any());
deserialized.readFields(new DataInputStream(new ByteArrayInputStream(bos.toByteArray())));
verifyResults(entityDescriptor, deserialized, payload, confVal);
}
Expand All @@ -69,12 +68,12 @@ public void testSegmentedWrite(InputDescriptor entityDescriptor, InputDescriptor
entityDescriptor.write(out);
out.close();

Mockito.verify(entityDescriptor).writeSegmented(eq(out), any(ByteBuffer.class));
verify(entityDescriptor).writeSegmented(eq(out), any());
deserialized.readFields(new DataInputStream(new ByteArrayInputStream(bos.toByteArray())));
verifyResults(entityDescriptor, deserialized, payload, confVal);
}

@Test (timeout=1000)
@Test (timeout=3000)
public void testEntityDescriptorHadoopSerialization() throws IOException {
/* This tests the alternate serialization code path
* if the DataOutput is not DataOutputBuffer
Expand Down
Loading

0 comments on commit 8e7a9ee

Please sign in to comment.