Skip to content

Commit

Permalink
Replaced BoundedThreadPool in HttpService since it was deprecated in …
Browse files Browse the repository at this point in the history 
…last jetty checked-in

Modified RemoteTest to take better style parameters and support reading keys from a text file
  • Loading branch information
@eliast
eliast committed May 29, 2009
1 parent 497e662 commit 79a6660
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 33 deletions.
1 change: 1 addition & 0 deletions .classpath
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,6 @@
<classpathentry kind="lib" path="lib/velocity-1.6.2.jar"/>
<classpathentry kind="lib" path="lib/commons-io-1.4.jar"/>
<classpathentry kind="lib" path="lib/commons-lang-2.4.jar"/>
<classpathentry kind="lib" path="lib/jopt-simple-3.1.jar"/>
<classpathentry kind="output" path="classes"/>
</classpath>
8 changes: 2 additions & 6 deletions bin/voldemort-remote-test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,6 @@
# limitations under the License.
#

if [ $# -lt 4 ];
then
echo 'USAGE: bin/voldemort-remote-test.sh bootstrap-url num-requests value-size start-number [rwd]'
exit 1
fi
bin_dir=$(dirname $0)

bin/run-class.sh voldemort.performance.RemoteTest ${1} ${2} ${3} ${4} ${5} ${6}
${bin_dir}/run-class.sh voldemort.performance.RemoteTest $@
Binary file added lib/jopt-simple-3.1.jar
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions src/java/voldemort/server/http/HttpService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.mortbay.jetty.nio.SelectChannelConnector;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.ServletHolder;
import org.mortbay.thread.BoundedThreadPool;
import org.mortbay.thread.QueuedThreadPool;

import voldemort.VoldemortException;
import voldemort.annotations.jmx.JmxGetter;
Expand Down Expand Up @@ -79,7 +79,7 @@ public void startInner() {
Connector connector = new SelectChannelConnector();
connector.setLowResourceMaxIdleTime(3000);
connector.setPort(this.port);
BoundedThreadPool threadPool = new BoundedThreadPool();
QueuedThreadPool threadPool = new QueuedThreadPool();
threadPool.setName("VoldemortHttp");
threadPool.setMaxThreads(this.numberOfThreads);
Server httpServer = new Server();
Expand Down
151 changes: 126 additions & 25 deletions test/integration/voldemort/performance/RemoteTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,21 @@

package voldemort.performance;

import static voldemort.utils.Utils.croak;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;

import joptsimple.OptionParser;
import joptsimple.OptionSet;
import voldemort.TestUtils;
import voldemort.client.ClientConfig;
import voldemort.client.SocketStoreClientFactory;
Expand All @@ -34,41 +42,112 @@ public class RemoteTest {

public static final int MAX_WORKERS = 8;

public static class KeyProvider {

private final List<String> keys;
private final AtomicInteger index;

public KeyProvider(int start, List<String> keys) {
this.index = new AtomicInteger(start);
this.keys = keys;
}

public String next() {
if(keys != null) {
return keys.get(index.getAndIncrement() % keys.size());
} else {
return Integer.toString(index.getAndIncrement());
}
}
}

public static void printUsage(PrintStream out, OptionParser parser) throws IOException {
out.println("Usage: $VOLDEMORT_HOME/bin/remote-test.sh \\");
out.println(" [options] bootstrapUrl storeName num-requests\n");
parser.printHelpOn(out);
System.exit(1);
}

public static void main(String[] args) throws Exception {
if(args.length < 4 || args.length > 5)
croak("USAGE: java " + RemoteTest.class.getName()
+ " url num_requests value_size start_num [rwd]");

System.err.println("Bootstraping cluster data.");
String url = args[0];
int numRequests = Integer.parseInt(args[1]);
int valueSize = Integer.parseInt(args[2]);
int startNum = Integer.parseInt(args[3]);
String ops = "rwd";
if(args.length > 4)
ops = args[4];
args = new String[] { "-r", "--request-file", "/Users/elias/dev/ads/voldemort/small_lcids",
"-d", "--", "tcp://localhost:6666", "test", "100" };

OptionParser parser = new OptionParser();
parser.accepts("r", "execute read operations").withOptionalArg();
parser.accepts("w", "execute write operations").withOptionalArg();
parser.accepts("d", "execute delete operations").withOptionalArg();
parser.accepts("randomize", "randomize operations via keys").withOptionalArg();
parser.accepts("request-file", "execute specific requests in order").withOptionalArg();
parser.accepts("start-key-index", "starting point when using int keys")
.withOptionalArg()
.ofType(Integer.class);
parser.accepts("value-size", "size in bytes for random value")
.withOptionalArg()
.ofType(Integer.class);

OptionSet options = parser.parse(args);

List<String> nonOptions = options.nonOptionArguments();
if(nonOptions.size() != 3) {
printUsage(System.err, parser);
}

String url = nonOptions.get(0);
String storeName = nonOptions.get(1);
int numRequests = Integer.parseInt(nonOptions.get(2));
int startNum = 0;
int valueSize = 1024;
String ops = "";
List<String> keys = null;

if(options.has("start-key-index")) {
startNum = (Integer) options.valueOf("start-key-index");
}

if(options.has("value-size")) {
startNum = (Integer) options.valueOf("value-size");
}

if(options.has("request-file")) {
keys = loadKeys((String) options.valueOf("request-file"));
}

if(options.has("r")) {
ops += "r";
}
if(options.has("w")) {
ops += "w";
}
if(options.has("d")) {
ops += "d";
}

if(ops.length() == 0) {
ops = "rwd";
}

System.err.println("Bootstraping cluster data.");
StoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setMaxThreads(20)
.setMaxConnectionsPerNode(MAX_WORKERS)
.setBootstrapUrls(url));
final StoreClient<String, String> store = factory.getStoreClient("test");
final StoreClient<String, String> store = factory.getStoreClient(storeName);

final String value = new String(TestUtils.randomBytes(valueSize));
ExecutorService service = Executors.newFixedThreadPool(MAX_WORKERS);

if(ops.contains("d")) {
System.err.println("Beginning delete test.");
final AtomicInteger count0 = new AtomicInteger(startNum);
final AtomicInteger successes = new AtomicInteger(0);
long start = System.currentTimeMillis();
final KeyProvider keyProvider0 = new KeyProvider(startNum, keys);
final CountDownLatch latch0 = new CountDownLatch(numRequests);
long start = System.currentTimeMillis();
for(int i = 0; i < numRequests; i++) {
service.execute(new Runnable() {

public void run() {
try {
String str = Integer.toString(count0.getAndIncrement());
store.delete(str);
store.delete(keyProvider0.next());
successes.getAndIncrement();
} catch(Exception e) {
e.printStackTrace();
Expand All @@ -87,16 +166,15 @@ public void run() {

if(ops.contains("w")) {
System.err.println("Beginning write test.");
final AtomicInteger count1 = new AtomicInteger(startNum);
long start = System.currentTimeMillis();
final KeyProvider keyProvider1 = new KeyProvider(startNum, keys);
final CountDownLatch latch1 = new CountDownLatch(numRequests);
long start = System.currentTimeMillis();
for(int i = 0; i < numRequests; i++) {
service.execute(new Runnable() {

public void run() {
try {
String str = Integer.toString(count1.getAndIncrement());
store.put(str, new Versioned<String>(value));
store.put(keyProvider1.next(), new Versioned<String>(value));
} catch(Exception e) {
e.printStackTrace();
} finally {
Expand All @@ -113,16 +191,15 @@ public void run() {

if(ops.contains("r")) {
System.err.println("Beginning read test.");
final KeyProvider keyProvider2 = new KeyProvider(startNum, keys);
final CountDownLatch latch2 = new CountDownLatch(numRequests);
long start = System.currentTimeMillis();
final AtomicInteger count2 = new AtomicInteger(startNum);
for(int i = 0; i < numRequests; i++) {
service.execute(new Runnable() {

public void run() {
try {
String str = Integer.toString(count2.getAndIncrement());
store.get(str);
store.get(keyProvider2.next());
} catch(Exception e) {
e.printStackTrace();
} finally {
Expand All @@ -140,4 +217,28 @@ public void run() {
System.exit(0);
}

public static List<String> loadKeys(String path) throws FileNotFoundException, IOException {

List<String> targets = new ArrayList<String>();
File file = new File(path);
BufferedReader reader = null;

try {
reader = new BufferedReader(new FileReader(file));
String text = null;
while((text = reader.readLine()) != null) {
targets.add(text);
}
} finally {
try {
if(reader != null) {
reader.close();
}
} catch(IOException e) {
e.printStackTrace();
}
}

return targets;
}
}

0 comments on commit 79a6660

Please sign in to comment.