Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test(cli): Add demo doc alignment test, serial implementation #2384

Merged
merged 3 commits into from
Aug 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"lint-fix": "eslint --fix .",
"lint:eslint": "eslint .",
"lint:types": "tsc",
"test": "exit 0"
"test": "ava"
},
"dependencies": {
"@endo/bundle-source": "^3.3.0",
Expand Down Expand Up @@ -57,6 +57,7 @@
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-eslint-comments": "^3.2.0",
"eslint-plugin-import": "^2.29.1",
"execa": "^9.3.0",
"prettier": "^3.2.5",
"typescript": "5.5.2"
},
Expand Down
18 changes: 18 additions & 0 deletions packages/cli/test/daemon-context.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/** @import { Context } from './types' */

/**
* Provides test setup and teardown hooks that purge the local endo
* daemon. In the future, we should create isolated daemon instances
* so that tests can be run in parallel.
*
* @type {Context}
*/
export const daemonContext = {
setup: async execa => {
await execa`endo purge -f`;
await execa`endo start`;
},
teardown: async execa => {
await execa`endo purge -f`;
},
};
9 changes: 9 additions & 0 deletions packages/cli/test/demo/confined-script.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
/** @import {TestRoutine} from '../types */

/** @type {TestRoutine} */
export const section = async (execa, testLine) => {
// If a runlet returns a promise for some value, it will print that value before exiting gracefully.
await testLine(execa`endo run runlet.js a b c`, {
stdout: "Hello, World! [ 'a', 'b', 'c' ]\n42",
});
};
60 changes: 60 additions & 0 deletions packages/cli/test/demo/counter-example.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/** @import {Context, TestRoutine} from '../types' */

/** @type {TestRoutine} */
export const section = async (execa, testLine) => {
// We can create an instance of the counter and give it a name.
await testLine(execa`endo make counter.js --name counter`, {
stdout: 'Object [Alleged: Counter] {}',
});

// Then, we can send messages to the counter and see their responses...
await testLine(execa`endo eval E(counter).incr() counter`, {
stdout: '1',
});
await testLine(execa`endo eval E(counter).incr() counter`, {
stdout: '2',
});
await testLine(execa`endo eval E(counter).incr() counter`, {
stdout: '3',
});

// Aside: in all the above cases, we use counter both as the property name...
await testLine(execa`endo eval E(c).incr() c:counter`, {
stdout: '4',
});

// Endo preserves the commands that led to the creation of the counter value...
await testLine(execa`endo restart`);
await testLine(execa`endo eval E(counter).incr() counter`, {
stdout: '1',
});
await testLine(execa`endo eval E(counter).incr() counter`, {
stdout: '2',
});
await testLine(execa`endo eval E(counter).incr() counter`, {
stdout: '3',
});

// Aside, since Eventual Send, the machinery under the E operator...
await testLine(execa`endo spawn greeter`);
await testLine(
execa`endo eval --worker greeter '${'Hello, World!'}' --name greeting`,
{
stdout: 'Hello, World!',
},
);
await testLine(execa`endo show greeting`, {
stdout: 'Hello, World!',
});
};

/** @type {Context} */
export const context = {
setup: async execa => {
await execa`endo make counter.js --name counter`;
await execa`endo eval E(counter).incr() counter`;
await execa`endo eval E(counter).incr() counter`;
await execa`endo eval E(counter).incr() counter`;
await execa`endo spawn greeter`;
},
};
52 changes: 52 additions & 0 deletions packages/cli/test/demo/doubler-agent.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/** @import {Context, TestRoutine} from '../types' */

/** @type {TestRoutine} */
export const section = async (execa, testLine) => {
// We make a doubler mostly the same way we made the counter...
await testLine(execa`endo mkguest doubler-handle doubler-agent`, {
stdout: 'Object [Alleged: EndoGuest] {}',
});
await testLine(
execa`endo make doubler.js --name doubler --powers doubler-agent`,
);

// This creates a doubler, but the doubler cannot respond until we resolve...
await testLine(execa`endo inbox`, {
stdout:
/^0\. "doubler-handle" requested "a counter, suitable for doubling"/,
});
await testLine(execa`endo resolve 0 counter`);

// Now we can get a response from the doubler.
await testLine(execa`endo eval E(doubler).incr() doubler`, {
stdout: '8',
});
await testLine(execa`endo eval E(doubler).incr() doubler`, {
stdout: '10',
});
await testLine(execa`endo eval E(doubler).incr() doubler`, {
stdout: '12',
});

// Also, in the optional second argument to request, doubler.js names...
await testLine(execa`endo restart`);
await testLine(execa`endo eval E(doubler).incr() doubler`, {
stdout: '2',
});
await testLine(execa`endo eval E(doubler).incr() doubler`, {
stdout: '4',
});
await testLine(execa`endo eval E(doubler).incr() doubler`, {
stdout: '6',
});
};

/** @type {Context} */
export const context = {
setup: async execa => {
await execa`endo mkguest doubler-handle doubler-agent`;
await execa`endo make doubler.js --name doubler --powers doubler-agent`;
await execa`endo inbox`;
await execa`endo resolve 0 counter`;
},
};
85 changes: 85 additions & 0 deletions packages/cli/test/demo/index.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
import test from 'ava';
import { $ } from 'execa';
import { makeSectionTest } from '../section.js';
import { withContext } from '../with-context.js';
import { daemonContext } from '../daemon-context.js';
import * as counterExample from './counter-example.js';
import * as doublerAgent from './doubler-agent.js';
import * as confinedScript from './confined-script.js';
import * as sendingMessages from './sending-messages.js';
import * as namesInTransit from './names-in-transit.js';
import * as mailboxesAreSymmetric from './mailboxes-are-symmetric.js';

test.serial(
'trivial',
makeSectionTest(
$({ cwd: 'demo' }),
withContext(daemonContext)(async (execa, testLine) => {
const maxim = 'a failing test is better than failure to test';
await testLine(execa`echo ${maxim}`, { stdout: maxim });
}),
),
);

test.serial(
'counter-example',
makeSectionTest(
$({ cwd: 'demo' }),
withContext(daemonContext)(counterExample.section),
),
);

test.serial(
'doubler-agent',
makeSectionTest(
$({ cwd: 'demo' }),
withContext(daemonContext, counterExample.context)(doublerAgent.section),
),
);

test.serial.failing(
'sending-messages',
makeSectionTest(
$({ cwd: 'demo' }),
withContext(
daemonContext,
counterExample.context,
doublerAgent.context,
)(sendingMessages.section),
),
);

test.serial.failing(
'names-in-transit',
makeSectionTest(
$({ cwd: 'demo' }),
withContext(
daemonContext,
counterExample.context,
doublerAgent.context,
sendingMessages.context,
)(namesInTransit.section),
),
);

test.serial.failing(
'mailboxes-are-symmetric',
makeSectionTest(
$({ cwd: 'demo' }),
withContext(
daemonContext,
counterExample.context,
doublerAgent.context,
sendingMessages.context,
namesInTransit.context,
)(mailboxesAreSymmetric.section),
),
);
grypez marked this conversation as resolved.
Show resolved Hide resolved

test.serial(
'confined-script',
makeSectionTest(
$({ cwd: 'demo' }),
withContext(daemonContext)(confinedScript.section),
),
);
14 changes: 14 additions & 0 deletions packages/cli/test/demo/mailboxes-are-symmetric.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/** @import {TestRoutine} from '../types' */

/** @type {TestRoutine} */
export const section = async (execa, testLine) => {
// Guests can also send their host messages...
await testLine(
execa`endo send HOST --as alice-agent ${'This is the @doubler you sent me.'}`,
);
await testLine(execa`endo inbox`, {
stdout: /^0\. "alice" sent "This is the @doubler you sent me\."/,
});
await testLine(execa`endo adopt 0 doubler doubler-from-alice`);
await testLine(execa`endo dismiss 0`);
};
27 changes: 27 additions & 0 deletions packages/cli/test/demo/names-in-transit.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/** @import {Context, TestRoutine} from '../types' */

/** @type {TestRoutine} */
export const section = async (execa, testLine) => {
// In this example, we send alice our "doubler" but let it appear...
await testLine(
execa`endo send alice ${'Please enjoy this @counter:doubler.'}`,
);
await testLine(execa`endo inbox --as alice-agent`, {
stdout: /^1\. "HOST" sent "Please enjoy this @counter\."/,
});
await testLine(execa`endo adopt --as alice-agent 1 counter --name redoubler`);
await testLine(execa`endo list --as alice-agent`, {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I imagine this is the first thing that fails in this test. There's an asymmetry in the API here: rather than endo list --as <agent>, you simply do endo list <agent>. Does that change cause these test cases to succeed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think your suggestion does what you think it does, but I want to limit the scope of this PR to correctly implementing a test of whether the expectations set in the demo's README.md match the observations from running the demo's commands. A future PR could make these tests pass either by changing the behavior of endo/cli or by altering the demo's expectations. In the former case it will be satisfying to associate the fixing change to the removal of failing in these tests. In the latter case it will be nice to have an example PR conjoining edits to the README.md with edits to these tests.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, although in this case it's the README that's wrong; --as was omitted from list in a deliberate design decision. I noticed the discrepancy myself a while back and even put up a PR for it (#2228), but we decided against making the change at the time. I don't think the situation is likely to change unless we complete a more thorough audit of each command for --as support, and until then I think it makes the most sense to update the readme and these tests.

Copy link
Contributor

@rekmarks rekmarks Aug 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(This comment is part of a thread, but may appear detached from it.) I don't think we actually resolved this point, but I'll let it go in exchange for you resurrecting #2228

Edit: I see you created #2409. This should be all set then. We ought to make these tests pass in order to close that issue. I also think we can resurrect #2228 if want to do so.

stdout: 'redoubler',
});
await testLine(execa`endo dismiss --as alice-agent 1`);
};

/** @type {Context} */
export const context = {
setup: async execa => {
await execa`endo send alice ${'Please enjoy this @counter:doubler.'}`;
await execa`endo inbox --as alice-agent`;
await execa`endo adopt --as alice-agent 1 counter --name redoubler`;
await execa`endo dismiss --as alice-agent 1`;
},
};
28 changes: 28 additions & 0 deletions packages/cli/test/demo/sending-messages.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/** @import {Context, TestRoutine} from '../types' */

/** @type {TestRoutine} */
export const section = async (execa, testLine) => {
// So far, we have run guest programs like the doubler. Guests and hosts can exchange messages...
await testLine(execa`endo mkguest alice alice-agent`, {
stdout: 'Object [Alleged: EndoGuest] {}',
});
await testLine(execa`endo send alice ${'Please enjoy this @doubler.'}`);
await testLine(execa`endo inbox --as alice-agent`, {
stdout: /^0\. "HOST" sent "Please enjoy this @doubler\."/,
});
await testLine(execa`endo adopt --as alice-agent 0 doubler`);
await testLine(execa`endo list --as alice-agent`, {
stdout: 'doubler',
});
await testLine(execa`endo dismiss --as alice-agent 0`);
};

/** @type {Context} */
export const context = {
setup: async execa => {
await execa`endo mkguest alice alice-agent`;
await execa`endo send alice ${'Please enjoy this @doubler.'}`;
await execa`endo adopt alice-agent 0 doubler`;
await execa`endo dismiss --as alice-agent 0`;
},
};
32 changes: 32 additions & 0 deletions packages/cli/test/section.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/** @import {Execa} from 'execa' */
/** @import {t} from 'ava' */
/** @import {TestRoutine} from './types' */

/**
* Transforms a testRoutine into an ava test.
* The testCommand function asserts that a given awaitable command produces the expected stdout and stderr.
*
* @param {Execa} execa - the command execution environment
* @param {TestRoutine} testRoutine - the test logic implementation
* @returns {(t: t) => Promise<void>}
*/
export function makeSectionTest(execa, testRoutine) {
return async t => {
const matchExpecation = (expectation, result, errMsg) => {
(expectation instanceof RegExp ? t.regex : t.is)(
result,
expectation ?? '',
errMsg,
);
};
const testCommand = async (command, expectation) => {
const result = await command;
if (expectation !== undefined) {
const errMsg = JSON.stringify({ expectation, result }, null, 2);
matchExpecation(expectation.stdout, result.stdout, errMsg);
matchExpecation(expectation.stderr, result.stderr, errMsg);
}
};
await testRoutine(execa, testCommand);
};
}
18 changes: 18 additions & 0 deletions packages/cli/test/types.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import type { Execa } from 'execa';

export type Expectation = {
stdout: RegExp | string | undefined;
stderr: RegExp | string | undefined;
};
export type TestCommand = (
command: ReturnType<Execa>,
expectation: Expectation,
) => Promise<true>;
export type TestRoutine = (
execa: Execa,
testCommnd: TestCommand,
) => Promise<void>;
export type Context = {
setup: (execa: Execa) => Promise<void>;
teardown?: (execa: Execa) => Promise<void>;
};
Loading
Loading