Improve suite docs

packages/suite/API.md

@@ -4,6 +4,21 @@ Tools for working with BigTest test suites, including typings for all of the
 elements of a test suite, helpers to validate a suite, as well as a DSL for
 creating test suites.
 
+## Typings
+
+BigTest test suites are represented as a tree-like data structure. The exact
+format of this data structure is described by the types in this packages. There
+are three variations of this structure:
+
+- {@link Test}: the baseline, which describes the common structure of a test suite,
+  but does not include the ability to execute the test suite, or its results.
+- {@link TestImplementation}: has the same structure as {@link Test}, but also
+  includes the ability to execute the test suite. This is what is normally
+  exported from a test file.
+- {@link TestResult}: has the same structure as {@link Test}, but represents
+  the result of running the test suite, and includes the results of each step
+  and assertion as well as the aggregate results of test nodes.
+
 ## Using the DSL
 
 When using the DSL you will usually import the {@link test} function and use it

packages/suite/src/dsl.ts

@@ -82,8 +82,8 @@ export class TestBuilder<C extends Context> implements TestImplementation {
    * ### With description and action
    *
    * ``` typescript
-   * .step("do the thing", async() {
-   *   await thing.do();
+   * .step("click the new post link", async () {
+   *   await Link('New post').click();
    * });
    * ```
    *
@@ -92,15 +92,15 @@ export class TestBuilder<C extends Context> implements TestImplementation {
    * ``` typescript
    * .step(
    *   {
-   *     description: "do the thing",
-   *     action: async() {
-   *       await things.do();
+   *     description: "click the new post link",
+   *     action: async () {
+   *       await Link('New post').click();
    *     }
    *   },
    *   {
-   *     description: "another thing",
-   *     action: async() {
-   *       await anotherThing.do();
+   *     description: "fill in the text",
+   *     action: async () {
+   *       await TextField('Title').fillIn('An introduction to BigTest');
    *     }
    *   }
    * );
@@ -112,31 +112,39 @@ export class TestBuilder<C extends Context> implements TestImplementation {
    * they can be used like this:
    *
    * ``` typescript
+   * .step(Link('New post').click())
+   * .step(TextField('Title').fillIn('An introduction to BigTest'))
+   * ```
+   *
+   * Or like this if you prefer:
+   *
+   * ``` typescript
    * .step(
    *   Link('New post').click(),
-   *   TextField('Text').fillIn('BigTest is cool!'),
-   *   Button('Submit').click(),
-   *   Headline('BigTest is cool!').exists()
+   *   TextField('Title').fillIn('An introduction to BigTest'),
    * )
    * ```
    *
    * ### Returning from context
    *
+   * Returning an object from a step merges it into the context, and it can
+   * then be used in subsequent steps.
+   *
    * ``` typescript
-   * .step("given a user", async() {
+   * .step("given a user", async () {
    *   return { user: await User.create() }
    * })
-   * .step("and the user has a post", async({ user }) {
+   * .step("and the user has a post", async ({ user }) {
    *   return { post: await Post.create({ user }) }
    * })
-   * .step("when I visit the post", async({ user, post }) {
+   * .step("when I visit the post", async ({ user, post }) {
    *   await Page.visit(`/users/${user.id}/posts/${post.id}`);
    * })
    * ```
    *
    * @param description The description of this step
-   * @param action An async function which receives the current context and may return an object which extends the context
-   * @typeParam R The return type of the action function can either be
+   * @param action An async function which receives the current context and may return an object which extends the context. See {@link Action}.
+   * @typeParam R The return type of the action function can optionally be an object which will be merged into the context.
    */
   step<R extends Context | void>(description: string, action: Action<C,R>): TestBuilder<R extends void ? C : C & R>;
   /**
@@ -166,8 +174,69 @@ export class TestBuilder<C extends Context> implements TestImplementation {
     });
   }
 
-  assertion(...assertions: AssertionList<C>): TestBuilder<C>;
+  /**
+   * Add one or more assertions to this test. The arguments to this function
+   * should be either a description and a check, or one or more objects which
+   * have `description` and `check` properties. Interactor assertions can be
+   * used as an assertion directly, but interactor actions cannot.
+   *
+   * The check is an async function, if it completes without error, the
+   * assertion passes. Its return value is ignored.
+   *
+   * ### With description and check
+   *
+   * ``` typescript
+   * .assertion("the new post link is no longer shown", async () {
+   *   await Link('New post').absent();
+   * });
+   * ```
+   *
+   * ### With assertion objects
+   *
+   * ``` typescript
+   * .step(
+   *   {
+   *     description: "the new post link is no longer shown",
+   *     check: async () {
+   *       await Link('New post').absent();
+   *     }
+   *   },
+   *   {
+   *     description: "the heading has changed",
+   *     check: async () {
+   *       await Heading('Create a new post').exists();
+   *     }
+   *   }
+   * );
+   * ```
+   *
+   * ### With interactor
+   *
+   * Interactor assertions implement the assertion object interface, so they
+   * can be used like this:
+   *
+   * ``` typescript
+   * .assertion(Link('New post').absent())
+   * .assertion(Headline('Create a new post').exists())
+   * ```
+   *
+   * Or like this if you prefer:
+   *
+   * ``` typescript
+   * .assertion(
+   *   Link('New post').absent(),
+   *   Headline('Create a new post').exists()
+   * )
+   * ```
+   *
+   * @param description The description of this assertion
+   * @param check An async function which receives the context. See {@link Check}.
+   */
   assertion(description: string, check: Check<C>): TestBuilder<C>;
+  /**
+   * @param steps A list of assertion objects, each of which must have a `description` and `check` property.
+   */
+  assertion(...assertions: AssertionList<C>): TestBuilder<C>;
   assertion(...args: [string, Check<C>] | AssertionList<C>): TestBuilder<C> {
     if(this.state === 'child') {
       throw new TestStructureError(`Cannot add step after adding ${this.state}`);
@@ -191,6 +260,45 @@ export class TestBuilder<C extends Context> implements TestImplementation {
     }, 'assertion');
   }
 
+  /**
+   * Add a child test to this test. Child tests are executed after all of a
+   * tests steps have run. Each child test may in turn add steps and
+   * assertions.
+   *
+   * A callback function must be provided as the second argument. This function
+   * receives a {@link TestBuilder} which you can use to construct the child
+   * test.  The finished child test is then returned from the callback
+   * function.
+   *
+   * ### Example
+   *
+   * ``` typescript
+   * .child("signing in", (test) => {
+   *   return (
+   *     test
+   *       .step(TextField('Email').fillIn('jonas@example.com'))
+   *       .step(TextField('Password').fillIn('password123'))
+   *       .step(Button('Submit').click()
+   *   )
+   * });
+   * ```
+   *
+   * Or more consisely:
+   *
+   * ``` typescript
+   * .child("signing in", (test) => test
+   *   .step(TextField('Email').fillIn('[email protected]'))
+   *   .step(TextField('Password').fillIn('password123'))
+   *   .step(Button('Submit').click()
+   * );
+   * ```
+   *
+   * *Note: the odd signature using a callback function was chosen because it
+   * improves type checking and inference when using TypeScript.*
+   *
+   * @param description The description of this child test
+   * @param childFn a callback function
+   */
   child(description: string, childFn: (inner: TestBuilder<C>) => TestBuilder<Context>): TestBuilder<C> {
     let child = childFn(test(description));
     return new TestBuilder({

packages/suite/src/index.ts

@@ -1,3 +1,3 @@
 export * from './interfaces';
 export { test, TestBuilder } from './dsl';
-export { validateTest, MAXIMUM_DEPTH } from './validate-test';
+export { validateTest, TestValidationError, MAXIMUM_DEPTH } from './validate-test';

packages/suite/src/interfaces.ts

@@ -1,13 +1,19 @@
 /**
- * A tree of metadata describing a test and all of its children. By
- * design, this tree is as stripped down as possible so that it can be
- * seamlessly passed around from system to system. It does not include
- * any references to the functions which comprise actions and
- * assertions since they are not serializable, and cannot be shared
- * between processes.
+ * A common base type for various nodes in test trees.
  */
-export interface Test extends Node {
+export interface Node {
+  /** The human readable description of the test */
   description: string;
+}
+
+/**
+ * A tree which describes a test and all of its children. This interface
+ * describes the shape of a test without including any of the functions which
+ * comprise actions and assertions. This allows the test to be serialized and
+ * shared.
+ */
+export interface Test extends Node {
+  /** @hidden */
   path?: string;
   steps: Node[];
   assertions: Node[];
@@ -16,59 +22,51 @@ export interface Test extends Node {
 
 
 /**
- * A tree of tests that is like the `Test` interface in every way
+ * A tree that is like the {@link Test} interface in every way
  * except that it contains the actual steps and assertions that will
- * be run. Most of the time this interface is not necessary and
- * components of the system will be working with the `Test` API, but
- * in the case of the harness which actually consumes the test
- * implementation, and in the case of the DSL which produces the test
- * implementation, it will be needed.
+ * be run.
+ *
+ * It represents the full implementation of a test and is is what is normally
+ * exported from a test file.
  */
 export interface TestImplementation extends Test {
-  description: string;
-  path?: string;
   steps: Step[];
   assertions: Assertion[];
   children: TestImplementation[];
 }
 
+/**
+ * An `async` function that accepts the current test context. If it resolves to
+ * another context, that context will be merged into the current context,
+ * otherwise, the context will be left alone.
+ */
 export type Action = (context: Context) => Promise<Context | void>;
 
 /**
- * A single operation that is part of the test. It contains an Action
- * which is an `async` function that accepts the current test
- * context. If it resolves to another context, that context will be
- * merged into the current context, otherwise, the context will be
- * left alone.
+ * A step which forms part of a test. Steps are executed in sequence. If one
+ * step fails, subsequent steps will be disregarded. Once all steps complete
+ * successfully, any assertions will run.
  */
 export interface Step extends Node {
-  description: string;
   action: Action;
 }
 
 export type Check = (context: Context) => Promise<void>;
 
 /**
- * A single assertion that is part of a test case. It accepts the
- * current text context which has been built up to this point. It
- * should throw an exception if the test is failing. Any non-error
- * result will be considered a pass.
+ * A single assertion that is part of a test case. It accepts the current text
+ * context which has been built up to this point. It should throw an exception
+ * if the test is failing. Any non-error result will be considered a pass.
  */
 export interface Assertion extends Node {
-  description: string;
   check: Check;
 }
 
 /**
- * Passed down the line from step to step and to each assertion of a
- * test.
+ * Passed down the line from step to step and to each assertion of a test.
  */
 export type Context = Record<string, unknown>;
 
-interface Node {
-  description: string;
-}
-
 /**
  * State indicator for various results.
  * - pending: not yet evaluating
@@ -80,37 +78,43 @@ interface Node {
 export type ResultStatus = 'pending' | 'running' | 'failed' | 'ok' | 'disregarded';
 
 /**
- * Represents the result for a single test in the tree. A TestResult is ok even if
- * one of its children is not, as long as all of its own steps and assertions pass.
+ * Represents the result of running a {@link Test}. The status of the test is
+ * an aggregate of the steps and assertions it contains. Only if all steps and
+ * assertions pass is the test marked as `ok`.
+ *
+ * A TestResult is ok even if one of its children is not, as long as all of its
+ * own steps and assertions pass.
  */
 export interface TestResult extends Test {
-  description: string;
-  path?: string;
+  status: ResultStatus;
   steps: StepResult[];
   assertions: AssertionResult[];
   children: TestResult[];
-  status: ResultStatus;
 }
 
 /**
- * The result of a single step
+ * The result of a single {@link Step}.
  */
 export interface StepResult extends Node {
-  description: string;
   status: ResultStatus;
+  /** If the status was `failed` then this may provide further details about the cause of failure */
   error?: ErrorDetails;
+  /** True if the failure was caused by a timeout */
   timeout?: boolean;
+  /** Any log events which are generated through uncaught errors, or log messages written to the console */
   logEvents?: LogEvent[];
 }
 
 /**
- * The result of a single assertion
+ * The result of a single {@link Assertion}.
  */
 export interface AssertionResult extends Node {
-  description: string;
   status: ResultStatus;
+  /** If the status was `failed` then this may provide further details about the cause of failure */
   error?: ErrorDetails;
+  /** True if the failure was caused by a timeout */
   timeout?: boolean;
+  /** Any log events which are generated through uncaught errors, or log messages written to the console */
   logEvents?: LogEvent[];
 }