Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: versioned api #2736

Merged
merged 18 commits into from
Mar 22, 2021
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ functions:
MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} \
AUTH=${AUTH} SSL=${SSL} \
ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
- command: expansions.update
params:
Expand Down Expand Up @@ -125,8 +126,10 @@ functions:
rm -f ./prepare_client_encryption.sh
fi

AUTH=${AUTH} SSL=${SSL} UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" \
NODE_VERSION=${NODE_VERSION} SKIP_DEPS=1 NO_EXIT=1 \
MONGODB_URI="${MONGODB_URI}" \
AUTH=${AUTH} SSL=${SSL} UNIFIED=${UNIFIED} \
MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} NO_EXIT=${NO_EXIT|1} \
bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
run checks:
- command: shell.exec
Expand Down Expand Up @@ -780,6 +783,22 @@ tasks:
VERSION: '2.6'
TOPOLOGY: sharded_cluster
- func: run tests
- name: test-latest-server-v1-api
tags:
- latest
- server
- v1-api
commands:
- func: install dependencies
- func: bootstrap mongo-orchestration
vars:
VERSION: latest
TOPOLOGY: server
REQUIRE_API_VERSION: '1'
- func: run tests
vars:
MONGODB_API_VERSION: '1'
NO_EXIT: ''
- name: test-atlas-connectivity
tags:
- atlas-connect
Expand Down Expand Up @@ -1192,6 +1211,7 @@ buildvariants:
- test-2.6-server
- test-2.6-replica_set
- test-2.6-sharded_cluster
- test-latest-server-v1-api
- test-atlas-connectivity
- test-atlas-data-lake
- test-auth-kerberos
Expand Down Expand Up @@ -1258,6 +1278,7 @@ buildvariants:
- test-2.6-server
- test-2.6-replica_set
- test-2.6-sharded_cluster
- test-latest-server-v1-api
- test-atlas-connectivity
- test-atlas-data-lake
- test-auth-kerberos
Expand Down Expand Up @@ -1353,6 +1374,7 @@ buildvariants:
- test-3.2-server
- test-3.2-replica_set
- test-3.2-sharded_cluster
- test-latest-server-v1-api
- test-atlas-connectivity
- test-atlas-data-lake
- test-auth-kerberos
Expand Down
7 changes: 5 additions & 2 deletions .evergreen/config.yml.in
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ functions:
MONGODB_VERSION=${VERSION} TOPOLOGY=${TOPOLOGY} \
AUTH=${AUTH} SSL=${SSL} \
ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
# run-orchestration generates expansion file with the MONGODB_URI for the cluster
- command: expansions.update
Expand Down Expand Up @@ -145,8 +146,10 @@ functions:
rm -f ./prepare_client_encryption.sh
fi

AUTH=${AUTH} SSL=${SSL} UNIFIED=${UNIFIED} MONGODB_URI="${MONGODB_URI}" \
NODE_VERSION=${NODE_VERSION} SKIP_DEPS=1 NO_EXIT=1 \
MONGODB_URI="${MONGODB_URI}" \
AUTH=${AUTH} SSL=${SSL} UNIFIED=${UNIFIED} \
MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} NO_EXIT=${NO_EXIT|1} \
bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh

"run checks":
Expand Down
23 changes: 23 additions & 0 deletions .evergreen/generate_evergreen_tasks.js
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ const OPERATING_SYSTEMS = [
)
);

// TODO: NODE-3060: enable skipped tests on windows
const WINDOWS_SKIP_TAGS = new Set(['atlas-connect', 'auth']);

const TASKS = [];
Expand Down Expand Up @@ -87,6 +88,28 @@ const BASE_TASKS = [];
MONGODB_VERSIONS.forEach(mongoVersion => {
TOPOLOGIES.forEach(topology => BASE_TASKS.push(makeTask({ mongoVersion, topology })));
});
BASE_TASKS.push({
name: `test-latest-server-v1-api`,
tags: ['latest', 'server', 'v1-api'],
commands: [
{ func: 'install dependencies' },
{
func: 'bootstrap mongo-orchestration',
vars: {
VERSION: 'latest',
TOPOLOGY: 'server',
REQUIRE_API_VERSION: '1'
}
},
{
func: 'run tests',
vars: {
MONGODB_API_VERSION: '1',
NO_EXIT: ''
}
}
]
});

// manually added tasks
Array.prototype.push.apply(TASKS, [
Expand Down
2 changes: 1 addition & 1 deletion .evergreen/run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,4 @@ else
npm install mongodb-client-encryption@">=1.2.1"
fi

MONGODB_UNIFIED_TOPOLOGY=${UNIFIED} MONGODB_URI=${MONGODB_URI} npm run ${TEST_NPM_SCRIPT}
MONGODB_API_VERSION=${MONGODB_API_VERSION} MONGODB_UNIFIED_TOPOLOGY=${UNIFIED} MONGODB_URI=${MONGODB_URI} npm run ${TEST_NPM_SCRIPT}
11 changes: 10 additions & 1 deletion src/cmap/auth/scram.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import * as crypto from 'crypto';
import { Binary, Document } from '../../bson';
import { MongoError, AnyError } from '../../error';
import { AuthProvider, AuthContext } from './auth_provider';
import { Callback, emitWarning, ns } from '../../utils';
import { Callback, ns, applyServerApiVersion, emitWarning } from '../../utils';
import type { MongoCredentials } from './mongo_credentials';
import type { HandshakeDocument } from '../connect';

Expand Down Expand Up @@ -112,6 +112,9 @@ function executeScram(cryptoMethod: CryptoMethod, authContext: AuthContext, call
const db = credentials.source;

const saslStartCmd = makeFirstMessage(cryptoMethod, credentials, nonce);
if (connection.serverApi) {
applyServerApiVersion(saslStartCmd, connection.serverApi);
}
connection.command(ns(`${db}.$cmd`), saslStartCmd, undefined, (_err, result) => {
const err = resolveError(_err, result);
if (err) {
Expand Down Expand Up @@ -197,6 +200,9 @@ function continueScramConversation(
conversationId: response.conversationId,
payload: new Binary(Buffer.from(clientFinal))
};
if (connection.serverApi) {
applyServerApiVersion(saslContinueCmd, connection.serverApi);
}

connection.command(ns(`${db}.$cmd`), saslContinueCmd, undefined, (_err, r) => {
const err = resolveError(_err, r);
Expand All @@ -219,6 +225,9 @@ function continueScramConversation(
conversationId: r.conversationId,
payload: Buffer.alloc(0)
};
if (connection.serverApi) {
applyServerApiVersion(retrySaslContinueCmd, connection.serverApi);
}

connection.command(ns(`${db}.$cmd`), retrySaslContinueCmd, undefined, callback);
});
Expand Down
13 changes: 12 additions & 1 deletion src/cmap/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,14 @@ import { Connection, ConnectionOptions, CryptoConnection } from './connection';
import { MongoError, MongoNetworkError, MongoNetworkTimeoutError, AnyError } from '../error';
import { AUTH_PROVIDERS, AuthMechanism } from './auth/defaultAuthProviders';
import { AuthContext } from './auth/auth_provider';
import { makeClientMetadata, ClientMetadata, Callback, CallbackWithType, ns } from '../utils';
import {
makeClientMetadata,
ClientMetadata,
Callback,
CallbackWithType,
ns,
applyServerApiVersion
} from '../utils';
import {
MAX_SUPPORTED_WIRE_VERSION,
MAX_SUPPORTED_SERVER_VERSION,
Expand Down Expand Up @@ -96,6 +103,10 @@ function performInitialHandshake(
handshakeOptions.socketTimeout = options.connectTimeoutMS;
}

if (conn.serverApi) {
applyServerApiVersion(handshakeDoc, conn.serverApi);
}

const start = new Date().getTime();
conn.command(ns('admin.$cmd'), handshakeDoc, handshakeOptions, (err, response) => {
if (err) {
Expand Down
6 changes: 5 additions & 1 deletion src/cmap/connection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import { applyCommonQueryOptions, getReadPreference, isSharded } from './wire_pr
import { ReadPreference, ReadPreferenceLike } from '../read_preference';
import { isTransactionCommand } from '../transactions';
import type { W, WriteConcern, WriteConcernOptions } from '../write_concern';
import type { SupportedNodeConnectionOptions } from '../mongo_client';
import type { ServerApi, SupportedNodeConnectionOptions } from '../mongo_client';

const kStream = Symbol('stream');
const kQueue = Symbol('queue');
Expand Down Expand Up @@ -107,6 +107,7 @@ export interface ConnectionOptions
hostAddress: HostAddress;
// Settings
autoEncrypter?: AutoEncrypter;
serverApi?: ServerApi;
monitorCommands: boolean;
connectionType: typeof Connection;
credentials?: MongoCredentials;
Expand Down Expand Up @@ -136,6 +137,7 @@ export class Connection extends EventEmitter {
closed: boolean;
destroyed: boolean;
lastIsMasterMS?: number;
serverApi?: ServerApi;
/** @internal */
[kDescription]: StreamDescription;
/** @internal */
Expand Down Expand Up @@ -168,6 +170,7 @@ export class Connection extends EventEmitter {
this.address = streamIdentifier(stream);
this.socketTimeout = options.socketTimeout ?? 0;
this.monitorCommands = options.monitorCommands;
this.serverApi = options.serverApi;
this.closed = false;
this.destroyed = false;

Expand Down Expand Up @@ -362,6 +365,7 @@ export class Connection extends EventEmitter {
: new Query(cmdNs, finalCmd, commandOptions);

const inTransaction = session && (session.inTransaction() || isTransactionCommand(finalCmd));

const commandResponseHandler = inTransaction
? (err?: AnyError, ...args: Document[]) => {
// We need to add a TransientTransactionError errorLabel, as stated in the transaction spec.
Expand Down
12 changes: 11 additions & 1 deletion src/connection_string.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import {
MongoClient,
MongoClientOptions,
MongoOptions,
PkFactory
PkFactory,
ServerApi
} from './mongo_client';
import { MongoCredentials } from './cmap/auth/mongo_credentials';
import type { TagSet } from './sdam/server_description';
Expand Down Expand Up @@ -572,6 +573,15 @@ export const OPTIONS = {
autoEncryption: {
type: 'record'
},
serverApi: {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this mean we are allowing the user to specify the api version in the connection string? The spec specifically states not to allow it as we don't want developers changing behaviour without code changes.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes it seems that way, but I was unable to add the option to MongoClientOptions without adding an entry to this list of transformers; I'll take another look at this now.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I added a check to throw an error if it is provided in the URI options, and a corresponding test.

target: 'serverApi',
transform({ values: [version] }): ServerApi {
if (typeof version === 'string') {
return { version };
}
return version as ServerApi;
}
},
checkKeys: {
type: 'boolean'
},
Expand Down
10 changes: 8 additions & 2 deletions src/cursor/abstract_cursor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,10 @@ export abstract class AbstractCursor extends EventEmitter {
return;
}

if (!this[kSession]) {
throw new Error('Should have a session when calling getMore');
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This ended up being the cause of a bug I had a very hard time tracking down. It only happens on latest running with --setParameter requireApiVersion=1. I had to make the changes below to avoid this error being thrown in a number of tests. @nbbeeken might this be related to your sessions work?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting, it could be, but its not obvious at first since it doesn't go through the client startSession method, its going directly to the topology one. If you can reproduce it can you take a look at:

if (cursor[kSession] == null && cursor[kTopology].hasSessionSupport()) {
cursor[kSession] = cursor[kTopology].startSession({ owner: cursor, explicit: false });
}

This is where this[kSession] should be getting assigned so also printing the TopologyDescription with all the server descriptions as well could help

util.inspect(cursor[kTopology].s.description, {depth: 6}) (6 usually goes plenty deep)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok whoops, scrolled just a bit to see the changes you made to that section, that looks like the same issue to me, since you moved grabbing the session into the callback, then executeOperation has been called so server selection has run, now the function will return that sessions are supported. I am curious though what state the topology is in such that you're getting a failure, and then what state it transitions to after executeOperation runs.

}

server.getMore(
cursorNs,
cursorId,
Expand Down Expand Up @@ -620,15 +624,17 @@ function next(

if (cursorId == null) {
// All cursors must operate within a session, one must be made implicitly if not explicitly provided
if (cursor[kSession] == null && cursor[kTopology].hasSessionSupport()) {
if (cursor[kSession] == null /*&& cursor[kTopology].hasSessionSupport()*/) {
cursor[kSession] = cursor[kTopology].startSession({ owner: cursor, explicit: false });
}

cursor._initialize(cursor[kSession], (err, state) => {
if (state) {
const response = state.response;
cursor[kServer] = state.server;
cursor[kSession] = state.session;
if (state.session) {
cursor[kSession] = state.session;
}

if (response.cursor) {
cursor[kId] =
Expand Down
3 changes: 3 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,9 @@ export type {
Auth,
DriverInfo,
MongoOptions,
ServerApi,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah just noticed, we don't have tooling to catch this unfortunately but api-extractor is satisfied when this is exported as a type, but if we want users to be able to use the enum it has to be exported normally, you can see on line 76 is where I grouped the "enums".

Maybe a test/tooling we want to add (not here, just at some point) is importing index.js in our javascript tests and assert the actual imports we expect at runtime.

(@durran just pinging you here for knowledge share since we all might run into this gap in tooling)

ServerApiVersion,
ServerApiVersionId,
SupportedNodeConnectionOptions,
SupportedTLSConnectionOptions,
SupportedTLSSocketOptions,
Expand Down
35 changes: 35 additions & 0 deletions src/mongo_client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,32 @@ import type { SrvPoller } from './sdam/srv_polling';
import type { Connection } from './cmap/connection';
import type { LEGAL_TLS_SOCKET_OPTIONS, LEGAL_TCP_SOCKET_OPTIONS } from './cmap/connect';

/** @public */
export const LogLevel = {
error: 'error',
warn: 'warn',
info: 'info',
debug: 'debug'
} as const;

/** @public */
export const ServerApiVersion = {
emadum marked this conversation as resolved.
Show resolved Hide resolved
v1: '1'
};

/** @public */
export type ServerApiVersionId = typeof ServerApiVersion[keyof typeof ServerApiVersion];

/** @public */
export interface ServerApi {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The spec calls for this to be a class, but I didn't think it would be idiomatic in Node to make users import the ServerApi class and pass a new ServerVersion(...) in the constructor, versus accepting a plain object and providing a TypeScript interface. Does this make sense, or should it be a class like the spec advises?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree about it being idiomatic for nodejs
I guess just to consider an alternative we could make a class similar to our ReadPreference where we have static predefined instances of it? looking something like

new MongoClient('...', { serverApi: ServerApi.V1 })

Just putting this out there to say we considered it but I think avoiding an import is more satisfying for users.

version: string | ServerApiVersionId;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of freezing the ServerApi on the getter in MongoClient, could we instead just define the properties as readonly here? Or does that not give us enough protection somehow?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think readonly only applies at the level of the TypeScript compiler, so someone using plain JS would be able mutate the property. I'd be interested to hear your thoughts on this @nbbeeken

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed the readonly modifier only effects type checking. I think freezing is worth it here, since it is a construction time only setting, we can just jump ahead of any bugs trying to change serverApi at a later time by doing this.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok great, thanks!

strict?: boolean;
deprecationErrors?: boolean;
}

/** @public */
export type LogLevelId = typeof LogLevel[keyof typeof LogLevel];

/** @public */
export interface DriverInfo {
name?: string;
Expand Down Expand Up @@ -198,6 +224,8 @@ export interface MongoClientOptions extends BSONSerializeOptions, SupportedNodeC
logger?: Logger;
/** Enable command monitoring for this client */
monitorCommands?: boolean;
/** Server API version */
serverApi?: ServerApi | ServerApiVersionId;
/** Optionally enable client side auto encryption */
autoEncryption?: AutoEncryptionOptions;
/** Allows a wrapping driver to amend the client metadata generated by the driver to include information about the wrapping driver */
Expand All @@ -218,6 +246,7 @@ export interface MongoClientPrivate {
readConcern?: ReadConcern;
writeConcern?: WriteConcern;
readPreference: ReadPreference;
serverApi: ServerApi;
bsonOptions: BSONSerializeOptions;
namespace: MongoDBNamespace;
logger: Logger;
Expand Down Expand Up @@ -296,6 +325,7 @@ export class MongoClient extends EventEmitter {
readConcern: this[kOptions].readConcern,
writeConcern: this[kOptions].writeConcern,
readPreference: this[kOptions].readPreference,
serverApi: this[kOptions].serverApi,
emadum marked this conversation as resolved.
Show resolved Hide resolved
bsonOptions: resolveBSONOptions(this[kOptions]),
namespace: ns('admin'),
logger: this[kOptions].logger
Expand All @@ -306,6 +336,10 @@ export class MongoClient extends EventEmitter {
return Object.freeze({ ...this[kOptions] });
}

get serverApi(): Readonly<ServerApi | undefined> {
return this[kOptions].serverApi && Object.freeze({ ...this[kOptions].serverApi });
}

get autoEncrypter(): AutoEncrypter | undefined {
return this[kOptions].autoEncrypter;
}
Expand Down Expand Up @@ -601,6 +635,7 @@ export interface MongoOptions
credentials?: MongoCredentials;
readPreference: ReadPreference;
readConcern: ReadConcern;
serverApi: ServerApi;
writeConcern: WriteConcern;
dbName: string;
metadata: ClientMetadata;
Expand Down
Loading