Skip to content

Commit

Permalink
refactor(gatsby): refactor pagination in preparation to querying lmdb…
Browse files Browse the repository at this point in the history 
… directly (#32135)

* refactor(gatsby): apply limit/skip in datastore, not nodeModel

- remove firstOnly from fast filters
- split nodeModel.runQuery() to two functions: findOne() and findAll()

* fix fast filters tests

* Switch to { entries, totalCount } as a query result

* fix one more test

* refactor pagination

* Make totalCount async

* add a comment about stats usage
  • Loading branch information
@vladar
vladar authored Jun 30, 2021
1 parent 5649416 commit a2224ab
Show file tree
Hide file tree
Showing 8 changed files with 341 additions and 177 deletions.
62 changes: 27 additions & 35 deletions packages/gatsby/src/datastore/__tests__/run-fast-filters.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,24 +144,23 @@ describe(`fast filter tests`, () => {
},
}

const resultSingular = await runFastFiltersAndSort({
const { entries: resultSingular } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: true,
queryArgs: { ...queryArgs, limit: 1 },
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})

const resultMany = await runFastFiltersAndSort({
const { entries: resultMany, totalCount } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: false,
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})

expect(resultSingular.map(o => o.id)).toEqual([mockNodes()[1].id])
expect(resultMany.map(o => o.id)).toEqual([mockNodes()[1].id])
expect(await totalCount()).toEqual(1)
})

it(`eq operator honors type`, async () => {
Expand All @@ -171,25 +170,24 @@ describe(`fast filter tests`, () => {
},
}

const resultSingular = await runFastFiltersAndSort({
const { entries: resultSingular } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: true,
queryArgs: { ...queryArgs, limit: 1 },
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})

const resultMany = await runFastFiltersAndSort({
const { entries: resultMany, totalCount } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: false,
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})

// `id-1` node is not of queried type, so results should be empty
expect(resultSingular).toEqual([])
expect(resultMany).toEqual([])
expect(await totalCount()).toEqual(0)
})

it(`non-eq operator`, async () => {
Expand All @@ -199,18 +197,16 @@ describe(`fast filter tests`, () => {
},
}

const resultSingular = await runFastFiltersAndSort({
const { entries: resultSingular } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: true,
queryArgs: { ...queryArgs, limit: 1 },
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})

const resultMany = await runFastFiltersAndSort({
const { entries: resultMany, totalCount } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: false,
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})
Expand All @@ -220,13 +216,13 @@ describe(`fast filter tests`, () => {
mockNodes()[2].id,
mockNodes()[3].id,
])
expect(await totalCount()).toEqual(2)
})
it(`return empty array in case of empty nodes`, async () => {
const queryArgs = { filter: {}, sort: {} }
const resultSingular = await runFastFiltersAndSort({
const queryArgs = { filter: {}, sort: {}, limit: 1 }
const { entries: resultSingular } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: true,
nodeTypeNames: [`NonExistentNodeType`],
filtersCache: new Map(),
})
Expand All @@ -241,10 +237,9 @@ describe(`fast filter tests`, () => {
},
}

const resultSingular = await runFastFiltersAndSort({
const { entries: resultSingular } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: true,
queryArgs: { ...queryArgs, limit: 1 },
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})
Expand All @@ -263,16 +258,16 @@ describe(`fast filter tests`, () => {
},
}

const resultMany = await runFastFiltersAndSort({
const { entries: resultMany, totalCount } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: false,
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})

expect(Array.isArray(resultMany)).toBe(true)
expect(resultMany.length).toEqual(2)
expect(await totalCount()).toEqual(2)

resultMany.map(node => {
expect(node.slog).toEqual(`def`)
Expand All @@ -285,10 +280,9 @@ describe(`fast filter tests`, () => {
},
}

const resultSingular = await runFastFiltersAndSort({
const { entries: resultSingular } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: true,
queryArgs: { ...queryArgs, limit: 1 },
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})
Expand All @@ -307,16 +301,16 @@ describe(`fast filter tests`, () => {
},
}

const resultMany = await runFastFiltersAndSort({
const { entries: resultMany, totalCount } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: false,
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})

expect(Array.isArray(resultMany)).toBe(true)
expect(resultMany.length).toEqual(2)
expect(await totalCount()).toEqual(2)

resultMany.map(node => {
expect(node.deep.flat.search.chain).toEqual(300)
Expand All @@ -329,10 +323,9 @@ describe(`fast filter tests`, () => {
},
}

const resultSingular = await runFastFiltersAndSort({
const { entries: resultSingular } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: true,
queryArgs: { ...queryArgs, limit: 1 },
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})
Expand All @@ -347,10 +340,9 @@ describe(`fast filter tests`, () => {
},
}

const resultMany = await runFastFiltersAndSort({
const { entries: resultMany } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: false,
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})
Expand All @@ -367,16 +359,16 @@ describe(`fast filter tests`, () => {
},
}

const resultMany = await runFastFiltersAndSort({
const { entries: resultMany, totalCount } = await runFastFiltersAndSort({
gqlType,
queryArgs,
firstOnly: false,
nodeTypeNames: [gqlType.name],
filtersCache: new Map(),
})

expect(Array.isArray(resultMany)).toBe(true)
expect(resultMany.map(({ id }) => id)).toEqual([`id_2`, `id_3`])
expect(await totalCount()).toEqual(2)
})
})
})
Expand Down
49 changes: 22 additions & 27 deletions packages/gatsby/src/datastore/in-memory/run-fast-filters.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
IFilterCache,
} from "./indexing"
import { IGraphQLRunnerStats } from "../../query/types"
import { IQueryResult } from "../types"

// The value is an object with arbitrary keys that are either filter values or,
// recursively, an object with the same struct. Ie. `{a: {a: {a: 2}}}`
Expand All @@ -44,8 +45,9 @@ interface IRunFilterArg {
sort:
| { fields: Array<string>; order: Array<boolean | "asc" | "desc"> }
| undefined
skip?: number
limit?: number
}
firstOnly: boolean
resolvedFields: Record<string, any>
nodeTypeNames: Array<string>
filtersCache: FiltersCache
Expand Down Expand Up @@ -315,51 +317,51 @@ function collectBucketForElemMatch(
* Filters and sorts a list of nodes using mongodb-like syntax.
*
* @param args raw graphql query filter/sort as an object
* @property {boolean} args.firstOnly true if you want to return only the first
* result found. This will return a collection of size 1. Not a single element
* @property {{filter?: Object, sort?: Object} | undefined} args.queryArgs
* @property {{filter?: Object, sort?: Object, skip?: number, limit?: number} | undefined} args.queryArgs
* @property {FiltersCache} args.filtersCache A cache of indexes where you can
* look up Nodes grouped by a FilterCacheKey, which yields a Map which holds
* an arr of Nodes for the value that the filter is trying to query against.
* This object lives in query/query-runner.js and is passed down runQuery.
* @returns Collection of results. Collection will be limited to 1
* if `firstOnly` is true
* @returns Collection of results. Collection will be sliced by `skip` and `limit`
*/
export function runFastFiltersAndSort(
args: IRunFilterArg
): Array<IGatsbyNode> | null {
export function runFastFiltersAndSort(args: IRunFilterArg): IQueryResult {
const {
queryArgs: { filter, sort } = {},
queryArgs: { filter, sort, limit, skip = 0 } = {},
resolvedFields = {},
firstOnly = false,
nodeTypeNames,
filtersCache,
stats,
} = args

const result = convertAndApplyFastFilters(
filter,
firstOnly,
nodeTypeNames,
filtersCache,
resolvedFields,
stats
)

return sortNodes(result, sort, resolvedFields, stats)
const sortedResult = sortNodes(result, sort, resolvedFields, stats)
const totalCount = async (): Promise<number> => sortedResult.length

const entries =
skip || limit
? sortedResult.slice(skip, limit ? skip + (limit ?? 0) : undefined)
: sortedResult

return { entries, totalCount }
}

/**
* Return a collection of results. Collection will be limited to 1 if `firstOnly` is true
* Return a collection of results.
*/
function convertAndApplyFastFilters(
filterFields: Array<IInputQuery> | undefined,
firstOnly: boolean,
nodeTypeNames: Array<string>,
filtersCache: FiltersCache,
resolvedFields: Record<string, any>,
stats: IGraphQLRunnerStats
): Array<IGatsbyNode> | null {
): Array<IGatsbyNode> {
const filters = filterFields
? prefixResolvedFields(
createDbQueriesFromObject(prepareQueryArgs(filterFields)),
Expand Down Expand Up @@ -393,11 +395,7 @@ function convertAndApplyFastFilters(
// If there is no filter then the ensureCache step will populate this:
const cache = filterCache.meta.orderedByCounter as Array<IGatsbyNode>

if (firstOnly || cache.length) {
return cache.slice(0)
}

return null
return cache.slice(0)
}

const result = applyFastFilters(filters, nodeTypeNames, filtersCache)
Expand All @@ -406,9 +404,6 @@ function convertAndApplyFastFilters(
if (stats) {
stats.totalIndexHits++
}
if (firstOnly) {
return result.slice(0, 1)
}
return result
}

Expand Down Expand Up @@ -447,14 +442,14 @@ function filterToStats(
* Returns same reference as input, sorted inline
*/
function sortNodes(
nodes: Array<IGatsbyNode> | null,
nodes: Array<IGatsbyNode>,
sort:
| { fields: Array<string>; order: Array<boolean | "asc" | "desc"> }
| undefined,
resolvedFields: any,
stats: IGraphQLRunnerStats
): Array<IGatsbyNode> | null {
if (!sort || !nodes || nodes.length === 0) {
): Array<IGatsbyNode> {
if (!sort || sort.fields?.length === 0 || !nodes || nodes.length === 0) {
return nodes
}

Expand Down
5 changes: 5 additions & 0 deletions packages/gatsby/src/datastore/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ export interface ILmdbDatabases {
nodesByType: Database<NodeId, NodeType>
}

export interface IQueryResult {
entries: Iterable<IGatsbyNode>
totalCount: () => Promise<number>
}

// Note: this type is compatible with lmdb-store ArrayLikeIterable
export interface IGatsbyIterable<T> extends Iterable<T> {
[Symbol.iterator](): Iterator<T>
Expand Down
Loading

0 comments on commit a2224ab

Please sign in to comment.