From 1188dd6396ead7d52ce111fe7cee630e469ea2a5 Mon Sep 17 00:00:00 2001 From: Aswin V Date: Thu, 7 Mar 2024 01:44:14 +0530 Subject: [PATCH] Pagination fixes (#2347) * `offset` -> `pageOffset`, `limit`-> `pageLimit` * Be backward compatible in API * Cleanup types and handle pagination qs * Cleanup unused code * Import type * Cleanup and fix lint error * Align params for sso-tracer * Move parsing to a common util function * pageLimit shouldn't be optional * Cap pageLimit to max value, split the boolean * Revert typings and assert non null * Refactor var name * Use util function to normalize pagination params across getAll and getByIndex * Normalize offset/limit for dynamo/mongo * Update query params in `FederatedSAMLApps` * Cap to max limit if passed limit is 0 * Sync lock file * Add a 3rd record and supply opts.pageLimit * Normalize offset/limit for mem/redis * Save the 3rd record in the store * Fix getAll tests * Give precedence to standard params over legacy * Use util function * Parse using util function * Refactor * Standardise pagination for `api/v1/dsync/events` * Standardise pagination for api/admin/connections * Standardise pagination for api/admin/directory-sync * Standardise pagination for `api/v1/dsync/groups` * Standardise pagination for `v1/dsync/users`, `v1/dsync/product` * Standardise pagination in fetchByProduct APIs * Update swagger for groups * Fix pagination params definition, add the params for users api * More swagger updates * Swagger spec update for dsync events * Add pagination params to apis fetching by product * Update qs in internal-ui * Remove type assertion * [Swagger WIP] Fix response format for paginated APIs * Add dsync events to swagger spec * Fix swagger spec for sso tracer * Fix swagger spec for federated-saml apps of a product * Update pageLimit to 50 * Use pageLimit value from internal-ui * Update UI SDK * Cleanup local pagination component * Update swagger version * Remove unused keys from locale * Fix tag for trace api spec * Fix param name for swagger * Fix swagger tag for trace * updated package-lock * updated package-lock --------- Co-authored-by: Deepak Prabhakara --- components/Pagination.tsx | 44 --- components/connection/ConnectionList.tsx | 2 +- components/dsync/DirectoryList.tsx | 2 +- ee/federated-saml/api/admin/index.ts | 12 +- ee/federated-saml/api/v1/product.ts | 12 +- internal-ui/src/dsync/DirectoryGroups.tsx | 4 +- internal-ui/src/dsync/DirectoryUsers.tsx | 4 +- .../src/dsync/DirectoryWebhookLogs.tsx | 4 +- .../src/federated-saml/FederatedSAMLApps.tsx | 2 +- internal-ui/src/shared/Pagination.tsx | 2 +- internal-ui/src/sso-tracer/SSOTracers.tsx | 4 +- lib/utils.ts | 27 ++ locales/en/common.json | 3 - npm/src/controller/api.ts | 23 +- npm/src/controller/setup-link.ts | 6 + npm/src/db/dynamoDb.ts | 6 +- npm/src/db/mem.ts | 46 +-- npm/src/db/mongo.ts | 52 +-- npm/src/db/redis.ts | 70 ++-- npm/src/db/sql/sql.ts | 33 +- npm/src/db/utils.ts | 16 + .../directory-sync/scim/DirectoryConfig.ts | 37 +- npm/src/directory-sync/scim/Groups.ts | 19 +- npm/src/directory-sync/scim/Users.ts | 19 +- .../scim/WebhookEventsLogger.ts | 79 +++++ npm/src/ee/federated-saml/app.ts | 19 +- npm/src/sso-tracer/index.ts | 23 +- npm/test/db/db.test.ts | 82 +++-- pages/admin/retraced/projects/index.tsx | 2 +- pages/api/admin/connections/index.ts | 15 +- pages/api/admin/directory-sync/index.ts | 11 +- pages/api/admin/setup-links/index.ts | 12 +- pages/api/admin/sso-tracer/index.ts | 7 +- pages/api/v1/dsync/events/index.ts | 12 +- pages/api/v1/dsync/groups/index.ts | 12 +- pages/api/v1/dsync/product.ts | 12 +- pages/api/v1/dsync/setuplinks/product.ts | 12 +- pages/api/v1/dsync/users/index.ts | 12 +- pages/api/v1/sso-traces/product.ts | 17 +- pages/api/v1/sso/product.ts | 12 +- pages/api/v1/sso/setuplinks/product.ts | 12 +- swagger/swagger.json | 324 ++++++++++++++++-- types.ts | 8 - types/base.ts | 18 +- 44 files changed, 806 insertions(+), 344 deletions(-) delete mode 100644 components/Pagination.tsx delete mode 100644 types.ts diff --git a/components/Pagination.tsx b/components/Pagination.tsx deleted file mode 100644 index 31fb30a43..000000000 --- a/components/Pagination.tsx +++ /dev/null @@ -1,44 +0,0 @@ -import ArrowLeftIcon from '@heroicons/react/24/outline/ArrowLeftIcon'; -import ArrowRightIcon from '@heroicons/react/24/outline/ArrowRightIcon'; -import { useTranslation } from 'next-i18next'; -import { ButtonOutline } from './ButtonOutline'; - -type PaginationProps = { - itemsCount: number; - offset: number; - onPrevClick: () => void; - onNextClick: () => void; -}; - -export const pageLimit = 15; - -export const Pagination = ({ itemsCount, offset, onPrevClick, onNextClick }: PaginationProps) => { - const { t } = useTranslation('common'); - - // Hide pagination if there are no items to paginate. - if ((itemsCount === 0 && offset === 0) || (itemsCount < pageLimit && offset === 0)) { - return null; - } - - const prevDisabled = offset === 0; - const nextDisabled = itemsCount < pageLimit || itemsCount === 0; - - return ( -
- - {t('prev')} - - - {t('next')} - -
- ); -}; diff --git a/components/connection/ConnectionList.tsx b/components/connection/ConnectionList.tsx index ccdfa12fd..8ebdf7ad1 100644 --- a/components/connection/ConnectionList.tsx +++ b/components/connection/ConnectionList.tsx @@ -3,8 +3,8 @@ import { useTranslation } from 'next-i18next'; import { useRouter } from 'next/router'; import { LinkPrimary } from '@components/LinkPrimary'; import { InputWithCopyButton } from '@components/ClipboardButton'; -import { pageLimit } from '@components/Pagination'; import { ConnectionList } from '@boxyhq/react-ui/sso'; +import { pageLimit } from '@boxyhq/internal-ui'; const SSOConnectionList = ({ setupLinkToken, diff --git a/components/dsync/DirectoryList.tsx b/components/dsync/DirectoryList.tsx index dcd905700..d1f2ad712 100644 --- a/components/dsync/DirectoryList.tsx +++ b/components/dsync/DirectoryList.tsx @@ -2,8 +2,8 @@ import LinkIcon from '@heroicons/react/24/outline/LinkIcon'; import { useTranslation } from 'next-i18next'; import { LinkPrimary } from '@components/LinkPrimary'; import { useRouter } from 'next/router'; -import { pageLimit } from '@components/Pagination'; import { DirectoryList } from '@boxyhq/react-ui/dsync'; +import { pageLimit } from '@boxyhq/internal-ui'; const DSyncDirectoryList = ({ setupLinkToken }: { setupLinkToken?: string }) => { const { t } = useTranslation('common'); diff --git a/ee/federated-saml/api/admin/index.ts b/ee/federated-saml/api/admin/index.ts index 47a6c867f..d6888a092 100644 --- a/ee/federated-saml/api/admin/index.ts +++ b/ee/federated-saml/api/admin/index.ts @@ -1,6 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; const handler = async (req: NextApiRequest, res: NextApiResponse) => { const { method } = req; @@ -35,12 +36,13 @@ const handlePOST = async (req: NextApiRequest, res: NextApiResponse) => { const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { samlFederatedController } = await jackson(); - const { offset, limit, pageToken } = req.query as { offset: string; limit: string; pageToken?: string }; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); - const pageOffset = parseInt(offset); - const pageLimit = parseInt(limit); - - const apps = await samlFederatedController.app.getAll({ pageOffset, pageLimit, pageToken }); + const apps = await samlFederatedController.app.getAll({ + pageOffset, + pageLimit, + pageToken, + }); if (apps.pageToken) { res.setHeader('jackson-pagetoken', apps.pageToken); diff --git a/ee/federated-saml/api/v1/product.ts b/ee/federated-saml/api/v1/product.ts index cabb31a76..c1dd407dc 100644 --- a/ee/federated-saml/api/v1/product.ts +++ b/ee/federated-saml/api/v1/product.ts @@ -1,6 +1,7 @@ import { NextApiRequest, NextApiResponse } from 'next'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -22,17 +23,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { samlFederatedController } = await jackson(); - const { product, pageOffset, pageLimit, pageToken } = req.query as { + const { product } = req.query as { product: string; - pageOffset: string; - pageLimit: string; - pageToken?: string; }; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + const apps = await samlFederatedController.app.getByProduct({ product, - pageOffset: parseInt(pageOffset), - pageLimit: parseInt(pageLimit), + pageOffset, + pageLimit, pageToken, }); diff --git a/internal-ui/src/dsync/DirectoryGroups.tsx b/internal-ui/src/dsync/DirectoryGroups.tsx index a9e854449..26d710560 100644 --- a/internal-ui/src/dsync/DirectoryGroups.tsx +++ b/internal-ui/src/dsync/DirectoryGroups.tsx @@ -21,8 +21,8 @@ export const DirectoryGroups = ({ const { paginate, setPaginate, pageTokenMap } = usePaginate(router!); const params = { - offset: paginate.offset, - limit: pageLimit, + pageOffset: paginate.offset, + pageLimit, }; // For DynamoDB diff --git a/internal-ui/src/dsync/DirectoryUsers.tsx b/internal-ui/src/dsync/DirectoryUsers.tsx index b79a50cfe..e7141336d 100644 --- a/internal-ui/src/dsync/DirectoryUsers.tsx +++ b/internal-ui/src/dsync/DirectoryUsers.tsx @@ -21,8 +21,8 @@ export const DirectoryUsers = ({ const { paginate, setPaginate, pageTokenMap } = usePaginate(router!); const params = { - offset: paginate.offset, - limit: pageLimit, + pageOffset: paginate.offset, + pageLimit, }; // For DynamoDB diff --git a/internal-ui/src/dsync/DirectoryWebhookLogs.tsx b/internal-ui/src/dsync/DirectoryWebhookLogs.tsx index 96c72d3cd..694e28ff6 100644 --- a/internal-ui/src/dsync/DirectoryWebhookLogs.tsx +++ b/internal-ui/src/dsync/DirectoryWebhookLogs.tsx @@ -35,8 +35,8 @@ export const DirectoryWebhookLogs = ({ const { paginate, setPaginate, pageTokenMap } = usePaginate(router!); const params = { - offset: paginate.offset, - limit: pageLimit, + pageOffset: paginate.offset, + pageLimit, }; // For DynamoDB diff --git a/internal-ui/src/federated-saml/FederatedSAMLApps.tsx b/internal-ui/src/federated-saml/FederatedSAMLApps.tsx index 040d99a5c..e2d973498 100644 --- a/internal-ui/src/federated-saml/FederatedSAMLApps.tsx +++ b/internal-ui/src/federated-saml/FederatedSAMLApps.tsx @@ -37,7 +37,7 @@ export const FederatedSAMLApps = ({ const { t } = useTranslation('common'); const { paginate, setPaginate, pageTokenMap } = usePaginate(router!); - let getAppsUrl = `${urls.getApps}?offset=${paginate.offset}&limit=${pageLimit}`; + let getAppsUrl = `${urls.getApps}?pageOffset=${paginate.offset}&pageLimit=${pageLimit}`; // For DynamoDB if (paginate.offset > 0 && pageTokenMap[paginate.offset - pageLimit]) { diff --git a/internal-ui/src/shared/Pagination.tsx b/internal-ui/src/shared/Pagination.tsx index bd1323f80..53301d6d2 100644 --- a/internal-ui/src/shared/Pagination.tsx +++ b/internal-ui/src/shared/Pagination.tsx @@ -3,7 +3,7 @@ import ArrowLeftIcon from '@heroicons/react/24/outline/ArrowLeftIcon'; import ArrowRightIcon from '@heroicons/react/24/outline/ArrowRightIcon'; import { ButtonOutline } from './ButtonOutline'; -export const pageLimit = 15; +export const pageLimit = 50; export const Pagination = ({ itemsCount, diff --git a/internal-ui/src/sso-tracer/SSOTracers.tsx b/internal-ui/src/sso-tracer/SSOTracers.tsx index a87416c63..a9a117e6d 100644 --- a/internal-ui/src/sso-tracer/SSOTracers.tsx +++ b/internal-ui/src/sso-tracer/SSOTracers.tsx @@ -20,8 +20,8 @@ export const SSOTracers = ({ const { paginate, setPaginate, pageTokenMap, setPageTokenMap } = usePaginate(router!); const params = { - offset: paginate.offset, - limit: pageLimit, + pageOffset: paginate.offset, + pageLimit, }; // For DynamoDB diff --git a/lib/utils.ts b/lib/utils.ts index ad91bf090..cc23067ad 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -2,6 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import micromatch from 'micromatch'; import type { OIDCSSOConnectionWithDiscoveryUrl, OIDCSSOConnectionWithMetadata } from '@boxyhq/saml-jackson'; import { JacksonError } from 'npm/src/controller/error'; +import type { PaginateApiParams } from 'types'; export const validateEmailWithACL = (email: string) => { const NEXTAUTH_ACL = process.env.NEXTAUTH_ACL || undefined; @@ -73,3 +74,29 @@ export const oidcMetadataParse = ( } return body; }; + +export const parsePaginateApiParams = (params: NextApiRequest['query']): PaginateApiParams => { + let pageOffset, pageLimit; + + if ('pageOffset' in params) { + pageOffset = params.pageOffset; + } else if ('offset' in params) { + pageOffset = params.offset; + } + + if ('pageLimit' in params) { + pageLimit = params.pageLimit; + } else if ('limit' in params) { + pageLimit = params.limit; + } + + pageOffset = parseInt(pageOffset); + pageLimit = parseInt(pageLimit); + const pageToken = params.pageToken as string; + + return { + pageOffset, + pageLimit, + pageToken, + }; +}; diff --git a/locales/en/common.json b/locales/en/common.json index 61549275f..78d60b885 100644 --- a/locales/en/common.json +++ b/locales/en/common.json @@ -30,15 +30,12 @@ "name": "Name", "new_directory": "New Directory", "new_setup_link": "New Setup Link", - "next": "Next", "connections": "Connections", "new_connection": "New Connection", "no_projects_found": "No projects found.", "oidc": "OIDC", "open_menu": "Open menu", "open_sidebar": "Open sidebar", - "prev": "Prev", - "previous": "Previous", "product": "Product", "save_changes": "Save Changes", "saved": "Saved", diff --git a/npm/src/controller/api.ts b/npm/src/controller/api.ts index 83e20204c..65087d94c 100644 --- a/npm/src/controller/api.ts +++ b/npm/src/controller/api.ts @@ -806,12 +806,20 @@ export class ConnectionAPIController implements IConnectionAPIController { * type: object * description: OIDC IdP metadata * responses: - * '200Get': + * '200GetByProduct': * description: Success - * schema: - * type: array - * items: - * $ref: '#/definitions/Connection' + * content: + * application/json: + * schema: + * type: object + * properties: + * data: + * type: array + * items: + * $ref: '#/definitions/Connection' + * pageToken: + * type: string + * description: token for pagination * '400Get': * description: Please provide a `product`. * '401Get': @@ -821,11 +829,14 @@ export class ConnectionAPIController implements IConnectionAPIController { * summary: Get SSO Connections by product * parameters: * - $ref: '#/parameters/productParamGet' + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' * operationId: get-connections-by-product * tags: [Single Sign On] * responses: * '200': - * $ref: '#/responses/200Get' + * $ref: '#/responses/200GetByProduct' * '400': * $ref: '#/responses/400Get' * '401': diff --git a/npm/src/controller/setup-link.ts b/npm/src/controller/setup-link.ts index ed873fc55..5998f6ac3 100644 --- a/npm/src/controller/setup-link.ts +++ b/npm/src/controller/setup-link.ts @@ -393,6 +393,9 @@ export class SetupLinkController { * summary: Get the Setup Links by product * parameters: * - $ref: '#/parameters/productParamGet' + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' * operationId: get-sso-setup-link-by-product * tags: [Setup Links | Single Sign On] * responses: @@ -407,6 +410,9 @@ export class SetupLinkController { * summary: Get the Setup Links by product * parameters: * - $ref: '#/parameters/productParamGet' + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' * operationId: get-dsync-setup-link-by-product * tags: [Setup Links | Directory Sync] * responses: diff --git a/npm/src/db/dynamoDb.ts b/npm/src/db/dynamoDb.ts index 0c79f3471..685efe77a 100644 --- a/npm/src/db/dynamoDb.ts +++ b/npm/src/db/dynamoDb.ts @@ -183,6 +183,10 @@ class DynamoDB implements DatabaseDriver { } async getAll(namespace: string, _?: number, pageLimit?: number, pageToken?: string): Promise { + const { limit: Limit } = dbutils.normalizeOffsetAndLimit({ + pageLimit, + maxLimit: this.options.pageLimit!, + }); const res = await this.client.send( new QueryCommand({ KeyConditionExpression: 'namespace = :namespace', @@ -190,7 +194,7 @@ class DynamoDB implements DatabaseDriver { ':namespace': { S: namespace }, }, TableName: tableName, - Limit: pageLimit && pageLimit > 0 ? pageLimit : undefined, + Limit, ExclusiveStartKey: pageToken ? JSON.parse(Buffer.from(pageToken, 'base64').toString()) : undefined, }) ); diff --git a/npm/src/db/mem.ts b/npm/src/db/mem.ts index e94aae1ea..3e116f00f 100644 --- a/npm/src/db/mem.ts +++ b/npm/src/db/mem.ts @@ -59,14 +59,18 @@ class Mem implements DatabaseDriver { _?: string, sortOrder?: SortOrder ): Promise { - const offsetAndLimitValueCheck = !dbutils.isNumeric(pageOffset) && !dbutils.isNumeric(pageLimit); const returnValue: string[] = []; - const skip = Number(offsetAndLimitValueCheck ? 0 : pageOffset); - let take = Number(offsetAndLimitValueCheck ? this.options.pageLimit : pageLimit); + const { offset: skip, limit } = dbutils.normalizeOffsetAndLimit({ + pageOffset, + pageLimit, + maxLimit: this.options.pageLimit!, + }); + + let take = limit; let count = 0; - take += skip; + take! += skip!; if (namespace) { const index = dbutils.keyFromParts(dbutils.createdAtPrefix, namespace); @@ -79,11 +83,11 @@ class Mem implements DatabaseDriver { const iterator: IterableIterator = sortOrder === 'ASC' ? val.values() : val.reverse().values(); for (const value of iterator) { - if (count >= take) { + if (count >= take!) { break; } - if (count >= skip) { + if (count >= skip!) { returnValue.push(this.store[dbutils.keyFromParts(namespace, value)]); } @@ -103,30 +107,30 @@ class Mem implements DatabaseDriver { _?: string, sortOrder?: SortOrder ): Promise { - const offsetAndLimitValueCheck = !dbutils.isNumeric(pageOffset) && !dbutils.isNumeric(pageLimit); - const skip = Number(offsetAndLimitValueCheck ? 0 : pageOffset); + const { offset: skip, limit } = dbutils.normalizeOffsetAndLimit({ + pageOffset, + pageLimit, + maxLimit: this.options.pageLimit!, + }); + + let take = limit; - let take = Number(offsetAndLimitValueCheck ? this.options.pageLimit : pageLimit); let count = 0; - take += skip; + take! += skip!; const dbKeys = Array.from((await this.indexes[dbutils.keyForIndex(namespace, idx)]) || []) as string[]; const iterator: IterableIterator = sortOrder === 'ASC' ? dbKeys.values() : dbKeys.reverse().values(); const ret: string[] = []; for (const dbKey of iterator || []) { - if (offsetAndLimitValueCheck) { - ret.push(await this.get(namespace, dbKey)); - } else { - if (count >= take) { - break; - } - if (count >= skip) { - ret.push(await this.get(namespace, dbKey)); - } - - count++; + if (count >= take!) { + break; } + if (count >= skip!) { + ret.push(await this.get(namespace, dbKey)); + } + + count++; } return { data: ret }; diff --git a/npm/src/db/mongo.ts b/npm/src/db/mongo.ts index fb80e7063..6641ba381 100644 --- a/npm/src/db/mongo.ts +++ b/npm/src/db/mongo.ts @@ -86,10 +86,20 @@ class Mongo implements DatabaseDriver { _?: string, sortOrder?: SortOrder ): Promise { + const { offset: skip, limit } = dbutils.normalizeOffsetAndLimit({ + pageOffset, + pageLimit, + maxLimit: this.options.pageLimit!, + }); + const docs = await this.collection .find( { namespace: namespace }, - { sort: { createdAt: sortOrder === 'ASC' ? 1 : -1 }, skip: pageOffset, limit: pageLimit } + { + sort: { createdAt: sortOrder === 'ASC' ? 1 : -1 }, + skip, + limit, + } ) .toArray(); @@ -103,31 +113,31 @@ class Mongo implements DatabaseDriver { async getByIndex( namespace: string, idx: Index, - offset?: number, - limit?: number, + pageOffset?: number, + pageLimit?: number, // eslint-disable-next-line @typescript-eslint/no-unused-vars _?: string, sortOrder?: SortOrder ): Promise { const sort: Sort = { createdAt: sortOrder === 'ASC' ? 'asc' : 'desc' }; - const docs = - dbutils.isNumeric(offset) && dbutils.isNumeric(limit) - ? await this.collection - .find( - { - indexes: dbutils.keyForIndex(namespace, idx), - }, - { sort, skip: offset, limit: limit } - ) - .toArray() - : await this.collection - .find( - { - indexes: dbutils.keyForIndex(namespace, idx), - }, - { sort } - ) - .toArray(); + const { offset: skip, limit } = dbutils.normalizeOffsetAndLimit({ + pageOffset, + pageLimit, + maxLimit: this.options.pageLimit!, + }); + + const docs = await this.collection + .find( + { + indexes: dbutils.keyForIndex(namespace, idx), + }, + { + sort, + skip, + limit, + } + ) + .toArray(); const ret: string[] = []; for (const doc of docs || []) { diff --git a/npm/src/db/redis.ts b/npm/src/db/redis.ts index 52a59fd43..2edcea291 100644 --- a/npm/src/db/redis.ts +++ b/npm/src/db/redis.ts @@ -44,10 +44,14 @@ class Redis implements DatabaseDriver { _?: string, sortOrder?: SortOrder ): Promise { - const offset = pageOffset ? Number(pageOffset) : 0; - const count = pageLimit ? Number(pageLimit) : this.options.pageLimit; const sortedSetKey = dbutils.keyFromParts(dbutils.createdAtPrefix, namespace); + const { offset, limit: count } = dbutils.normalizeOffsetAndLimit({ + pageOffset, + pageLimit, + maxLimit: this.options.pageLimit!, + }); + const zRangeOptions = { BY: 'SCORE', REV: sortOrder === 'ASC', @@ -92,49 +96,43 @@ class Redis implements DatabaseDriver { // eslint-disable-next-line @typescript-eslint/no-unused-vars _?: string ): Promise { - const offsetAndLimitValueCheck = !dbutils.isNumeric(pageOffset) && !dbutils.isNumeric(pageLimit); - let take = Number(offsetAndLimitValueCheck ? this.options.pageLimit : pageLimit); - const skip = Number(offsetAndLimitValueCheck ? 0 : pageOffset); + const { offset: skip, limit } = dbutils.normalizeOffsetAndLimit({ + pageOffset, + pageLimit, + maxLimit: this.options.pageLimit!, + }); + let take = limit; + let count = 0; - take += skip; + take! += skip!; const returnValue: string[] = []; const keyArray: string[] = []; const idxKey = dbutils.keyForIndex(namespace, idx); const dbKeys = await this.client.sMembers(dbutils.keyFromParts(dbutils.indexPrefix, idxKey)); - if (!offsetAndLimitValueCheck) { - for await (const { value } of this.client.zScanIterator( - dbutils.keyFromParts(dbutils.createdAtPrefix, namespace), - count + 1 - )) { - if (dbKeys.indexOf(value) !== -1) { - if (count >= take) { - break; - } - if (count >= skip) { - keyArray.push(dbutils.keyFromParts(namespace, value)); - } - count++; + for await (const { value } of this.client.zScanIterator( + dbutils.keyFromParts(dbutils.createdAtPrefix, namespace), + count + 1 + )) { + if (dbKeys.indexOf(value) !== -1) { + if (count >= take!) { + break; } - } - if (keyArray.length > 0) { - const value = await this.client.MGET(keyArray); - for (let i = 0; i < value.length; i++) { - const valueObject = JSON.parse(value[i].toString()); - if (valueObject !== null && valueObject !== '') { - returnValue.push(valueObject); - } + if (count >= skip!) { + keyArray.push(dbutils.keyFromParts(namespace, value)); } + count++; } - return { data: returnValue || [] }; - } else { - const ret: string[] = []; - for (const dbKey of dbKeys || []) { - if (offsetAndLimitValueCheck) { - ret.push(await this.get(namespace, dbKey)); - } - } - return { data: ret }; } + if (keyArray.length > 0) { + const value = await this.client.MGET(keyArray); + for (let i = 0; i < value.length; i++) { + const valueObject = JSON.parse(value[i].toString()); + if (valueObject !== null && valueObject !== '') { + returnValue.push(valueObject); + } + } + } + return { data: returnValue || [] }; } async put(namespace: string, key: string, val: Encrypted, ttl = 0, ...indexes: any[]): Promise { diff --git a/npm/src/db/sql/sql.ts b/npm/src/db/sql/sql.ts index 64b681931..eee5a494e 100644 --- a/npm/src/db/sql/sql.ts +++ b/npm/src/db/sql/sql.ts @@ -171,7 +171,11 @@ class Sql implements DatabaseDriver { _?: string, sortOrder?: SortOrder ): Promise { - const skipOffsetAndLimitValue = !dbutils.isNumeric(pageOffset) && !dbutils.isNumeric(pageLimit); + const { offset: skip, limit: take } = dbutils.normalizeOffsetAndLimit({ + pageOffset, + pageLimit, + maxLimit: this.options.pageLimit!, + }); const res = await this.storeRepository.find({ where: { namespace: namespace }, @@ -179,8 +183,8 @@ class Sql implements DatabaseDriver { order: { ['createdAt']: sortOrder || 'DESC', }, - take: skipOffsetAndLimitValue ? this.options.pageLimit : pageLimit, - skip: skipOffsetAndLimitValue ? 0 : pageOffset, + take, + skip, }); return { data: res || [] }; @@ -195,21 +199,20 @@ class Sql implements DatabaseDriver { _?: string, sortOrder?: SortOrder ): Promise { - const skipOffsetAndLimitValue = !dbutils.isNumeric(pageOffset) && !dbutils.isNumeric(pageLimit); + const { offset: skip, limit: take } = dbutils.normalizeOffsetAndLimit({ + pageOffset, + pageLimit, + maxLimit: this.options.pageLimit!, + }); const sort = { id: sortOrder || 'DESC', }; - const res = skipOffsetAndLimitValue - ? await this.indexRepository.find({ - where: { key: dbutils.keyForIndex(namespace, idx) }, - order: sort, - }) - : await this.indexRepository.find({ - where: { key: dbutils.keyForIndex(namespace, idx) }, - take: skipOffsetAndLimitValue ? this.options.pageLimit : pageLimit, - skip: skipOffsetAndLimitValue ? 0 : pageOffset, - order: sort, - }); + const res = await this.indexRepository.find({ + where: { key: dbutils.keyForIndex(namespace, idx) }, + take, + skip, + order: sort, + }); const ret: Encrypted[] = []; if (res) { diff --git a/npm/src/db/utils.ts b/npm/src/db/utils.ts index 7de55993b..2627d058d 100644 --- a/npm/src/db/utils.ts +++ b/npm/src/db/utils.ts @@ -23,6 +23,22 @@ export const sleep = (ms: number): Promise => { export function isNumeric(num) { return !isNaN(num); } +export const normalizeOffsetAndLimit = ({ + pageLimit, + pageOffset, + maxLimit, +}: { + pageOffset?: number; + pageLimit?: number; + maxLimit: number; +}) => { + const skipOffset = pageOffset === undefined || !isNumeric(pageOffset); + // maxLimit capped to 50 by default unless set from env (db.options.pageLimit) + const capToMaxLimit = + pageLimit === undefined || pageLimit === 0 || !isNumeric(pageLimit) || pageLimit > maxLimit; + + return { offset: skipOffset ? 0 : pageOffset, limit: capToMaxLimit ? maxLimit : pageLimit }; +}; export const indexPrefix = '_index'; export const createdAtPrefix = '_createdAt'; export const modifiedAtPrefix = '_modifiedAt'; diff --git a/npm/src/directory-sync/scim/DirectoryConfig.ts b/npm/src/directory-sync/scim/DirectoryConfig.ts index 65d355fd8..5bbd9310c 100644 --- a/npm/src/directory-sync/scim/DirectoryConfig.ts +++ b/npm/src/directory-sync/scim/DirectoryConfig.ts @@ -133,6 +133,24 @@ export class DirectoryConfig { * in: query * required: false * type: string + * pageOffset: + * name: pageOffset + * description: Starting point from which the set of records are retrieved + * in: query + * required: false + * type: string + * pageLimit: + * name: pageLimit + * description: Number of records to be fetched for the page + * in: query + * required: false + * type: string + * pageToken: + * name: pageToken + * description: Token used for DynamoDB pagination + * in: query + * required: false + * type: string */ /** @@ -602,6 +620,9 @@ export class DirectoryConfig { * summary: Get directory connections by product * parameters: * - $ref: '#/parameters/product' + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' * tags: * - Directory Sync * produces: @@ -609,10 +630,18 @@ export class DirectoryConfig { * responses: * '200': * description: Success - * schema: - * type: array - * items: - * $ref: '#/definitions/Directory' + * content: + * application/json: + * schema: + * type: object + * properties: + * data: + * type: array + * items: + * $ref: '#/definitions/Directory' + * pageToken: + * type: string + * description: token for pagination */ public async filterBy( params: FilterByParams = {} diff --git a/npm/src/directory-sync/scim/Groups.ts b/npm/src/directory-sync/scim/Groups.ts index d14ba411f..06a5718a4 100644 --- a/npm/src/directory-sync/scim/Groups.ts +++ b/npm/src/directory-sync/scim/Groups.ts @@ -207,6 +207,9 @@ export class Groups extends Base { * - $ref: '#/parameters/tenant' * - $ref: '#/parameters/product' * - $ref: '#/parameters/directoryId' + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' * tags: * - Directory Sync * produces: @@ -214,10 +217,18 @@ export class Groups extends Base { * responses: * 200: * description: Success - * schema: - * type: array - * items: - * $ref: '#/definitions/Group' + * content: + * application/json: + * schema: + * type: object + * properties: + * data: + * type: array + * items: + * $ref: '#/definitions/Group' + * pageToken: + * type: string + * description: token for pagination */ public async getAll( params: PaginationParams & { diff --git a/npm/src/directory-sync/scim/Users.ts b/npm/src/directory-sync/scim/Users.ts index 10f20008a..22f4ed7c2 100644 --- a/npm/src/directory-sync/scim/Users.ts +++ b/npm/src/directory-sync/scim/Users.ts @@ -197,6 +197,9 @@ export class Users extends Base { * - $ref: '#/parameters/tenant' * - $ref: '#/parameters/product' * - $ref: '#/parameters/directoryId' + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' * tags: * - Directory Sync * produces: @@ -204,10 +207,18 @@ export class Users extends Base { * responses: * 200: * description: Success - * schema: - * type: array - * items: - * $ref: '#/definitions/User' + * content: + * application/json: + * schema: + * type: object + * properties: + * data: + * type: array + * items: + * $ref: '#/definitions/User' + * pageToken: + * type: string + * description: token for pagination */ public async getAll({ pageOffset, diff --git a/npm/src/directory-sync/scim/WebhookEventsLogger.ts b/npm/src/directory-sync/scim/WebhookEventsLogger.ts index 1157fe248..f5a901a02 100644 --- a/npm/src/directory-sync/scim/WebhookEventsLogger.ts +++ b/npm/src/directory-sync/scim/WebhookEventsLogger.ts @@ -14,6 +14,55 @@ type GetAllParams = PaginationParams & { directoryId?: string; }; +/** + * @swagger + * definitions: + * Event: + * type: object + * example: + * { + * "id": "id1", + * "webhook_endpoint": "https://example.com/webhook", + * "created_at": "2024-03-05T17:06:26.074Z", + * "status_code": 200, + * "delivered": true, + * "payload": { + * "directory_id": "58b5cd9dfaa39d47eb8f5f88631f9a629a232016", + * "event": "user.created", + * "tenant": "boxyhq", + * "product": "jackson", + * "data": { + * "id": "038e767b-9bc6-4dbd-975e-fbc38a8e7d82", + * "first_name": "Deepak", + * "last_name": "Prabhakara", + * "email": "deepak@boxyhq.com", + * "active": true, + * "raw": { + * "schemas": ["urn:ietf:params:scim:schemas:core:2.0:User"], + * "userName": "deepak@boxyhq.com", + * "name": { + * "givenName": "Deepak", + * "familyName": "Prabhakara" + * }, + * "emails": [ + * { + * "primary": true, + * "value": "deepak@boxyhq.com", + * "type": "work" + * } + * ], + * "title": "CEO", + * "displayName": "Deepak Prabhakara", + * "locale": "en-US", + * "externalId": "00u1ldzzogFkXFmvT5d7", + * "groups": [], + * "active": true, + * "id": "038e767b-9bc6-4dbd-975e-fbc38a8e7d82" + * } + * } + * } + * } + */ export class WebhookEventsLogger extends Base { constructor({ db }: { db: DatabaseStore }) { super({ db }); @@ -43,6 +92,36 @@ export class WebhookEventsLogger extends Base { return await this.eventStore().get(id); } + /** + * @swagger + * /api/v1/dsync/events: + * get: + * summary: Get event logs for a directory + * parameters: + * - $ref: '#/parameters/directoryId' + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' + * tags: + * - Directory Sync + * produces: + * - application/json + * responses: + * 200: + * description: Success + * content: + * application/json: + * schema: + * type: object + * properties: + * data: + * type: array + * items: + * $ref: '#/definitions/Event' + * pageToken: + * type: string + * description: token for pagination + */ // Get the event logs for a directory paginated public async getAll(params: GetAllParams = {}) { const { pageOffset, pageLimit, directoryId } = params; diff --git a/npm/src/ee/federated-saml/app.ts b/npm/src/ee/federated-saml/app.ts index 94f40bbbc..0556954b0 100644 --- a/npm/src/ee/federated-saml/app.ts +++ b/npm/src/ee/federated-saml/app.ts @@ -325,6 +325,9 @@ export class App { * in: query * required: true * type: string + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' * tags: * - Identity Federation * produces: @@ -332,10 +335,18 @@ export class App { * responses: * 200: * description: Success - * schema: - * type: array - * items: - * $ref: '#/definitions/SAMLFederationApp' + * content: + * application/json: + * schema: + * type: object + * properties: + * data: + * type: array + * items: + * $ref: '#/definitions/SAMLFederationApp' + * pageToken: + * type: string + * description: token for pagination */ public async getByProduct({ product, pageOffset, pageLimit, pageToken }: GetByProductParams) { await throwIfInvalidLicense(this.opts.boxyhqLicenseKey); diff --git a/npm/src/sso-tracer/index.ts b/npm/src/sso-tracer/index.ts index 42352e8b2..c486cef2f 100644 --- a/npm/src/sso-tracer/index.ts +++ b/npm/src/sso-tracer/index.ts @@ -113,7 +113,7 @@ class SSOTracer { * required: true * type: string * tags: - * - SAML Traces + * - SSO Traces * produces: * - application/json * responses: @@ -164,17 +164,28 @@ class SSOTracer { * summary: Get all traces for a product * parameters: * - $ref: '#/parameters/product' + * - $ref: '#/parameters/pageOffset' + * - $ref: '#/parameters/pageLimit' + * - $ref: '#/parameters/pageToken' * tags: - * - SAML Traces + * - SSO Traces * produces: * - application/json * responses: * '200': * description: Success - * schema: - * type: array - * items: - * $ref: '#/definitions/SSOTrace' + * content: + * application/json: + * schema: + * type: object + * properties: + * data: + * type: array + * items: + * $ref: '#/definitions/SSOTrace' + * pageToken: + * type: string + * description: token for pagination */ public async getTracesByProduct(params: GetByProductParams) { const { product, pageOffset, pageLimit, pageToken } = params; diff --git a/npm/test/db/db.test.ts b/npm/test/db/db.test.ts index 118c5c241..fe376b463 100644 --- a/npm/test/db/db.test.ts +++ b/npm/test/db/db.test.ts @@ -22,17 +22,24 @@ const record2 = { city: 'London', }; -const records = [record1, record2]; +const record3 = { + id: '3', + name: 'Samuel Jackson', + city: 'Delhi', +}; + +const records = [record1, record2, record3]; const memDbConfig = { engine: 'mem', ttl: 1, + pageLimit: 2, }; const redisDbConfig = { engine: 'redis', url: 'redis://localhost:6379', - pageLimit: 50, + pageLimit: 2, }; const postgresDbConfig = { @@ -41,11 +48,13 @@ const postgresDbConfig = { type: 'postgres', ttl: 1, cleanupLimit: 10, + pageLimit: 2, }; const mongoDbConfig = { engine: 'mongo', url: 'mongodb://localhost:27017/jackson', + pageLimit: 2, }; const mysqlDbConfig = { @@ -54,6 +63,7 @@ const mysqlDbConfig = { type: 'mysql', ttl: 1, cleanupLimit: 10, + pageLimit: 2, }; const planetscaleDbConfig = { @@ -61,6 +71,7 @@ const planetscaleDbConfig = { url: process.env.PLANETSCALE_URL, ttl: 1, cleanupLimit: 10, + pageLimit: 2, // ssl: { // rejectUnauthorized: true, // }, @@ -72,6 +83,7 @@ const mariadbDbConfig = { type: 'mariadb', ttl: 1, cleanupLimit: 10, + pageLimit: 2, }; const mssqlDbConfig = { @@ -80,6 +92,7 @@ const mssqlDbConfig = { url: 'sqlserver://localhost:1433;database=master;username=sa;password=123ABabc!', ttl: 1, cleanupLimit: 10, + pageLimit: 2, }; const dynamoDbConfig = { @@ -87,6 +100,7 @@ const dynamoDbConfig = { url: process.env.DYNAMODB_URL, ttl: 1, cleanupLimit: 10, + pageLimit: 2, dynamodb: { region: 'us-east-1', readCapacityUnits: 5, @@ -229,6 +243,24 @@ tap.test('dbs', async () => { value: record2.name, } ); + + // wait 100ms to ensure that the record is written with a different timestamp + await new Promise((resolve) => setTimeout(resolve, 100)); + + await connectionStore.put( + record3.id, + record3, + { + // secondary index on city + name: 'city', + value: record3.city, + }, + { + // secondary index on name + name: 'name', + value: record3.name, + } + ); }); tap.test('get(): ' + dbType, async (t) => { @@ -240,33 +272,23 @@ tap.test('dbs', async () => { }); tap.test('getAll(): ' + dbType, async (t) => { - const allRecords = await connectionStore.getAll(); - const allRecordOutput = {}; - let allRecordInput = {}; - for (const keyValue in records) { - const keyVal = records[keyValue.toString()]; - allRecordOutput[keyVal]; - } - for (const keyValue in allRecords.data) { - const keyVal = records[keyValue.toString()]; - allRecordInput[allRecords.data[keyVal]]; - } - t.same(allRecordInput, allRecordOutput, 'unable to getAll records'); - allRecordInput = {}; - let allRecordsWithPagination = await connectionStore.getAll(0, 2); - for (const keyValue in allRecordsWithPagination.data) { - const keyVal = records[keyValue.toString()]; - allRecordInput[allRecordsWithPagination.data[keyVal]]; - } - - t.same(allRecordInput, allRecordOutput, 'unable to getAll records'); - allRecordsWithPagination = await connectionStore.getAll(0, 0); - for (const keyValue in allRecordsWithPagination.data) { - const keyVal = records[keyValue.toString()]; - allRecordInput[allRecordsWithPagination.data[keyVal]]; - } - - t.same(allRecordInput, allRecordOutput, 'unable to getAll records'); + const testMessage = + dbType === 'dynamodb' // dynamodb doesn't support sort order + ? 'should return all the records upto options.pageLimit' + : 'should return all the records upto options.pageLimit in DESC order by creation time'; + const wanted = dbType === 'dynamodb' ? records.slice(1, 3) : [...records].reverse().slice(0, 2); + // getAll without pagination params + t.same((await connectionStore.getAll()).data, wanted, `without pagination params ` + testMessage); + // getAll with pagination params + t.same((await connectionStore.getAll(0, 2)).data, wanted, `with pagination params ` + testMessage); + // getAll with pageLimit set to 0 + t.same((await connectionStore.getAll(0, 0)).data, wanted, `with pageLimit set to 0 ` + testMessage); + // getAll with pageLimit > options.pageLimit + t.same( + (await connectionStore.getAll(0, 3)).data, + wanted, + `with pageLimit > options.pageLimit ` + testMessage + ); const oneRecordWithPagination = await connectionStore.getAll(0, 1); t.same( @@ -291,7 +313,7 @@ tap.test('dbs', async () => { t.match(sortedRecordsAsc, [record1, record2], 'records are sorted in ASC order'); const { data: sortedRecordsDesc } = await connectionStore.getAll(0, 2, undefined, 'DESC'); - t.match(sortedRecordsDesc, [record2, record1], 'records are sorted in DESC order'); + t.match(sortedRecordsDesc, [record3, record2], 'records are sorted in DESC order'); } }); diff --git a/pages/admin/retraced/projects/index.tsx b/pages/admin/retraced/projects/index.tsx index 02bd5fe31..dd12fb5f5 100644 --- a/pages/admin/retraced/projects/index.tsx +++ b/pages/admin/retraced/projects/index.tsx @@ -7,7 +7,7 @@ import { useProjects } from '@lib/ui/retraced'; import Loading from '@components/Loading'; import { useTranslation } from 'next-i18next'; import router from 'next/router'; -import { Pagination, pageLimit } from '@components/Pagination'; +import { Pagination, pageLimit } from '@boxyhq/internal-ui'; import usePaginate from '@lib/ui/hooks/usePaginate'; import { LinkPrimary } from '@components/LinkPrimary'; import { errorToast } from '@components/Toaster'; diff --git a/pages/api/admin/connections/index.ts b/pages/api/admin/connections/index.ts index 7d8d98265..72559d45d 100644 --- a/pages/api/admin/connections/index.ts +++ b/pages/api/admin/connections/index.ts @@ -1,7 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import jackson from '@lib/jackson'; -import { oidcMetadataParse, strategyChecker } from '@lib/utils'; +import { oidcMetadataParse, parsePaginateApiParams, strategyChecker } from '@lib/utils'; import { adminPortalSSODefaults } from '@lib/env'; const handler = async (req: NextApiRequest, res: NextApiResponse) => { @@ -26,20 +26,15 @@ const handler = async (req: NextApiRequest, res: NextApiResponse) => { const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { adminController, connectionAPIController } = await jackson(); - const { pageOffset, pageLimit, isSystemSSO, pageToken } = req.query as { - pageOffset: string; - pageLimit: string; + const { isSystemSSO } = req.query as { isSystemSSO?: string; // if present will be '' else undefined - pageToken?: string; }; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + const { tenant: adminPortalSSOTenant, product: adminPortalSSOProduct } = adminPortalSSODefaults; - const paginatedConnectionList = await adminController.getAllConnection( - +(pageOffset || 0), - +(pageLimit || 0), - pageToken - ); + const paginatedConnectionList = await adminController.getAllConnection(pageOffset, pageLimit, pageToken); const connections = isSystemSSO === undefined diff --git a/pages/api/admin/directory-sync/index.ts b/pages/api/admin/directory-sync/index.ts index e56d70f17..800df4770 100644 --- a/pages/api/admin/directory-sync/index.ts +++ b/pages/api/admin/directory-sync/index.ts @@ -1,6 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import type { DirectoryType } from '@boxyhq/saml-jackson'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; const handler = async (req: NextApiRequest, res: NextApiResponse) => { const { method } = req; @@ -43,19 +44,15 @@ const handlePOST = async (req: NextApiRequest, res: NextApiResponse) => { const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { directorySyncController } = await jackson(); - const { pageOffset, pageLimit, pageToken } = req.query as { - pageOffset: string; - pageLimit: string; - pageToken?: string; - }; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); const { data, error, pageToken: nextPageToken, } = await directorySyncController.directories.getAll({ - pageOffset: +(pageOffset || 0), - pageLimit: +(pageLimit || 0), + pageOffset, + pageLimit, pageToken, }); diff --git a/pages/api/admin/setup-links/index.ts b/pages/api/admin/setup-links/index.ts index 42fbb133f..a48942ee0 100644 --- a/pages/api/admin/setup-links/index.ts +++ b/pages/api/admin/setup-links/index.ts @@ -1,5 +1,6 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; const handler = async (req: NextApiRequest, res: NextApiResponse) => { const { method } = req; @@ -45,14 +46,13 @@ const handleDELETE = async (req: NextApiRequest, res: NextApiResponse) => { const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { setupLinkController } = await jackson(); - const { token, service, pageOffset, pageLimit, pageToken } = req.query as { - pageOffset: string; - pageLimit: string; - pageToken?: string; + const { token, service } = req.query as { token: string; service: string; }; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + if (!token && !service) { return res.status(404).json({ error: { @@ -73,8 +73,8 @@ const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { if (service) { const setupLinksPaginated = await setupLinkController.filterBy({ service: service as any, - pageLimit: parseInt(pageLimit), - pageOffset: parseInt(pageOffset), + pageOffset, + pageLimit, pageToken, }); diff --git a/pages/api/admin/sso-tracer/index.ts b/pages/api/admin/sso-tracer/index.ts index a01c73a77..1097996f1 100644 --- a/pages/api/admin/sso-tracer/index.ts +++ b/pages/api/admin/sso-tracer/index.ts @@ -1,6 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import jackson from '@lib/jackson'; import type { IAdminController } from '@boxyhq/saml-jackson'; +import { parsePaginateApiParams } from '@lib/utils'; const handler = async (req: NextApiRequest, res: NextApiResponse) => { const { method } = req; @@ -23,10 +24,8 @@ const handler = async (req: NextApiRequest, res: NextApiResponse) => { // Get SAML Traces const handleGET = async (req: NextApiRequest, res: NextApiResponse, adminController: IAdminController) => { - const { offset, limit, pageToken } = req.query as { offset: string; limit: string; pageToken?: string }; - - const pageOffset = parseInt(offset); - const pageLimit = parseInt(limit); + const params = req.query; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(params); const tracesPaginated = await adminController.getAllSSOTraces(pageOffset, pageLimit, pageToken); diff --git a/pages/api/v1/dsync/events/index.ts b/pages/api/v1/dsync/events/index.ts index 907aa5c9a..67cdd3039 100644 --- a/pages/api/v1/dsync/events/index.ts +++ b/pages/api/v1/dsync/events/index.ts @@ -1,5 +1,6 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -28,14 +29,13 @@ const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { tenant: string; product: string; directoryId: string; - offset: string; - limit: string; - pageToken: string; }; let tenant = searchParams.tenant || ''; let product = searchParams.product || ''; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + // If tenant and product are not provided, retrieve the from directory if ((!tenant || !product) && searchParams.directoryId) { const { data: directory } = await directorySyncController.directories.get(searchParams.directoryId); @@ -49,9 +49,9 @@ const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { } const events = await directorySyncController.webhookLogs.setTenantAndProduct(tenant, product).getAll({ - pageOffset: parseInt(searchParams.offset || '0'), - pageLimit: parseInt(searchParams.limit || '15'), - pageToken: searchParams.pageToken || undefined, + pageOffset, + pageLimit, + pageToken, directoryId: searchParams.directoryId, }); diff --git a/pages/api/v1/dsync/groups/index.ts b/pages/api/v1/dsync/groups/index.ts index f8a6bfba2..8ecbe0629 100644 --- a/pages/api/v1/dsync/groups/index.ts +++ b/pages/api/v1/dsync/groups/index.ts @@ -1,5 +1,6 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { const { method } = req; @@ -21,14 +22,13 @@ const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { tenant: string; product: string; directoryId: string; - offset: string; - limit: string; - pageToken: string; }; let tenant = searchParams.tenant || ''; let product = searchParams.product || ''; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + // If tenant and product are not provided, retrieve the from directory if ((!tenant || !product) && searchParams.directoryId) { const { data: directory } = await directorySyncController.directories.get(searchParams.directoryId); @@ -42,9 +42,9 @@ const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { } const { data, error } = await directorySyncController.groups.setTenantAndProduct(tenant, product).getAll({ - pageOffset: parseInt(searchParams.offset || '0'), - pageLimit: parseInt(searchParams.limit || '15'), - pageToken: searchParams.pageToken || undefined, + pageOffset, + pageLimit, + pageToken, directoryId: searchParams.directoryId, }); diff --git a/pages/api/v1/dsync/product.ts b/pages/api/v1/dsync/product.ts index 721b5fd20..e051fe74e 100644 --- a/pages/api/v1/dsync/product.ts +++ b/pages/api/v1/dsync/product.ts @@ -1,4 +1,5 @@ import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; import { NextApiRequest, NextApiResponse } from 'next'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -23,21 +24,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { directorySyncController } = await jackson(); - const { product, pageOffset, pageLimit, pageToken } = req.query as { + const { product } = req.query as { product: string; - pageOffset: string; - pageLimit: string; - pageToken?: string; }; if (!product) { throw new Error('Please provide a product'); } + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + const connections = await directorySyncController.directories.filterBy({ product, - pageOffset: parseInt(pageOffset), - pageLimit: parseInt(pageLimit), + pageOffset, + pageLimit, pageToken, }); diff --git a/pages/api/v1/dsync/setuplinks/product.ts b/pages/api/v1/dsync/setuplinks/product.ts index 4fad33dd2..e4fdabc9e 100644 --- a/pages/api/v1/dsync/setuplinks/product.ts +++ b/pages/api/v1/dsync/setuplinks/product.ts @@ -1,6 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import type { SetupLinkService } from '@boxyhq/saml-jackson'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; const service: SetupLinkService = 'dsync'; @@ -27,22 +28,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { setupLinkController } = await jackson(); - const { product, pageOffset, pageLimit, pageToken } = req.query as { + const { product } = req.query as { product: string; - pageOffset: string; - pageLimit: string; - pageToken?: string; }; if (!product) { throw new Error('Please provide a product'); } + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + const setupLinks = await setupLinkController.filterBy({ product, service, - pageOffset: parseInt(pageOffset), - pageLimit: parseInt(pageLimit), + pageOffset, + pageLimit, pageToken, }); diff --git a/pages/api/v1/dsync/users/index.ts b/pages/api/v1/dsync/users/index.ts index 9be613232..0d7a5729d 100644 --- a/pages/api/v1/dsync/users/index.ts +++ b/pages/api/v1/dsync/users/index.ts @@ -1,5 +1,6 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { const { method } = req; @@ -21,14 +22,13 @@ const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { tenant: string; product: string; directoryId: string; - offset: string; - limit: string; - pageToken: string; }; let tenant = searchParams.tenant || ''; let product = searchParams.product || ''; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + // If tenant and product are not provided, retrieve the from directory if ((!tenant || !product) && searchParams.directoryId) { const { data: directory } = await directorySyncController.directories.get(searchParams.directoryId); @@ -42,9 +42,9 @@ const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { } const { data, error } = await directorySyncController.users.setTenantAndProduct(tenant, product).getAll({ - pageOffset: parseInt(searchParams.offset || '0'), - pageLimit: parseInt(searchParams.limit || '15'), - pageToken: searchParams.pageToken || undefined, + pageOffset, + pageLimit, + pageToken, directoryId: searchParams.directoryId, }); diff --git a/pages/api/v1/sso-traces/product.ts b/pages/api/v1/sso-traces/product.ts index 1a9b3595d..6e8295a69 100644 --- a/pages/api/v1/sso-traces/product.ts +++ b/pages/api/v1/sso-traces/product.ts @@ -1,4 +1,5 @@ import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; import { NextApiRequest, NextApiResponse } from 'next'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -17,23 +18,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) } } -// Get the saml traces filtered by the product +// Get the sso traces filtered by the product const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { adminController } = await jackson(); - const { product, pageOffset, pageLimit, pageToken } = req.query as { + const { product } = req.query as { product: string; - pageOffset: string; - pageLimit: string; - pageToken?: string; }; - const traces = await adminController.getTracesByProduct( - product, - parseInt(pageOffset), - parseInt(pageLimit), - pageToken - ); + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + + const traces = await adminController.getTracesByProduct(product, pageOffset, pageLimit, pageToken); res.json(traces); }; diff --git a/pages/api/v1/sso/product.ts b/pages/api/v1/sso/product.ts index 0cfb124a7..47239cdab 100644 --- a/pages/api/v1/sso/product.ts +++ b/pages/api/v1/sso/product.ts @@ -1,4 +1,5 @@ import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; import { NextApiRequest, NextApiResponse } from 'next'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -23,17 +24,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { connectionAPIController } = await jackson(); - const { product, pageOffset, pageLimit, pageToken } = req.query as { + const { product } = req.query as { product: string; - pageOffset: string; - pageLimit: string; - pageToken?: string; }; + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + const connections = await connectionAPIController.getConnectionsByProduct({ product, - pageOffset: parseInt(pageOffset), - pageLimit: parseInt(pageLimit), + pageOffset, + pageLimit, pageToken, }); diff --git a/pages/api/v1/sso/setuplinks/product.ts b/pages/api/v1/sso/setuplinks/product.ts index 3111d5742..bfe36a6cf 100644 --- a/pages/api/v1/sso/setuplinks/product.ts +++ b/pages/api/v1/sso/setuplinks/product.ts @@ -1,6 +1,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import type { SetupLinkService } from '@boxyhq/saml-jackson'; import jackson from '@lib/jackson'; +import { parsePaginateApiParams } from '@lib/utils'; const service: SetupLinkService = 'sso'; @@ -27,22 +28,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) const handleGET = async (req: NextApiRequest, res: NextApiResponse) => { const { setupLinkController } = await jackson(); - const { product, pageOffset, pageLimit, pageToken } = req.query as { + const { product } = req.query as { product: string; - pageOffset: string; - pageLimit: string; - pageToken?: string; }; if (!product) { throw new Error('Please provide a product'); } + const { pageOffset, pageLimit, pageToken } = parsePaginateApiParams(req.query); + const setupLinks = await setupLinkController.filterBy({ product, service, - pageOffset: parseInt(pageOffset), - pageLimit: parseInt(pageLimit), + pageOffset, + pageLimit, pageToken, }); diff --git a/swagger/swagger.json b/swagger/swagger.json index e0a96377b..ac2d0905f 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -2,7 +2,7 @@ "openapi": "3.0.3", "info": { "title": "Enterprise SSO & Directory Sync", - "version": "1.19.2", + "version": "1.20.5", "description": "This is the API documentation for SAML Jackson service.", "termsOfService": "https://boxyhq.com/terms.html", "contact": { @@ -278,6 +278,15 @@ "parameters": [ { "$ref": "#/parameters/productParamGet" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" } ], "operationId": "get-connections-by-product", @@ -286,7 +295,7 @@ ], "responses": { "200": { - "$ref": "#/responses/200Get" + "$ref": "#/responses/200GetByProduct" }, "400": { "$ref": "#/responses/400Get" @@ -644,6 +653,15 @@ "parameters": [ { "$ref": "#/parameters/productParamGet" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" } ], "operationId": "get-sso-setup-link-by-product", @@ -669,6 +687,15 @@ "parameters": [ { "$ref": "#/parameters/productParamGet" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" } ], "operationId": "get-dsync-setup-link-by-product", @@ -701,7 +728,7 @@ } ], "tags": [ - "SAML Traces" + "SSO Traces" ], "produces": [ "application/json" @@ -722,10 +749,19 @@ "parameters": [ { "$ref": "#/parameters/product" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" } ], "tags": [ - "SAML Traces" + "SSO Traces" ], "produces": [ "application/json" @@ -733,10 +769,23 @@ "responses": { "200": { "description": "Success", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/SSOTrace" + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/SSOTrace" + } + }, + "pageToken": { + "type": "string", + "description": "token for pagination" + } + } + } } } } @@ -985,6 +1034,15 @@ "parameters": [ { "$ref": "#/parameters/product" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" } ], "tags": [ @@ -996,10 +1054,23 @@ "responses": { "200": { "description": "Success", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Directory" + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/Directory" + } + }, + "pageToken": { + "type": "string", + "description": "token for pagination" + } + } + } } } } @@ -1052,6 +1123,15 @@ }, { "$ref": "#/parameters/directoryId" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" } ], "tags": [ @@ -1063,10 +1143,23 @@ "responses": { "200": { "description": "Success", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Group" + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/Group" + } + }, + "pageToken": { + "type": "string", + "description": "token for pagination" + } + } + } } } } @@ -1119,6 +1212,15 @@ }, { "$ref": "#/parameters/directoryId" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" } ], "tags": [ @@ -1130,10 +1232,72 @@ "responses": { "200": { "description": "Success", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/User" + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/User" + } + }, + "pageToken": { + "type": "string", + "description": "token for pagination" + } + } + } + } + } + } + } + } + }, + "/api/v1/dsync/events": { + "get": { + "summary": "Get event logs for a directory", + "parameters": [ + { + "$ref": "#/parameters/directoryId" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" + } + ], + "tags": [ + "Directory Sync" + ], + "produces": [ + "application/json" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/Event" + } + }, + "pageToken": { + "type": "string", + "description": "token for pagination" + } + } + } } } } @@ -1435,6 +1599,15 @@ "in": "query", "required": true, "type": "string" + }, + { + "$ref": "#/parameters/pageOffset" + }, + { + "$ref": "#/parameters/pageLimit" + }, + { + "$ref": "#/parameters/pageToken" } ], "tags": [ @@ -1446,10 +1619,23 @@ "responses": { "200": { "description": "Success", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/SAMLFederationApp" + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/SAMLFederationApp" + } + }, + "pageToken": { + "type": "string", + "description": "token for pagination" + } + } + } } } } @@ -1742,6 +1928,53 @@ } } }, + "Event": { + "type": "object", + "example": { + "id": "id1", + "webhook_endpoint": "https://example.com/webhook", + "created_at": "2024-03-05T17:06:26.074Z", + "status_code": 200, + "delivered": true, + "payload": { + "directory_id": "58b5cd9dfaa39d47eb8f5f88631f9a629a232016", + "event": "user.created", + "tenant": "boxyhq", + "product": "jackson", + "data": { + "id": "038e767b-9bc6-4dbd-975e-fbc38a8e7d82", + "first_name": "Deepak", + "last_name": "Prabhakara", + "email": "deepak@boxyhq.com", + "active": true, + "raw": { + "schemas": [ + "urn:ietf:params:scim:schemas:core:2.0:User" + ], + "userName": "deepak@boxyhq.com", + "name": { + "givenName": "Deepak", + "familyName": "Prabhakara" + }, + "emails": [ + { + "primary": true, + "value": "deepak@boxyhq.com", + "type": "work" + } + ], + "title": "CEO", + "displayName": "Deepak Prabhakara", + "locale": "en-US", + "externalId": "00u1ldzzogFkXFmvT5d7", + "groups": [], + "active": true, + "id": "038e767b-9bc6-4dbd-975e-fbc38a8e7d82" + } + } + } + } + }, "SAMLFederationApp": { "type": "object", "properties": { @@ -1799,6 +2032,28 @@ }, "401Get": { "description": "Unauthorized" + }, + "200GetByProduct": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/Connection" + } + }, + "pageToken": { + "type": "string", + "description": "token for pagination" + } + } + } + } + } } }, "parameters": { @@ -2138,6 +2393,27 @@ "in": "query", "required": false, "type": "string" + }, + "pageOffset": { + "name": "pageOffset", + "description": "Starting point from which the set of records are retrieved", + "in": "query", + "required": false, + "type": "string" + }, + "pageLimit": { + "name": "pageLimit", + "description": "Number of records to be fetched for the page", + "in": "query", + "required": false, + "type": "string" + }, + "pageToken": { + "name": "pageToken", + "description": "Token used for DynamoDB pagination", + "in": "query", + "required": false, + "type": "string" } }, "components": {}, diff --git a/types.ts b/types.ts deleted file mode 100644 index 169db490c..000000000 --- a/types.ts +++ /dev/null @@ -1,8 +0,0 @@ -export type ApiSuccess = { data: T; pageToken?: string }; - -export interface ApiError extends Error { - info?: string; - status: number; -} - -export type ApiResponse = ApiSuccess | { error: ApiError }; diff --git a/types/base.ts b/types/base.ts index 67a0d175a..64bb7891c 100644 --- a/types/base.ts +++ b/types/base.ts @@ -1,10 +1,10 @@ -export type ApiError = { - code?: string; - message: string; - values: { [key: string]: string }; -}; +export type ApiSuccess = { data: T; pageToken?: string }; -export type ApiResponse = { - data: T | null; - error: ApiError | null; -}; +export interface ApiError extends Error { + info?: string; + status: number; +} + +export type ApiResponse = ApiSuccess | { error: ApiError }; + +export type PaginateApiParams = { pageOffset: number; pageLimit: number } & { pageToken?: string };