Add sort order option to `getAll` function (#1565)

* add sort order

* add tests

* wip

* Fix the sort

* Cleanup

* Tweak

* Update tests

* Fix the test

* test db

* Run test

* Add precision 6

* Run test

* Test planetscale

* Test migration

* Remove test migration

* Update migration

* npm install

* modifiedAt column type change

* Remove test migration

* add migrations

* Fix

* Rename migration

* Fix the migration file name

* Fix conflict

* Update package-lock

* Fix Planetscale migration

---------

Co-authored-by: Deepak Prabhakara <deepak@boxyhq.com>

npm/migration/mariadb/1695120700240-md_sortorder.ts

@@ -0,0 +1,16 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MdSortorder1695120700240 implements MigrationInterface {
+    name = 'MdSortorder1695120700240'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`createdAt\` \`createdAt\` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)`);
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`modifiedAt\` \`modifiedAt\` timestamp(6) NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`modifiedAt\` \`modifiedAt\` timestamp(0) NULL`);
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`createdAt\` \`createdAt\` timestamp(0) NOT NULL DEFAULT CURRENT_TIMESTAMP()`);
+    }
+
+}

npm/migration/mysql/1695120580071-ms_sortorder.ts

@@ -0,0 +1,16 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MsSortorder1695120580071 implements MigrationInterface {
+    name = 'MsSortorder1695120580071'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`createdAt\` \`createdAt\` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)`);
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`modifiedAt\` \`modifiedAt\` timestamp(6) NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`modifiedAt\` \`modifiedAt\` timestamp(0) NULL`);
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`createdAt\` \`createdAt\` timestamp(0) NOT NULL DEFAULT CURRENT_TIMESTAMP`);
+    }
+
+}

npm/migration/planetscale/1695120599689-ms_sortorder.ts

@@ -0,0 +1,16 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MsSortorder1695120599689 implements MigrationInterface {
+    name = 'MsSortorder1695120599689'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`createdAt\` \`createdAt\` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)`);
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`modifiedAt\` \`modifiedAt\` timestamp(6) NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`modifiedAt\` \`modifiedAt\` timestamp(0) NULL`);
+        await queryRunner.query(`ALTER TABLE \`jackson_store\` CHANGE \`createdAt\` \`createdAt\` timestamp(0) NOT NULL DEFAULT CURRENT_TIMESTAMP`);
+    }
+
+}

npm/src/db/db.ts

@@ -5,6 +5,7 @@ import {
   EncryptionKey,
   Index,
   Records,
+  SortOrder,
   Storable,
 } from '../typings';
 import * as encrypter from './encrypter';
@@ -61,9 +62,10 @@ class DB implements DatabaseDriver {
     namespace: string,
     pageOffset?: number,
     pageLimit?: number,
-    pageToken?: string
+    pageToken?: string,
+    sortOrder?: SortOrder
   ): Promise<Records> {
-    const res = await this.db.getAll(namespace, pageOffset, pageLimit, pageToken);
+    const res = await this.db.getAll(namespace, pageOffset, pageLimit, pageToken, sortOrder);
     const encryptionKey = this.encryptionKey;
     return {
       data: res.data.map((r) => {

npm/src/db/mem.ts

@@ -1,6 +1,6 @@
 // This is an in-memory implementation to be used with testing and prototyping only
 
-import { DatabaseDriver, DatabaseOption, Index, Encrypted, Records } from '../typings';
+import { DatabaseDriver, DatabaseOption, Index, Encrypted, Records, SortOrder } from '../typings';
 import * as dbutils from './utils';
 
 class Mem implements DatabaseDriver {
@@ -52,7 +52,13 @@ class Mem implements DatabaseDriver {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async getAll(namespace: string, pageOffset?: number, pageLimit?: number, _?: string): Promise<Records> {
+  async getAll(
+    namespace: string,
+    pageOffset?: number,
+    pageLimit?: number,
+    _?: string,
+    sortOrder?: SortOrder
+  ): Promise<Records> {
     const offsetAndLimitValueCheck = !dbutils.isNumeric(pageOffset) && !dbutils.isNumeric(pageLimit);
     const returnValue: string[] = [];
     const skip = Number(offsetAndLimitValueCheck ? 0 : pageOffset);
@@ -70,7 +76,7 @@ class Mem implements DatabaseDriver {
       }
 
       const val: string[] = Array.from(this.indexes[index]);
-      const iterator: IterableIterator<string> = val.reverse().values();
+      const iterator: IterableIterator<string> = sortOrder === 'ASC' ? val.values() : val.reverse().values();
 
       for (const value of iterator) {
         if (count >= take) {

npm/src/db/mongo.ts

@@ -1,5 +1,5 @@
 import { Collection, Db, MongoClient, UpdateOptions } from 'mongodb';
-import { DatabaseDriver, DatabaseOption, Encrypted, Index, Records } from '../typings';
+import { DatabaseDriver, DatabaseOption, Encrypted, Index, Records, SortOrder } from '../typings';
 import * as dbutils from './utils';
 
 type _Document = {
@@ -79,14 +79,24 @@ class Mongo implements DatabaseDriver {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async getAll(namespace: string, pageOffset?: number, pageLimit?: number, _?: string): Promise<Records> {
+  async getAll(
+    namespace: string,
+    pageOffset?: number,
+    pageLimit?: number,
+    _?: string,
+    sortOrder?: SortOrder
+  ): Promise<Records> {
     const docs = await this.collection
-      .find({ namespace: namespace }, { sort: { createdAt: -1 }, skip: pageOffset, limit: pageLimit })
+      .find(
+        { namespace: namespace },
+        { sort: { createdAt: sortOrder === 'ASC' ? 1 : -1 }, skip: pageOffset, limit: pageLimit }
+      )
       .toArray();
 
     if (docs) {
       return { data: docs.map(({ value }) => value) };
     }
+
     return { data: [] };
   }
 

npm/src/db/planetscale/entity/JacksonStore.ts

@@ -30,13 +30,15 @@ export class JacksonStore {
 
   @Column({
     type: 'timestamp',
-    default: () => 'CURRENT_TIMESTAMP',
+    precision: 6,
+    default: () => 'CURRENT_TIMESTAMP(6)',
     nullable: false,
   })
   createdAt?: Date;
 
   @Column({
     type: 'timestamp',
+    precision: 6,
     nullable: true,
   })
   modifiedAt?: string;

npm/src/db/redis.ts

@@ -1,5 +1,5 @@
 import * as redis from 'redis';
-import { DatabaseDriver, DatabaseOption, Encrypted, Index, Records } from '../typings';
+import { DatabaseDriver, DatabaseOption, Encrypted, Index, Records, SortOrder } from '../typings';
 import * as dbutils from './utils';
 
 class Redis implements DatabaseDriver {
@@ -37,37 +37,51 @@ class Redis implements DatabaseDriver {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async getAll(namespace: string, pageOffset?: number, pageLimit?: number, _?: string): Promise<Records> {
-    const offsetAndLimitValueCheck = !dbutils.isNumeric(pageOffset) && !dbutils.isNumeric(pageLimit);
-    let take = Number(offsetAndLimitValueCheck ? this.options.pageLimit : pageLimit);
-    const skip = Number(offsetAndLimitValueCheck ? 0 : pageOffset);
-    const returnValue: string[] = [];
-    const keyArray: string[] = [];
-    let count = 0;
-    take += skip;
-    for await (const { value } of this.client.zScanIterator(
-      dbutils.keyFromParts(dbutils.createdAtPrefix, namespace),
-      Math.min(take, 1000)
-    )) {
-      if (count >= take) {
-        break;
-      }
-      if (count >= skip) {
-        keyArray.push(dbutils.keyFromParts(namespace, value));
-      }
-      count++;
+  async getAll(
+    namespace: string,
+    pageOffset?: number,
+    pageLimit?: number,
+    _?: string,
+    sortOrder?: SortOrder
+  ): Promise<Records> {
+    const offset = pageOffset ? Number(pageOffset) : 0;
+    const count = pageLimit ? Number(pageLimit) : this.options.pageLimit;
+    const sortedSetKey = dbutils.keyFromParts(dbutils.createdAtPrefix, namespace);
+
+    const zRangeOptions = {
+      BY: 'SCORE',
+      REV: sortOrder === 'ASC',
+      LIMIT: {
+        offset,
+        count,
+      },
+    };
+
+    const elements =
+      sortOrder === 'ASC'
+        ? await this.client.zRange(sortedSetKey, +Infinity, -Infinity, zRangeOptions)
+        : await this.client.zRange(sortedSetKey, -Infinity, +Infinity, zRangeOptions);
+
+    if (elements.length === 0) {
+      return { data: [] };
     }
 
-    if (keyArray.length > 0) {
-      const value = await this.client.MGET(keyArray);
-      for (let i = 0; i < value.length; i++) {
-        const valueObject = JSON.parse(value[i].toString());
-        if (valueObject !== null && valueObject !== '') {
-          returnValue.push(valueObject);
-        }
+    const keyArray = elements.map((element) => {
+      return dbutils.keyFromParts(namespace, element);
+    });
+
+    const records: string[] = [];
+    const values = await this.client.MGET(keyArray);
+
+    for (let i = 0; i < values.length; i++) {
+      const valueObject = JSON.parse(values[i].toString());
+
+      if (valueObject !== null && valueObject !== '') {
+        records.push(valueObject);
       }
     }
-    return { data: returnValue || [] };
+
+    return { data: records };
   }
 
   async getByIndex(

npm/src/db/sql/mariadb/entity/JacksonStore.ts

@@ -30,13 +30,15 @@ export class JacksonStore {
 
   @Column({
     type: 'timestamp',
-    default: () => 'CURRENT_TIMESTAMP',
+    precision: 6,
+    default: () => 'CURRENT_TIMESTAMP(6)',
     nullable: false,
   })
   createdAt?: Date;
 
   @Column({
     type: 'timestamp',
+    precision: 6,
     nullable: true,
   })
   modifiedAt?: string;

npm/src/db/sql/sql.ts

@@ -2,7 +2,7 @@
 
 require('reflect-metadata');
 
-import { DatabaseDriver, DatabaseOption, Index, Encrypted, Records } from '../../typings';
+import { DatabaseDriver, DatabaseOption, Index, Encrypted, Records, SortOrder } from '../../typings';
 import { DataSource, DataSourceOptions, In, IsNull } from 'typeorm';
 import * as dbutils from '../utils';
 import * as mssql from './mssql';
@@ -164,17 +164,25 @@ class Sql implements DatabaseDriver {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  async getAll(namespace: string, pageOffset?: number, pageLimit?: number, _?: string): Promise<Records> {
+  async getAll(
+    namespace: string,
+    pageOffset?: number,
+    pageLimit?: number,
+    _?: string,
+    sortOrder?: SortOrder
+  ): Promise<Records> {
     const skipOffsetAndLimitValue = !dbutils.isNumeric(pageOffset) && !dbutils.isNumeric(pageLimit);
+
     const res = await this.storeRepository.find({
       where: { namespace: namespace },
       select: ['value', 'iv', 'tag'],
       order: {
-        ['createdAt']: 'DESC',
+        ['createdAt']: sortOrder || 'DESC',
       },
       take: skipOffsetAndLimitValue ? this.options.pageLimit : pageLimit,
       skip: skipOffsetAndLimitValue ? 0 : pageOffset,
     });
+
     return { data: res || [] };
   }
 

npm/src/db/store.ts

@@ -1,4 +1,4 @@
-import { Index, Records, Storable } from '../typings';
+import { Index, Records, SortOrder, Storable } from '../typings';
 import * as dbutils from './utils';
 
 class Store implements Storable {
@@ -16,8 +16,13 @@ class Store implements Storable {
     return await this.db.get(this.namespace, dbutils.keyDigest(key));
   }
 
-  async getAll(pageOffset?: number, pageLimit?: number, pageToken?: string): Promise<Records> {
-    return await this.db.getAll(this.namespace, pageOffset, pageLimit, pageToken);
+  async getAll(
+    pageOffset?: number,
+    pageLimit?: number,
+    pageToken?: string,
+    sortOrder?: SortOrder
+  ): Promise<Records> {
+    return await this.db.getAll(this.namespace, pageOffset, pageLimit, pageToken, sortOrder);
   }
 
   async getByIndex(

npm/src/typings.ts

@@ -325,7 +325,13 @@ export interface Records<T = any> {
 }
 
 export interface DatabaseDriver {
-  getAll(namespace: string, pageOffset?: number, pageLimit?: number, pageToken?: string): Promise<Records>;
+  getAll(
+    namespace: string,
+    pageOffset?: number,
+    pageLimit?: number,
+    pageToken?: string,
+    sortOrder?: SortOrder
+  ): Promise<Records>;
   get(namespace: string, key: string): Promise<any>;
   put(namespace: string, key: string, val: any, ttl: number, ...indexes: Index[]): Promise<any>;
   delete(namespace: string, key: string): Promise<any>;
@@ -340,7 +346,12 @@ export interface DatabaseDriver {
 }
 
 export interface Storable {
-  getAll(pageOffset?: number, pageLimit?: number, pageToken?: string): Promise<Records>;
+  getAll(
+    pageOffset?: number,
+    pageLimit?: number,
+    pageToken?: string,
+    sortOrder?: SortOrder
+  ): Promise<Records>;
   get(key: string): Promise<any>;
   put(key: string, val: any, ...indexes: Index[]): Promise<any>;
   delete(key: string): Promise<any>;
@@ -549,3 +560,5 @@ export type GetByProductParams = {
   pageLimit?: number;
   pageToken?: string;
 };
+
+export type SortOrder = 'ASC' | 'DESC';

npm/test/db/db.test.ts

@@ -30,6 +30,7 @@ const memDbConfig = <DatabaseOption>{
 const redisDbConfig = <DatabaseOption>{
   engine: 'redis',
   url: 'redis://localhost:6379',
+  pageLimit: 50,
 };
 
 const postgresDbConfig = <DatabaseOption>{
@@ -166,6 +167,7 @@ if (process.env.DYNAMODB_URL) {
     }
   );
 }
+
 tap.before(async () => {
   for (const idx in dbs) {
     const opts = dbs[idx];
@@ -276,6 +278,14 @@ tap.test('dbs', async () => {
         1,
         "getAll pagination should get only 1 record, order doesn't matter"
       );
+
+      if (!dbEngine.startsWith('dynamodb')) {
+        const { data: sortedRecordsAsc } = await connectionStore.getAll(0, 2, undefined, 'ASC');
+        t.match(sortedRecordsAsc, [record1, record2], 'records are sorted in ASC order');
+
+        const { data: sortedRecordsDesc } = await connectionStore.getAll(0, 2, undefined, 'DESC');
+        t.match(sortedRecordsDesc, [record2, record1], 'records are sorted in DESC order');
+      }
     });
 
     tap.test('getByIndex(): ' + dbEngine, async (t) => {