feat: User pagination using offsets (#1062)

Offset pagination and cursor pagination supported
This commit is contained in:
Steven Masley 2022-04-22 15:27:55 -05:00 committed by GitHub
parent 2a95917557
commit 548de7d6f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 446 additions and 71 deletions

View File

@ -15,7 +15,7 @@ coderd/database/dump.sql: $(wildcard coderd/database/migrations/*.sql)
.PHONY: coderd/database/dump.sql
# Generates Go code for querying the database.
coderd/database/generate: fmt/sql coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql)
coderd/database/generate: coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql)
coderd/database/generate.sh
.PHONY: coderd/database/generate
@ -34,22 +34,10 @@ else
endif
.PHONY: fmt/prettier
fmt/sql: $(wildcard coderd/database/queries/*.sql)
for fi in coderd/database/queries/*.sql; do \
npx sql-formatter \
--language postgresql \
--lines-between-queries 2 \
--tab-indent \
$$fi \
--output $$fi; \
done
sed -i 's/@ /@/g' ./coderd/database/queries/*.sql
fmt/terraform: $(wildcard *.tf)
terraform fmt -recursive
fmt: fmt/prettier fmt/sql fmt/terraform
fmt: fmt/prettier fmt/terraform
.PHONY: fmt
gen: coderd/database/generate peerbroker/proto provisionersdk/proto provisionerd/proto apitypings/generate

View File

@ -35,13 +35,17 @@ type Options struct {
Pubsub database.Pubsub
AgentConnectionUpdateFrequency time.Duration
AWSCertificates awsidentity.Certificates
AzureCertificates x509.VerifyOptions
GoogleTokenValidator *idtoken.Validator
ICEServers []webrtc.ICEServer
SecureAuthCookie bool
SSHKeygenAlgorithm gitsshkey.Algorithm
TURNServer *turnconn.Server
// APIRateLimit is the minutely throughput rate limit per user or ip.
// Setting a rate limit <0 will disable the rate limiter across the entire
// app. Specific routes may have their own limiters.
APIRateLimit int
AWSCertificates awsidentity.Certificates
AzureCertificates x509.VerifyOptions
GoogleTokenValidator *idtoken.Validator
ICEServers []webrtc.ICEServer
SecureAuthCookie bool
SSHKeygenAlgorithm gitsshkey.Algorithm
TURNServer *turnconn.Server
}
// New constructs the Coder API into an HTTP handler.
@ -52,6 +56,9 @@ func New(options *Options) (http.Handler, func()) {
if options.AgentConnectionUpdateFrequency == 0 {
options.AgentConnectionUpdateFrequency = 3 * time.Second
}
if options.APIRateLimit == 0 {
options.APIRateLimit = 512
}
api := &api{
Options: options,
}
@ -61,7 +68,7 @@ func New(options *Options) (http.Handler, func()) {
r.Use(
chitrace.Middleware(),
// Specific routes can specify smaller limits.
httpmw.RateLimitPerMinute(512),
httpmw.RateLimitPerMinute(options.APIRateLimit),
debugLogRequest(api.Logger),
)
r.Get("/", func(w http.ResponseWriter, r *http.Request) {

View File

@ -55,6 +55,7 @@ type Options struct {
AzureCertificates x509.VerifyOptions
GoogleTokenValidator *idtoken.Validator
SSHKeygenAlgorithm gitsshkey.Algorithm
APIRateLimit int
}
// New constructs an in-memory coderd instance and returns
@ -125,6 +126,7 @@ func New(t *testing.T, options *Options) *codersdk.Client {
GoogleTokenValidator: options.GoogleTokenValidator,
SSHKeygenAlgorithm: options.SSHKeygenAlgorithm,
TURNServer: turnServer,
APIRateLimit: options.APIRateLimit,
})
t.Cleanup(func() {
cancelFunc()

View File

@ -3,6 +3,7 @@ package databasefake
import (
"context"
"database/sql"
"sort"
"strings"
"sync"
@ -164,11 +165,72 @@ func (q *fakeQuerier) GetUserCount(_ context.Context) (int64, error) {
return int64(len(q.users)), nil
}
func (q *fakeQuerier) GetUsers(_ context.Context) ([]database.User, error) {
func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams) ([]database.User, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()
return q.users, nil
users := q.users
// Database orders by created_at
sort.Slice(users, func(i, j int) bool {
if users[i].CreatedAt.Equal(users[j].CreatedAt) {
// Technically the postgres database also orders by uuid. So match
// that behavior
return users[i].ID.String() < users[j].ID.String()
}
return users[i].CreatedAt.Before(users[j].CreatedAt)
})
if params.AfterUser != uuid.Nil {
found := false
for i := range users {
if users[i].ID == params.AfterUser {
// We want to return all users after index i.
if i+1 >= len(users) {
return []database.User{}, nil
}
users = users[i+1:]
found = true
break
}
}
// If no users after the time, then we return an empty list.
if !found {
return []database.User{}, nil
}
}
if params.Search != "" {
tmp := make([]database.User, 0, len(users))
for i, user := range users {
if strings.Contains(user.Email, params.Search) {
tmp = append(tmp, users[i])
} else if strings.Contains(user.Username, params.Search) {
tmp = append(tmp, users[i])
} else if strings.Contains(user.Name, params.Search) {
tmp = append(tmp, users[i])
}
}
users = tmp
}
if params.OffsetOpt > 0 {
if int(params.OffsetOpt) > len(users)-1 {
return []database.User{}, nil
}
users = users[params.OffsetOpt:]
}
if params.LimitOpt > 0 {
if int(params.LimitOpt) > len(users) {
params.LimitOpt = int32(len(users))
}
users = users[:params.LimitOpt]
}
tmp := make([]database.User, len(users))
copy(tmp, users)
return tmp, nil
}
func (q *fakeQuerier) GetWorkspacesByTemplateID(_ context.Context, arg database.GetWorkspacesByTemplateIDParams) ([]database.Workspace, error) {

View File

@ -38,7 +38,7 @@ type querier interface {
GetUserByEmailOrUsername(ctx context.Context, arg GetUserByEmailOrUsernameParams) (User, error)
GetUserByID(ctx context.Context, id uuid.UUID) (User, error)
GetUserCount(ctx context.Context) (int64, error)
GetUsers(ctx context.Context) ([]User, error)
GetUsers(ctx context.Context, arg GetUsersParams) ([]User, error)
GetWorkspaceAgentByAuthToken(ctx context.Context, authToken uuid.UUID) (WorkspaceAgent, error)
GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (WorkspaceAgent, error)
GetWorkspaceAgentByInstanceID(ctx context.Context, authInstanceID string) (WorkspaceAgent, error)

View File

@ -1856,10 +1856,60 @@ SELECT
id, email, name, revoked, login_type, hashed_password, created_at, updated_at, username
FROM
users
WHERE
CASE
-- This allows using the last element on a page as effectively a cursor.
-- This is an important option for scripts that need to paginate without
-- duplicating or missing data.
WHEN $1 :: uuid != '00000000-00000000-00000000-00000000' THEN (
-- The pagination cursor is the last user of the previous page.
-- The query is ordered by the created_at field, so select all
-- users after the cursor. We also want to include any users
-- that share the created_at (super rare).
created_at >= (
SELECT
created_at
FROM
users
WHERE
id = $1
)
-- Omit the cursor from the final.
AND id != $1
)
ELSE true
END
AND CASE
WHEN $2 :: text != '' THEN (
email LIKE concat('%', $2, '%')
OR username LIKE concat('%', $2, '%')
OR 'name' LIKE concat('%', $2, '%')
)
ELSE true
END
ORDER BY
-- Deterministic and consistent ordering of all users, even if they share
-- a timestamp. This is to ensure consistent pagination.
(created_at, id) ASC OFFSET $3
LIMIT
-- A null limit means "no limit", so -1 means return all
NULLIF($4 :: int, -1)
`
func (q *sqlQuerier) GetUsers(ctx context.Context) ([]User, error) {
rows, err := q.db.QueryContext(ctx, getUsers)
type GetUsersParams struct {
AfterUser uuid.UUID `db:"after_user" json:"after_user"`
Search string `db:"search" json:"search"`
OffsetOpt int32 `db:"offset_opt" json:"offset_opt"`
LimitOpt int32 `db:"limit_opt" json:"limit_opt"`
}
func (q *sqlQuerier) GetUsers(ctx context.Context, arg GetUsersParams) ([]User, error) {
rows, err := q.db.QueryContext(ctx, getUsers,
arg.AfterUser,
arg.Search,
arg.OffsetOpt,
arg.LimitOpt,
)
if err != nil {
return nil, err
}

View File

@ -56,4 +56,42 @@ WHERE
SELECT
*
FROM
users;
users
WHERE
CASE
-- This allows using the last element on a page as effectively a cursor.
-- This is an important option for scripts that need to paginate without
-- duplicating or missing data.
WHEN @after_user :: uuid != '00000000-00000000-00000000-00000000' THEN (
-- The pagination cursor is the last user of the previous page.
-- The query is ordered by the created_at field, so select all
-- users after the cursor. We also want to include any users
-- that share the created_at (super rare).
created_at >= (
SELECT
created_at
FROM
users
WHERE
id = @after_user
)
-- Omit the cursor from the final.
AND id != @after_user
)
ELSE true
END
AND CASE
WHEN @search :: text != '' THEN (
email LIKE concat('%', @search, '%')
OR username LIKE concat('%', @search, '%')
OR 'name' LIKE concat('%', @search, '%')
)
ELSE true
END
ORDER BY
-- Deterministic and consistent ordering of all users, even if they share
-- a timestamp. This is to ensure consistent pagination.
(created_at, id) ASC OFFSET @offset_opt
LIMIT
-- A null limit means "no limit", so -1 means return all
NULLIF(@limit_opt :: int, -1);

View File

@ -13,6 +13,12 @@ import (
// RateLimitPerMinute returns a handler that limits requests per-minute based
// on IP, endpoint, and user ID (if available).
func RateLimitPerMinute(count int) func(http.Handler) http.Handler {
// -1 is no rate limit
if count <= 0 {
return func(handler http.Handler) http.Handler {
return handler
}
}
return httprate.Limit(
count,
1*time.Minute,

View File

@ -7,9 +7,11 @@ import (
"errors"
"fmt"
"net/http"
"strconv"
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/render"
"github.com/google/uuid"
"github.com/moby/moby/pkg/namesgenerator"
"golang.org/x/xerrors"
@ -23,25 +25,6 @@ import (
"github.com/coder/coder/cryptorand"
)
// Lists all the users
func (api *api) users(rw http.ResponseWriter, r *http.Request) {
users, err := api.Database.GetUsers(r.Context())
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("get users: %s", err.Error()),
})
return
}
var res []codersdk.User
for _, user := range users {
res = append(res, convertUser(user))
}
httpapi.Write(rw, http.StatusOK, res)
}
// Returns whether the initial user has been created or not.
func (api *api) firstUser(rw http.ResponseWriter, r *http.Request) {
userCount, err := api.Database.GetUserCount(r.Context())
@ -162,6 +145,67 @@ func (api *api) postFirstUser(rw http.ResponseWriter, r *http.Request) {
})
}
func (api *api) users(rw http.ResponseWriter, r *http.Request) {
var (
afterArg = r.URL.Query().Get("after_user")
limitArg = r.URL.Query().Get("limit")
offsetArg = r.URL.Query().Get("offset")
searchName = r.URL.Query().Get("search")
)
// createdAfter is a user uuid.
createdAfter := uuid.Nil
if afterArg != "" {
after, err := uuid.Parse(afterArg)
if err != nil {
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
Message: fmt.Sprintf("after_user must be a valid uuid: %s", err.Error()),
})
return
}
createdAfter = after
}
// Default to no limit and return all users.
pageLimit := -1
if limitArg != "" {
limit, err := strconv.Atoi(limitArg)
if err != nil {
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
Message: fmt.Sprintf("limit must be an integer: %s", err.Error()),
})
return
}
pageLimit = limit
}
// The default for empty string is 0.
offset, err := strconv.ParseInt(offsetArg, 10, 64)
if offsetArg != "" && err != nil {
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
Message: fmt.Sprintf("offset must be an integer: %s", err.Error()),
})
return
}
users, err := api.Database.GetUsers(r.Context(), database.GetUsersParams{
AfterUser: createdAfter,
OffsetOpt: int32(offset),
LimitOpt: int32(pageLimit),
Search: searchName,
})
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: err.Error(),
})
return
}
render.Status(r, http.StatusOK)
render.JSON(rw, r, convertUsers(users))
}
// Creates a new user.
func (api *api) postUsers(rw http.ResponseWriter, r *http.Request) {
apiKey := httpmw.APIKey(r)
@ -949,3 +993,11 @@ func convertUser(user database.User) codersdk.User {
Name: user.Name,
}
}
func convertUsers(users []database.User) []codersdk.User {
converted := make([]codersdk.User, 0, len(users))
for _, u := range users {
converted = append(converted, convertUser(u))
}
return converted
}

View File

@ -2,6 +2,7 @@ package coderd_test
import (
"context"
"fmt"
"net/http"
"testing"
@ -318,12 +319,13 @@ func TestGetUsers(t *testing.T) {
client := coderdtest.New(t, nil)
user := coderdtest.CreateFirstUser(t, client)
client.CreateUser(context.Background(), codersdk.CreateUserRequest{
Email: "bruno@coder.com",
Username: "bruno",
Email: "alice@email.com",
Username: "alice",
Password: "password",
OrganizationID: user.OrganizationID,
})
users, err := client.GetUsers(context.Background())
// No params is all users
users, err := client.Users(context.Background(), codersdk.UsersRequest{})
require.NoError(t, err)
require.Len(t, users, 2)
}
@ -546,3 +548,134 @@ func TestWorkspaceByUserAndName(t *testing.T) {
require.NoError(t, err)
})
}
// TestPaginatedUsers creates a list of users, then tries to paginate through
// them using different page sizes.
func TestPaginatedUsers(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := coderdtest.New(t, &coderdtest.Options{APIRateLimit: -1})
coderdtest.CreateFirstUser(t, client)
me, err := client.User(context.Background(), codersdk.Me)
require.NoError(t, err)
allUsers := make([]codersdk.User, 0)
allUsers = append(allUsers, me)
specialUsers := make([]codersdk.User, 0)
org, err := client.CreateOrganization(ctx, me.ID, codersdk.CreateOrganizationRequest{
Name: "default",
})
require.NoError(t, err)
// When 100 users exist
total := 100
// Create users
for i := 0; i < total; i++ {
email := fmt.Sprintf("%d@coder.com", i)
username := fmt.Sprintf("user%d", i)
if i%2 == 0 {
email = fmt.Sprintf("%d@gmail.com", i)
username = fmt.Sprintf("specialuser%d", i)
}
// One side effect of having to use the api vs the db calls directly, is you cannot
// mock time. Ideally I could pass in mocked times and space these users out.
//
// But this also serves as a good test. Postgres has microsecond precision on its timestamps.
// If 2 users share the same created_at, that could cause an issue if you are strictly paginating via
// timestamps. The pagination goes by timestamps and uuids.
newUser, err := client.CreateUser(context.Background(), codersdk.CreateUserRequest{
Email: email,
Username: username,
Password: "password",
OrganizationID: org.ID,
})
require.NoError(t, err)
allUsers = append(allUsers, newUser)
if i%2 == 0 {
specialUsers = append(specialUsers, newUser)
}
}
assertPagination(ctx, t, client, 10, allUsers, nil)
assertPagination(ctx, t, client, 5, allUsers, nil)
assertPagination(ctx, t, client, 3, allUsers, nil)
assertPagination(ctx, t, client, 1, allUsers, nil)
// Try a search
gmailSearch := func(request codersdk.UsersRequest) codersdk.UsersRequest {
request.Search = "gmail"
return request
}
assertPagination(ctx, t, client, 3, specialUsers, gmailSearch)
assertPagination(ctx, t, client, 7, specialUsers, gmailSearch)
usernameSearch := func(request codersdk.UsersRequest) codersdk.UsersRequest {
request.Search = "specialuser"
return request
}
assertPagination(ctx, t, client, 3, specialUsers, usernameSearch)
assertPagination(ctx, t, client, 1, specialUsers, usernameSearch)
}
// Assert pagination will page through the list of all users using the given
// limit for each page. The 'allUsers' is the expected full list to compare
// against.
func assertPagination(ctx context.Context, t *testing.T, client *codersdk.Client, limit int, allUsers []codersdk.User,
opt func(request codersdk.UsersRequest) codersdk.UsersRequest) {
var count int
if opt == nil {
opt = func(request codersdk.UsersRequest) codersdk.UsersRequest {
return request
}
}
// Check the first page
page, err := client.Users(ctx, opt(codersdk.UsersRequest{
Limit: limit,
}))
require.NoError(t, err, "first page")
require.Equalf(t, page, allUsers[:limit], "first page, limit=%d", limit)
count += len(page)
for {
if len(page) == 0 {
break
}
afterCursor := page[len(page)-1].ID
// Assert each page is the next expected page
// This is using a cursor, and only works if all users created_at
// is unique.
page, err = client.Users(ctx, opt(codersdk.UsersRequest{
Limit: limit,
AfterUser: afterCursor,
}))
require.NoError(t, err, "next cursor page")
// Also check page by offset
offsetPage, err := client.Users(ctx, opt(codersdk.UsersRequest{
Limit: limit,
Offset: count,
}))
require.NoError(t, err, "next offset page")
var expected []codersdk.User
if count+limit > len(allUsers) {
expected = allUsers[count:]
} else {
expected = allUsers[count : count+limit]
}
require.Equalf(t, page, expected, "next users, after=%s, limit=%d", afterCursor, limit)
require.Equalf(t, offsetPage, expected, "offset users, offset=%d, limit=%d", count, limit)
// Also check the before
prevPage, err := client.Users(ctx, opt(codersdk.UsersRequest{
Offset: count - limit,
Limit: limit,
}))
require.NoError(t, err, "prev page")
require.Equal(t, allUsers[count-limit:count], prevPage, "prev users")
count += len(page)
}
}

View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"time"
"github.com/google/uuid"
@ -13,6 +14,20 @@ import (
// Me is used as a replacement for your own ID.
var Me = uuid.Nil
type UsersRequest struct {
AfterUser uuid.UUID `json:"after_user"`
Search string `json:"search"`
// Limit sets the maximum number of users to be returned
// in a single page. If the limit is <= 0, there is no limit
// and all users are returned.
Limit int `json:"limit"`
// Offset is used to indicate which page to return. An offset of 0
// returns the first 'limit' number of users.
// To get the next page, use offset=<limit>*<page_number>.
// Offset is 0 indexed, so the first record sits at offset 0.
Offset int `json:"offset"`
}
// User represents a user in Coder.
type User struct {
ID uuid.UUID `json:"id" validate:"required"`
@ -136,19 +151,6 @@ func (c *Client) UpdateUserProfile(ctx context.Context, userID uuid.UUID, req Up
return user, json.NewDecoder(res.Body).Decode(&user)
}
func (c *Client) GetUsers(ctx context.Context) ([]User, error) {
res, err := c.request(ctx, http.MethodGet, "/api/v2/users", nil)
if err != nil {
return []User{}, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return []User{}, readBodyAsError(res)
}
var users []User
return users, json.NewDecoder(res.Body).Decode(&users)
}
// CreateAPIKey generates an API key for the user ID provided.
func (c *Client) CreateAPIKey(ctx context.Context, userID uuid.UUID) (*GenerateAPIKeyResponse, error) {
res, err := c.request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/users/%s/keys", uuidOrMe(userID)), nil)
@ -210,6 +212,34 @@ func (c *Client) User(ctx context.Context, id uuid.UUID) (User, error) {
return user, json.NewDecoder(res.Body).Decode(&user)
}
// Users returns all users according to the request parameters. If no parameters are set,
// the default behavior is to return all users in a single page.
func (c *Client) Users(ctx context.Context, req UsersRequest) ([]User, error) {
res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/users"), nil, func(r *http.Request) {
q := r.URL.Query()
if req.AfterUser != uuid.Nil {
q.Set("after_user", req.AfterUser.String())
}
if req.Limit > 0 {
q.Set("limit", strconv.Itoa(req.Limit))
}
q.Set("offset", strconv.Itoa(req.Offset))
q.Set("search", req.Search)
r.URL.RawQuery = q.Encode()
})
if err != nil {
return []User{}, err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return []User{}, readBodyAsError(res)
}
var users []User
return users, json.NewDecoder(res.Body).Decode(&users)
}
// OrganizationsByUser returns all organizations the user is a member of.
func (c *Client) OrganizationsByUser(ctx context.Context, userID uuid.UUID) ([]Organization, error) {
res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/users/%s/organizations", uuidOrMe(userID)), nil)

View File

@ -71,13 +71,20 @@ export interface TemplateVersion {
}
// From codersdk/users.go:17:6.
export interface UsersRequest {
readonly search: string
readonly limit: int
readonly offset: int
}
// From codersdk/users.go:32:6.
export interface User {
readonly email: string
readonly username: string
readonly name: string
}
// From codersdk/users.go:25:6.
// From codersdk/users.go:40:6.
export interface CreateFirstUserRequest {
readonly email: string
readonly username: string
@ -85,42 +92,42 @@ export interface CreateFirstUserRequest {
readonly organization: string
}
// From codersdk/users.go:38:6.
// From codersdk/users.go:53:6.
export interface CreateUserRequest {
readonly email: string
readonly username: string
readonly password: string
}
// From codersdk/users.go:45:6.
// From codersdk/users.go:60:6.
export interface UpdateUserProfileRequest {
readonly email: string
readonly username: string
readonly name?: string
}
// From codersdk/users.go:52:6.
// From codersdk/users.go:67:6.
export interface LoginWithPasswordRequest {
readonly email: string
readonly password: string
}
// From codersdk/users.go:58:6.
// From codersdk/users.go:73:6.
export interface LoginWithPasswordResponse {
readonly session_token: string
}
// From codersdk/users.go:63:6.
// From codersdk/users.go:78:6.
export interface GenerateAPIKeyResponse {
readonly key: string
}
// From codersdk/users.go:67:6.
// From codersdk/users.go:82:6.
export interface CreateOrganizationRequest {
readonly name: string
}
// From codersdk/users.go:72:6.
// From codersdk/users.go:87:6.
export interface CreateWorkspaceRequest {
readonly name: string
}