diff --git a/go.mod b/go.mod
index b5ac06e7da..a5a1306614 100644
--- a/go.mod
+++ b/go.mod
@@ -99,7 +99,6 @@ require (
 	github.com/creack/pty v1.1.18
 	github.com/dave/dst v0.27.2
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
-	github.com/djherbis/times v1.6.0
 	github.com/elastic/go-sysinfo v1.11.0
 	github.com/fatih/color v1.16.0
 	github.com/fatih/structs v1.1.0
diff --git a/go.sum b/go.sum
index 469ce08938..4e44308a0e 100644
--- a/go.sum
+++ b/go.sum
@@ -278,8 +278,6 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
 github.com/dhui/dktest v0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw=
-github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
-github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
 github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
 github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=
@@ -1212,7 +1210,6 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
diff --git a/provisionersdk/cleanup.go b/provisionersdk/cleanup.go
index 8f940546cb..b515c636b4 100644
--- a/provisionersdk/cleanup.go
+++ b/provisionersdk/cleanup.go
@@ -5,7 +5,6 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/djherbis/times"
 	"github.com/spf13/afero"
 	"golang.org/x/xerrors"
 
@@ -27,13 +26,9 @@ func CleanStaleSessions(ctx context.Context, workDirectory string, fs afero.Fs,
 		if fi.IsDir() && isValidSessionDir(dirName) {
 			sessionDirPath := filepath.Join(workDirectory, dirName)
 
-			accessTime := fi.ModTime() // fallback to modTime if accessTime is not available (afero)
-			if fi.Sys() != nil {
-				timeSpec := times.Get(fi)
-				accessTime = timeSpec.AccessTime()
-			}
+			modTime := fi.ModTime() // fallback to modTime if modTime is not available (afero)
 
-			if accessTime.Add(staleSessionRetention).After(now) {
+			if modTime.Add(staleSessionRetention).After(now) {
 				continue
 			}
 
diff --git a/provisionersdk/cleanup_test.go b/provisionersdk/cleanup_test.go
index cf0296cb05..ae9a370c1e 100644
--- a/provisionersdk/cleanup_test.go
+++ b/provisionersdk/cleanup_test.go
@@ -19,16 +19,18 @@ import (
 
 const workDirectory = "/tmp/coder/provisioner-34/work"
 
+var now = time.Date(2023, time.June, 3, 4, 5, 6, 0, time.UTC)
+
 func TestStaleSessions(t *testing.T) {
 	t.Parallel()
 
-	prepare := func() (afero.Fs, time.Time, slog.Logger) {
-		fs := afero.NewMemMapFs()
-		now := time.Date(2023, time.June, 3, 4, 5, 6, 0, time.UTC)
+	prepare := func() (afero.Fs, slog.Logger) {
+		tempDir := t.TempDir()
+		fs := afero.NewBasePathFs(afero.NewOsFs(), tempDir)
 		logger := slogtest.Make(t, nil).
 			Leveled(slog.LevelDebug).
 			Named("cleanup-test")
-		return fs, now, logger
+		return fs, logger
 	}
 
 	t.Run("all sessions are stale", func(t *testing.T) {
@@ -37,7 +39,7 @@ func TestStaleSessions(t *testing.T) {
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
 
-		fs, now, logger := prepare()
+		fs, logger := prepare()
 
 		// given
 		first := provisionersdk.SessionDir(uuid.NewString())
@@ -62,7 +64,7 @@ func TestStaleSessions(t *testing.T) {
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
 
-		fs, now, logger := prepare()
+		fs, logger := prepare()
 
 		// given
 		first := provisionersdk.SessionDir(uuid.NewString())
@@ -86,7 +88,7 @@ func TestStaleSessions(t *testing.T) {
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
 
-		fs, now, logger := prepare()
+		fs, logger := prepare()
 
 		// given
 		first := provisionersdk.SessionDir(uuid.NewString())
@@ -104,9 +106,9 @@ func TestStaleSessions(t *testing.T) {
 	})
 }
 
-func addSessionFolder(t *testing.T, fs afero.Fs, sessionName string, accessTime time.Time) {
+func addSessionFolder(t *testing.T, fs afero.Fs, sessionName string, modTime time.Time) {
 	err := fs.MkdirAll(filepath.Join(workDirectory, sessionName), 0o755)
 	require.NoError(t, err, "can't create session folder")
-	fs.Chtimes(filepath.Join(workDirectory, sessionName), accessTime, accessTime)
+	require.NoError(t, fs.Chtimes(filepath.Join(workDirectory, sessionName), now, modTime), "can't chtime of session dir")
 	require.NoError(t, err, "can't set times")
 }