feature: Load workspace build logs from streaming (#1997)

.vscode/settings.json

@@ -72,6 +72,7 @@
     "VMID",
     "weblinks",
     "webrtc",
+    "workspacebuilds",
     "xerrors",
     "xstate",
     "yamux"

site/can-ndjson-stream.d.ts

@@ -0,0 +1,4 @@
+declare module "can-ndjson-stream" {
+  function ndjsonStream<TValueType>(body: ReadableStream<Uint8Array> | null): Promise<ReadableStream<TValueType>>
+  export default ndjsonStream
+}

site/package.json

@@ -35,6 +35,7 @@
     "@xstate/inspect": "0.6.5",
     "@xstate/react": "3.0.0",
     "axios": "0.26.1",
+    "can-ndjson-stream": "1.0.2",
     "cronstrue": "2.5.0",
     "dayjs": "1.11.2",
     "formik": "2.2.9",

site/src/api/api.ts

@@ -1,4 +1,5 @@
 import axios, { AxiosRequestHeaders } from "axios"
+import ndjsonStream from "can-ndjson-stream"
 import * as Types from "./types"
 import { WorkspaceBuildTransition } from "./types"
 import * as TypesGen from "./typesGenerated"
@@ -271,6 +272,20 @@ export const getWorkspaceBuildLogs = async (buildname: string): Promise<TypesGen
   return response.data
 }
 
+export const streamWorkspaceBuildLogs = async (
+  buildname: string,
+): Promise<ReadableStreamDefaultReader<TypesGen.ProvisionerJobLog>> => {
+  // Axios does not support HTTP stream in the browser
+  // https://github.com/axios/axios/issues/1474
+  // So we are going to use window.fetch and return a "stream" reader
+  const reader = await window
+    .fetch(`/api/v2/workspacebuilds/${buildname}/logs?follow=true`)
+    .then((res) => ndjsonStream<TypesGen.ProvisionerJobLog>(res.body))
+    .then((stream) => stream.getReader())
+
+  return reader
+}
+
 export const putWorkspaceExtension = async (
   workspaceId: string,
   extendWorkspaceRequest: TypesGen.PutExtendWorkspaceRequest,

site/src/components/WorkspaceBuildLogs/WorkspaceBuildLogs.stories.tsx

@@ -13,3 +13,9 @@ export const Example = Template.bind({})
 Example.args = {
   logs: MockWorkspaceBuildLogs,
 }
+
+export const Loading = Template.bind({})
+Loading.args = {
+  logs: MockWorkspaceBuildLogs,
+  isWaitingForLogs: true,
+}

site/src/components/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx

@@ -1,3 +1,4 @@
+import CircularProgress from "@material-ui/core/CircularProgress"
 import { makeStyles } from "@material-ui/core/styles"
 import dayjs from "dayjs"
 import { FC } from "react"
@@ -5,6 +6,10 @@ import { ProvisionerJobLog } from "../../api/typesGenerated"
 import { MONOSPACE_FONT_FAMILY } from "../../theme/constants"
 import { Logs } from "../Logs/Logs"
 
+const Language = {
+  seconds: "seconds",
+}
+
 type Stage = ProvisionerJobLog["stage"]
 
 const groupLogsByStage = (logs: ProvisionerJobLog[]) => {
@@ -35,16 +40,17 @@ const getStageDurationInSeconds = (logs: ProvisionerJobLog[]) => {
 
 export interface WorkspaceBuildLogsProps {
   logs: ProvisionerJobLog[]
+  isWaitingForLogs: boolean
 }
 
-export const WorkspaceBuildLogs: FC<WorkspaceBuildLogsProps> = ({ logs }) => {
+export const WorkspaceBuildLogs: FC<WorkspaceBuildLogsProps> = ({ logs, isWaitingForLogs }) => {
   const groupedLogsByStage = groupLogsByStage(logs)
   const stages = Object.keys(groupedLogsByStage)
   const styles = useStyles()
 
   return (
     <div className={styles.logs}>
-      {stages.map((stage) => {
+      {stages.map((stage, stageIndex) => {
         const logs = groupedLogsByStage[stage]
         const isEmpty = logs.every((log) => log.output === "")
         const lines = logs.map((log) => ({
@@ -52,12 +58,20 @@ export const WorkspaceBuildLogs: FC<WorkspaceBuildLogsProps> = ({ logs }) => {
           output: log.output,
         }))
         const duration = getStageDurationInSeconds(logs)
+        const isLastStage = stageIndex === stages.length - 1
+        const shouldDisplaySpinner = isWaitingForLogs && isLastStage
+        const shouldDisplayDuration = !isWaitingForLogs && duration
 
         return (
           <div key={stage}>
             <div className={styles.header}>
               <div>{stage}</div>
-              {duration && <div className={styles.duration}>{duration} seconds</div>}
+              {shouldDisplaySpinner && <CircularProgress size={14} className={styles.spinner} />}
+              {shouldDisplayDuration && (
+                <div className={styles.duration}>
+                  {duration} {Language.seconds}
+                </div>
+              )}
             </div>
             {!isEmpty && <Logs lines={lines} className={styles.codeBlock} />}
           </div>
@@ -78,6 +92,7 @@ const useStyles = makeStyles((theme) => ({
     fontSize: theme.typography.body1.fontSize,
     padding: theme.spacing(2),
     paddingLeft: theme.spacing(4),
+    paddingRight: theme.spacing(4),
     borderBottom: `1px solid ${theme.palette.divider}`,
     backgroundColor: theme.palette.background.paper,
     display: "flex",
@@ -94,4 +109,8 @@ const useStyles = makeStyles((theme) => ({
     padding: theme.spacing(2),
     paddingLeft: theme.spacing(4),
   },
+
+  spinner: {
+    marginLeft: "auto",
+  },
 }))

site/src/pages/WorkspaceBuildPage/WorkspaceBuildPage.test.tsx

@@ -1,9 +1,21 @@
 import { screen } from "@testing-library/react"
+import * as API from "../../api/api"
 import { MockWorkspaceBuild, MockWorkspaceBuildLogs, renderWithAuth } from "../../testHelpers/renderHelpers"
 import { WorkspaceBuildPage } from "./WorkspaceBuildPage"
 
 describe("WorkspaceBuildPage", () => {
   it("renders the stats and logs", async () => {
+    jest.spyOn(API, "streamWorkspaceBuildLogs").mockResolvedValueOnce({
+      read() {
+        return Promise.resolve({
+          value: undefined,
+          done: true,
+        })
+      },
+      releaseLock: jest.fn(),
+      closed: Promise.resolve(undefined),
+      cancel: jest.fn(),
+    })
     renderWithAuth(<WorkspaceBuildPage />, { route: `/builds/${MockWorkspaceBuild.id}`, path: "/builds/:buildId" })
 
     await screen.findByText(MockWorkspaceBuild.workspace_name)

site/src/pages/WorkspaceBuildPage/WorkspaceBuildPage.tsx

@@ -29,6 +29,7 @@ export const WorkspaceBuildPage: FC = () => {
   const buildId = useBuildId()
   const [buildState] = useMachine(workspaceBuildMachine, { context: { buildId } })
   const { logs, build } = buildState.context
+  const isWaitingForLogs = !buildState.matches("logs.loaded")
   const styles = useStyles()
 
   return (
@@ -40,7 +41,7 @@ export const WorkspaceBuildPage: FC = () => {
 
         {build && <WorkspaceBuildStats build={build} />}
         {!logs && <Loader />}
-        {logs && <WorkspaceBuildLogs logs={sortLogsByCreatedAt(logs)} />}
+        {logs && <WorkspaceBuildLogs logs={sortLogsByCreatedAt(logs)} isWaitingForLogs={isWaitingForLogs} />}
       </Stack>
     </Margins>
   )

site/src/xServices/workspaceBuild/workspaceBuildXService.ts

@@ -9,19 +9,28 @@ type LogsContext = {
   getBuildError?: Error | unknown
   // Logs
   logs?: ProvisionerJobLog[]
-  getBuildLogsError?: Error | unknown
 }
 
+type LogsEvent =
+  | {
+      type: "ADD_LOG"
+      log: ProvisionerJobLog
+    }
+  | {
+      type: "NO_MORE_LOGS"
+    }
+
 export const workspaceBuildMachine = createMachine(
   {
     id: "workspaceBuildState",
     schema: {
       context: {} as LogsContext,
+      events: {} as LogsEvent,
       services: {} as {
         getWorkspaceBuild: {
           data: WorkspaceBuild
         }
-        getWorkspaceBuildLogs: {
+        getLogs: {
           data: ProvisionerJobLog[]
         }
       },
@@ -50,23 +59,36 @@ export const workspaceBuildMachine = createMachine(
         },
       },
       logs: {
-        initial: "gettingLogs",
+        initial: "gettingExistentLogs",
         states: {
-          gettingLogs: {
-            entry: "clearGetBuildLogsError",
+          gettingExistentLogs: {
             invoke: {
-              src: "getWorkspaceBuildLogs",
+              id: "getLogs",
+              src: "getLogs",
               onDone: {
-                target: "idle",
-                actions: "assignLogs",
-              },
-              onError: {
-                target: "idle",
-                actions: "assignGetBuildLogsError",
+                actions: ["assignLogs"],
+                target: "watchingLogs",
               },
             },
           },
-          idle: {},
+          watchingLogs: {
+            id: "watchingLogs",
+            invoke: {
+              id: "streamWorkspaceBuildLogs",
+              src: "streamWorkspaceBuildLogs",
+            },
+          },
+          loaded: {
+            type: "final",
+          },
+        },
+        on: {
+          ADD_LOG: {
+            actions: "addLog",
+          },
+          NO_MORE_LOGS: {
+            target: "logs.loaded",
+          },
         },
       },
     },
@@ -87,16 +109,32 @@ export const workspaceBuildMachine = createMachine(
       assignLogs: assign({
         logs: (_, event) => event.data,
       }),
-      assignGetBuildLogsError: assign({
-        getBuildLogsError: (_, event) => event.data,
-      }),
-      clearGetBuildLogsError: assign({
-        getBuildLogsError: (_) => undefined,
+      addLog: assign({
+        logs: (context, event) => {
+          const previousLogs = context.logs ?? []
+          return [...previousLogs, event.log]
+        },
       }),
     },
     services: {
       getWorkspaceBuild: (ctx) => API.getWorkspaceBuild(ctx.buildId),
-      getWorkspaceBuildLogs: (ctx) => API.getWorkspaceBuildLogs(ctx.buildId),
+      getLogs: async (ctx) => API.getWorkspaceBuildLogs(ctx.buildId),
+      streamWorkspaceBuildLogs: (ctx) => async (callback) => {
+        const reader = await API.streamWorkspaceBuildLogs(ctx.buildId)
+
+        // Watching for the stream
+        // eslint-disable-next-line no-constant-condition, @typescript-eslint/no-unnecessary-condition
+        while (true) {
+          const { value, done } = await reader.read()
+
+          if (done) {
+            callback("NO_MORE_LOGS")
+            break
+          }
+
+          callback({ type: "ADD_LOG", log: value })
+        }
+      },
     },
   },
 )

site/tsconfig.test.json

@@ -1,5 +1,5 @@
 {
   "extends": "./tsconfig.json",
   "exclude": ["node_modules", "_jest"],
-  "include": ["**/*.stories.tsx", "**/*.test.tsx"]
+  "include": ["**/*.stories.tsx", "**/*.test.tsx", "**/*.d.ts"]
 }

site/yarn.lock

@@ -4731,6 +4731,18 @@ camelcase@^6.2.0:
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a"
   integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
 
+can-namespace@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/can-namespace/-/can-namespace-1.0.0.tgz#0b8fafafbb11352b9ead4222ffe3822405b43e99"
+  integrity sha512-1sBY/SLwwcmxz3NhyVhLjt2uD/dZ7V1mII82/MIXSDn5QXnslnosJnjlP8+yTx2uTCRvw1jlFDElRs4pX7AG5w==
+
+can-ndjson-stream@1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/can-ndjson-stream/-/can-ndjson-stream-1.0.2.tgz#6a8131f9c8c697215163b3fe49a0c02e4439cb47"
+  integrity sha512-//tM8wcTV42SyD1JGua7WMVftZEeTwapcHJTTe3vJwuVywXD01CJbdEkgwRYjy2evIByVJV21ZKBdSv5ygIw1w==
+  dependencies:
+    can-namespace "^1.0.0"
+
 caniuse-api@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/caniuse-api/-/caniuse-api-3.0.0.tgz#5e4d90e2274961d46291997df599e3ed008ee4c0"