test: verify that enterprise tests are being run (#12871)

Makefile

@@ -382,9 +382,9 @@ install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT)
 	cp "$<" "$$output_file"
 .PHONY: install
 
-BOLD := $(shell tput bold)
-GREEN := $(shell tput setaf 2)
-RESET := $(shell tput sgr0)
+BOLD := $(shell tput bold 2>/dev/null)
+GREEN := $(shell tput setaf 2 2>/dev/null)
+RESET := $(shell tput sgr0 2>/dev/null)
 
 fmt: fmt/eslint fmt/prettier fmt/terraform fmt/shfmt fmt/go
 .PHONY: fmt

site/e2e/constants.ts

@@ -33,4 +33,7 @@ export const gitAuth = {
   installationsPath: "/installations",
 };
 
+export const requireEnterpriseTests = Boolean(
+  process.env.CODER_E2E_REQUIRE_ENTERPRISE_TESTS,
+);
 export const enterpriseLicense = process.env.CODER_E2E_ENTERPRISE_LICENSE ?? "";

site/e2e/global.setup.ts

@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { expect, test } from "@playwright/test";
 import { Language } from "pages/CreateUserPage/CreateUserForm";
 import * as constants from "./constants";
 import { storageState } from "./playwright.config";
@@ -18,7 +18,12 @@ test("setup deployment", async ({ page }) => {
   await page.getByTestId("button-select-template").isVisible();
 
   // Setup license
-  if (constants.enterpriseLicense) {
+  if (constants.requireEnterpriseTests || constants.enterpriseLicense) {
+    // Make sure that we have something that looks like a real license
+    expect(constants.enterpriseLicense).toBeTruthy();
+    expect(constants.enterpriseLicense.length).toBeGreaterThan(92); // the signature alone should be this long
+    expect(constants.enterpriseLicense.split(".").length).toBe(3); // otherwise it's invalid
+
     await page.goto("/deployment/licenses", { waitUntil: "domcontentloaded" });
 
     await page.getByText("Add a license").click();

site/e2e/helpers.ts

@@ -18,6 +18,7 @@ import {
   coderPort,
   enterpriseLicense,
   prometheusPort,
+  requireEnterpriseTests,
 } from "./constants";
 import {
   Agent,
@@ -33,6 +34,10 @@ import {
 
 // requiresEnterpriseLicense will skip the test if we're not running with an enterprise license
 export function requiresEnterpriseLicense() {
+  if (requireEnterpriseTests) {
+    return;
+  }
+
   test.skip(!enterpriseLicense);
 }
 

site/e2e/reporter.ts

@@ -11,12 +11,13 @@ import type {
 import axios from "axios";
 import * as fs from "fs/promises";
 import type { Writable } from "stream";
-import { coderdPProfPort } from "./constants";
+import { coderdPProfPort, enterpriseLicense } from "./constants";
 
 class CoderReporter implements Reporter {
   config: FullConfig | null = null;
   testOutput = new Map<string, Array<[Writable, string]>>();
   passedCount = 0;
+  skippedCount = 0;
   failedTests: TestCase[] = [];
   timedOutTests: TestCase[] = [];
 
@@ -31,45 +32,56 @@ class CoderReporter implements Reporter {
   }
 
   onStdOut(chunk: string, test?: TestCase, _?: TestResult): void {
-    for (const line of filteredServerLogLines(chunk)) {
-      console.log(`[stdout] ${line}`);
-    }
+    // If there's no associated test, just print it now
     if (!test) {
+      for (const line of logLines(chunk)) {
+        console.log(`[stdout] ${line}`);
+      }
       return;
     }
+    // Will be printed if the test fails
     this.testOutput.get(test.id)!.push([process.stdout, chunk]);
   }
 
   onStdErr(chunk: string, test?: TestCase, _?: TestResult): void {
-    for (const line of filteredServerLogLines(chunk)) {
-      console.error(`[stderr] ${line}`);
-    }
+    // If there's no associated test, just print it now
     if (!test) {
+      for (const line of logLines(chunk)) {
+        console.error(`[stderr] ${line}`);
+      }
       return;
     }
+    // Will be printed if the test fails
     this.testOutput.get(test.id)!.push([process.stderr, chunk]);
   }
 
   async onTestEnd(test: TestCase, result: TestResult) {
-    console.log(`==> Finished test ${test.title}: ${result.status}`);
+    try {
+      if (test.expectedStatus === "skipped") {
+        console.log(`==> Skipping test ${test.title}`);
+        this.skippedCount++;
+        return;
+      }
 
-    if (result.status === "passed") {
-      this.passedCount++;
-    }
+      console.log(`==> Finished test ${test.title}: ${result.status}`);
 
-    if (result.status === "failed") {
-      this.failedTests.push(test);
-    }
+      if (result.status === "passed") {
+        this.passedCount++;
+        return;
+      }
 
-    if (result.status === "timedOut") {
-      this.timedOutTests.push(test);
-    }
+      if (result.status === "failed") {
+        this.failedTests.push(test);
+      }
 
-    const fsTestTitle = test.title.replaceAll(" ", "-");
-    const outputFile = `test-results/debug-pprof-goroutine-${fsTestTitle}.txt`;
-    await exportDebugPprof(outputFile);
+      if (result.status === "timedOut") {
+        this.timedOutTests.push(test);
+      }
+
+      const fsTestTitle = test.title.replaceAll(" ", "-");
+      const outputFile = `test-results/debug-pprof-goroutine-${fsTestTitle}.txt`;
+      await exportDebugPprof(outputFile);
 
-    if (result.status !== "passed") {
       console.log(`Data from pprof has been saved to ${outputFile}`);
       console.log("==> Output");
       const output = this.testOutput.get(test.id)!;
@@ -90,13 +102,22 @@ class CoderReporter implements Reporter {
           console.log(attachment);
         }
       }
+    } finally {
+      this.testOutput.delete(test.id);
     }
-    this.testOutput.delete(test.id);
   }
 
   onEnd(result: FullResult) {
     console.log(`==> Tests ${result.status}`);
+    if (!enterpriseLicense) {
+      console.log(
+        "==> Enterprise tests were skipped, because no license was provided",
+      );
+    }
     console.log(`${this.passedCount} passed`);
+    if (this.skippedCount > 0) {
+      console.log(`${this.skippedCount} skipped`);
+    }
     if (this.failedTests.length > 0) {
       console.log(`${this.failedTests.length} failed`);
       for (const test of this.failedTests) {
@@ -112,11 +133,7 @@ class CoderReporter implements Reporter {
   }
 }
 
-const shouldPrintLine = (line: string) =>
-  ["  error=EOF", "coderd: audit_log"].every((noise) => !line.includes(noise));
-
-const filteredServerLogLines = (chunk: string): string[] =>
-  chunk.trimEnd().split("\n").filter(shouldPrintLine);
+const logLines = (chunk: string): string[] => chunk.trimEnd().split("\n");
 
 const exportDebugPprof = async (outputFile: string) => {
   const response = await axios.get(

site/e2e/tests/enterprise.spec.ts

@@ -0,0 +1,10 @@
+import { expect, test } from "@playwright/test";
+import { requiresEnterpriseLicense } from "../helpers";
+
+test("license was added successfully", async ({ page }) => {
+  requiresEnterpriseLicense();
+
+  await page.goto("/deployment/licenses", { waitUntil: "domcontentloaded" });
+  const license = page.locator(".MuiPaper-root", { hasText: "#1" });
+  await expect(license).toBeVisible();
+});