feat(scaletest/templates): run all scenarios even on failure (#10290)

We now also end all failed phases and add the `error` tag in Grafana.

.github/workflows/ci.yaml

@@ -223,7 +223,7 @@ jobs:
           go-version: 1.20.10
 
       - name: Install shfmt
-        run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.5.0
+        run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.7.0
 
       - name: make fmt
         run: |

scaletest/templates/scaletest-runner/scripts/lib.sh

@@ -87,7 +87,7 @@ end_phase() {
 	log "End phase ${phase_num}: ${phase}"
 	echo "$(date -Ins) END:${phase_num}: ${phase}" >>"${SCALETEST_PHASE_FILE}"
 
-	GRAFANA_EXTRA_TAGS="${PHASE_TYPE:-phase-default}" annotate_grafana_end "phase" "Phase ${phase_num}: ${phase}"
+	GRAFANA_EXTRA_TAGS="${PHASE_TYPE:-phase-default}" GRAFANA_ADD_TAGS="${PHASE_ADD_TAGS:-}" annotate_grafana_end "phase" "Phase ${phase_num}: ${phase}"
 }
 get_phase() {
 	if [[ -f "${SCALETEST_PHASE_FILE}" ]]; then
@@ -183,11 +183,20 @@ annotate_grafana_end() {
 
 	log "Annotating Grafana (end=${end}): ${text} [${tags}]"
 
-	json="$(
-		jq \
-			--argjson timeEnd "${end}" \
-			'{timeEnd: $timeEnd}' <<<'{}'
-	)"
+	if [[ -n ${GRAFANA_ADD_TAGS:-} ]]; then
+		json="$(
+			jq -n \
+				--argjson timeEnd "${end}" \
+				--argjson tags "${tags},${GRAFANA_ADD_TAGS}" \
+				'{timeEnd: $timeEnd, tags: $tags | split(",")}'
+		)"
+	else
+		json="$(
+			jq -n \
+				--argjson timeEnd "${end}" \
+				'{timeEnd: $timeEnd}'
+		)"
+	fi
 	if [[ ${DRY_RUN} == 1 ]]; then
 		log "Would have patched Grafana annotation: id=${id}, data=${json}"
 		return 0

scaletest/templates/scaletest-runner/scripts/run.sh

@@ -26,8 +26,12 @@ end_phase
 
 wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}"
 
+declare -A failed=()
 for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
 	start_phase "Load scenario: ${scenario}"
+
+	set +e
+	status=0
 	case "${scenario}" in
 	"SSH Traffic")
 		coder exp scaletest workspace-traffic \
@@ -37,6 +41,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
 			--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m" \
 			--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_SSH_TRAFFIC_DURATION}m30s" \
 			--output json:"${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
+		status=$?
 		show_json "${SCALETEST_RESULTS_DIR}/traffic-ssh.json"
 		;;
 	"Web Terminal Traffic")
@@ -46,6 +51,7 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
 			--timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m" \
 			--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_WEB_TERMINAL_TRAFFIC_DURATION}m30s" \
 			--output json:"${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json"
+		status=$?
 		show_json "${SCALETEST_RESULTS_DIR}/traffic-web-terminal.json"
 		;;
 	"Dashboard Traffic")
@@ -54,13 +60,39 @@ for scenario in "${SCALETEST_PARAM_LOAD_SCENARIOS[@]}"; do
 			--job-timeout "${SCALETEST_PARAM_LOAD_SCENARIO_DASHBOARD_TRAFFIC_DURATION}m30s" \
 			--output json:"${SCALETEST_RESULTS_DIR}/traffic-dashboard.json" \
 			>"${SCALETEST_RESULTS_DIR}/traffic-dashboard-output.log"
+		status=$?
 		show_json "${SCALETEST_RESULTS_DIR}/traffic-dashboard.json"
 		;;
+
+	# Debug scenarios, for testing the runner.
+	"debug:success")
+		maybedryrun "$DRY_RUN" sleep 10
+		status=0
+		;;
+	"debug:error")
+		maybedryrun "$DRY_RUN" sleep 10
+		status=1
+		;;
 	esac
-	end_phase
+	set -e
+	if ((status > 0)); then
+		log "Load scenario failed: ${scenario} (exit=${status})"
+		failed+=(["${scenario}"]="$status")
+		PHASE_ADD_TAGS=error end_phase
+	else
+		end_phase
+	fi
 
 	wait_baseline "${SCALETEST_PARAM_LOAD_SCENARIO_BASELINE_DURATION}"
 done
 
+if ((${#failed[@]} > 0)); then
+	log "Load scenarios failed: ${!failed[*]}"
+	for scenario in "${!failed[@]}"; do
+		log "  ${scenario}: exit=${failed[$scenario]}"
+	done
+	exit 1
+fi
+
 log "Scaletest complete!"
 set_status Complete