danny8632/darkforge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trying to get testing working

Browse Source
This commit is contained in:
Danny
2026-03-20 07:20:31 +01:00
parent 379451020f
commit 826c1c890a
2 changed files with 312 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

321
tests/run-tests.sh
View File

@@ -2,20 +2,20 @@
# ============================================================================
# DarkForge Linux — Integration Test Runner
# ============================================================================
# Purpose: Run automated integration tests on an Arch Linux host with QEMU.
# Purpose: Run automated integration tests on an Arch Linux host.
# Generates a machine-readable report (JSON + human-readable summary)
# that can be fed back to the development process for fixing issues.
#
# Requirements:
# - Arch Linux (x86_64) host
# - Packages: qemu-full ovmf rust cargo base-devel git wget
# - Packages: qemu-full edk2-ovmf rust cargo base-devel git wget
# sudo pacman -S qemu-full edk2-ovmf rust cargo base-devel git wget
# - ~30GB free disk space
# - Internet access (for downloading sources during sign test)
# - Internet access (for package signing tests)
#
# Usage:
# bash tests/run-tests.sh # run all tests
# bash tests/run-tests.sh --quick # skip QEMU tests (dpack only)
# bash tests/run-tests.sh --quick # skip QEMU + long tests
# bash tests/run-tests.sh --report # generate report and exit
#
# Output:
@@ -29,6 +29,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
REPORT_JSON="${SCRIPT_DIR}/report.json"
REPORT_TXT="${SCRIPT_DIR}/report.txt"
LOG_DIR="${SCRIPT_DIR}/logs"
QUICK_MODE=false
# Parse args
@@ -38,6 +39,8 @@ for arg in "$@"; do
esac
done
mkdir -p "${LOG_DIR}"
# --- Colors -----------------------------------------------------------------
RED='\033[0;31m'
GREEN='\033[0;32m'
@@ -60,6 +63,9 @@ record_test() {
local detail="${3:-}"
local duration="${4:-0}"
# Escape double quotes in detail for valid JSON
detail=$(echo "$detail" | sed 's/"/\\"/g' | tr '\n' ' ')
TESTS+=("{\"name\":\"${name}\",\"status\":\"${status}\",\"detail\":\"${detail}\",\"duration_s\":${duration}}")
case "$status" in
@@ -69,6 +75,23 @@ record_test() {
esac
}
timed_test() {
# Usage: timed_test "test.name" command args...
local name="$1"; shift
local t0=$(date +%s)
local output
output=$("$@" 2>&1)
local rc=$?
local t1=$(date +%s)
local dur=$((t1 - t0))
if [ $rc -eq 0 ]; then
record_test "$name" "pass" "" "$dur"
else
record_test "$name" "fail" "$(echo "$output" | tail -5 | tr '\n' ' ')" "$dur"
fi
return $rc
}
# ============================================================================
# TEST SUITE 1: Host Environment
# ============================================================================
@@ -89,7 +112,7 @@ else
fi
# Check required tools
for tool in gcc g++ make git wget curl cargo rustc qemu-system-x86_64 sha256sum; do
for tool in gcc g++ make git wget curl cargo rustc qemu-system-x86_64 sha256sum tar xz python3; do
if command -v "$tool" >/dev/null 2>&1; then
record_test "host.tool.${tool}" "pass"
else
@@ -101,14 +124,37 @@ for tool in gcc g++ make git wget curl cargo rustc qemu-system-x86_64 sha256sum;
fi
done
# Check OVMF
# Check nested virtualization support
if grep -qE '(vmx|svm)' /proc/cpuinfo 2>/dev/null; then
record_test "host.nested_virt" "pass"
else
record_test "host.nested_virt" "skip" "No VMX/SVM — QEMU boot tests will be slower"
fi
# Check OVMF — search all known paths including 4m variant (newer Arch/edk2)
OVMF_PATH=""
for p in /usr/share/ovmf/x64/OVMF.fd /usr/share/edk2/x64/OVMF.fd /usr/share/OVMF/OVMF.fd /usr/share/edk2-ovmf/x64/OVMF.fd; do
for p in \
/usr/share/edk2/x64/OVMF_CODE.4m.fd \
/usr/share/edk2/x64/OVMF_CODE.fd \
/usr/share/edk2-ovmf/x64/OVMF_CODE.4m.fd \
/usr/share/edk2-ovmf/x64/OVMF_CODE.fd \
/usr/share/OVMF/OVMF_CODE.4m.fd \
/usr/share/OVMF/OVMF_CODE.fd \
/usr/share/edk2/x64/OVMF.fd \
/usr/share/edk2-ovmf/x64/OVMF.fd \
/usr/share/ovmf/x64/OVMF.fd \
/usr/share/OVMF/OVMF.fd \
/usr/share/ovmf/OVMF.fd; do
if [ -f "$p" ]; then
OVMF_PATH="$p"
break
fi
done
# Last resort: search with find
if [ -z "$OVMF_PATH" ]; then
OVMF_PATH=$(find /usr/share -name "OVMF_CODE*.fd" -o -name "OVMF.fd" 2>/dev/null | head -1)
fi
if [ -n "$OVMF_PATH" ]; then
record_test "host.ovmf" "pass" "${OVMF_PATH}"
elif [ "$QUICK_MODE" = true ]; then
@@ -138,20 +184,19 @@ echo -e "\n${BOLD}=== Test Suite 2: dpack Build ===${NC}\n"
cd "${PROJECT_ROOT}/src/dpack"
# Build
# Build release
t_start=$(date +%s)
if cargo build --release 2>"${SCRIPT_DIR}/dpack-build.log"; then
if cargo build --release 2>"${LOG_DIR}/dpack-build.log"; then
t_end=$(date +%s)
record_test "dpack.build" "pass" "" "$((t_end - t_start))"
else
t_end=$(date +%s)
# Extract the error
err=$(tail -5 "${SCRIPT_DIR}/dpack-build.log" | tr '\n' ' ' | tr '"' "'")
err=$(tail -5 "${LOG_DIR}/dpack-build.log" | tr '\n' ' ' | tr '"' "'")
record_test "dpack.build" "fail" "${err}" "$((t_end - t_start))"
fi
# Check for warnings
WARNINGS=$(grep -c "^warning" "${SCRIPT_DIR}/dpack-build.log" 2>/dev/null || echo "0")
# Check for warnings (fix: separate the grep exit code from the count)
WARNINGS=$(grep -c "^warning" "${LOG_DIR}/dpack-build.log" 2>/dev/null) || WARNINGS=0
if [ "$WARNINGS" -eq 0 ]; then
record_test "dpack.no_warnings" "pass"
else
@@ -160,18 +205,19 @@ fi
# Unit tests
t_start=$(date +%s)
if cargo test 2>"${SCRIPT_DIR}/dpack-test.log"; then
if cargo test 2>"${LOG_DIR}/dpack-test.log"; then
t_end=$(date +%s)
record_test "dpack.unit_tests" "pass" "" "$((t_end - t_start))"
else
t_end=$(date +%s)
err=$(grep "^test result" "${SCRIPT_DIR}/dpack-test.log" | tr '"' "'")
err=$(grep "^test result" "${LOG_DIR}/dpack-test.log" | tr '"' "'")
record_test "dpack.unit_tests" "fail" "${err}" "$((t_end - t_start))"
fi
# CLI smoke tests
DPACK="${PROJECT_ROOT}/src/dpack/target/release/dpack"
if [ -x "$DPACK" ]; then
# Basic CLI tests
if $DPACK --version >/dev/null 2>&1; then
record_test "dpack.cli.version" "pass"
else
@@ -183,9 +229,77 @@ if [ -x "$DPACK" ]; then
else
record_test "dpack.cli.help" "fail" "dpack --help failed"
fi
# Extended CLI tests with temp config
DPACK_TEST_DIR=$(mktemp -d /tmp/dpack-test-XXXXX)
mkdir -p "${DPACK_TEST_DIR}"/{db,repos,sources,build}
ln -sf "${PROJECT_ROOT}/src/repos/core" "${DPACK_TEST_DIR}/repos/core"
ln -sf "${PROJECT_ROOT}/src/repos/extra" "${DPACK_TEST_DIR}/repos/extra"
ln -sf "${PROJECT_ROOT}/src/repos/desktop" "${DPACK_TEST_DIR}/repos/desktop"
ln -sf "${PROJECT_ROOT}/src/repos/gaming" "${DPACK_TEST_DIR}/repos/gaming"
cat > "${DPACK_TEST_DIR}/dpack.conf" << DCONF
[paths]
db_dir = "${DPACK_TEST_DIR}/db"
repo_dir = "${DPACK_TEST_DIR}/repos"
source_dir = "${DPACK_TEST_DIR}/sources"
build_dir = "${DPACK_TEST_DIR}/build"
[[repos]]
name = "core"
path = "${DPACK_TEST_DIR}/repos/core"
priority = 0
[[repos]]
name = "extra"
path = "${DPACK_TEST_DIR}/repos/extra"
priority = 10
[[repos]]
name = "desktop"
path = "${DPACK_TEST_DIR}/repos/desktop"
priority = 20
[[repos]]
name = "gaming"
path = "${DPACK_TEST_DIR}/repos/gaming"
priority = 30
DCONF
DPACK_CMD="$DPACK --config ${DPACK_TEST_DIR}/dpack.conf"
if $DPACK_CMD list >/dev/null 2>&1; then
record_test "dpack.cli.list" "pass"
else
record_test "dpack.cli.list" "fail" "Exit code $?"
fi
if $DPACK_CMD check >/dev/null 2>&1; then
record_test "dpack.cli.check" "pass"
else
record_test "dpack.cli.check" "fail" "Exit code $?"
fi
if $DPACK_CMD search zlib 2>/dev/null | grep -q "zlib"; then
record_test "dpack.cli.search" "pass"
else
record_test "dpack.cli.search" "fail" "zlib not found in search"
fi
if $DPACK_CMD info zlib 2>/dev/null | grep -qi "compression\|zlib"; then
record_test "dpack.cli.info" "pass"
else
record_test "dpack.cli.info" "fail" "info zlib returned no useful output"
fi
rm -rf "${DPACK_TEST_DIR}"
else
record_test "dpack.cli.version" "skip" "Binary not built"
record_test "dpack.cli.help" "skip" "Binary not built"
record_test "dpack.cli.list" "skip" "Binary not built"
record_test "dpack.cli.check" "skip" "Binary not built"
record_test "dpack.cli.search" "skip" "Binary not built"
record_test "dpack.cli.info" "skip" "Binary not built"
fi
cd "${PROJECT_ROOT}"
@@ -206,27 +320,70 @@ for repo in core extra desktop gaming; do
fi
done
# Validate TOML syntax (basic parse check)
# Validate TOML syntax (check required sections)
TOML_ERRORS=0
for toml in $(find "${PROJECT_ROOT}/src/repos" -name "*.toml" 2>/dev/null); do
# Check required sections exist
if ! grep -q '\[package\]' "$toml" || ! grep -q '\[source\]' "$toml" || ! grep -q '\[build\]' "$toml"; then
pkg_name=$(basename "$(dirname "$toml")")
record_test "repos.toml.${pkg_name}" "fail" "Missing required section"
((TOML_ERRORS++))
fi
pkg_name=$(basename "$(dirname "$toml")")
for section in '\[package\]' '\[source\]' '\[build\]'; do
if ! grep -q "$section" "$toml"; then
record_test "repos.validate.${pkg_name}" "fail" "Missing ${section}"
((TOML_ERRORS++))
break
fi
done
done
if [ "$TOML_ERRORS" -eq 0 ]; then
total=$(find "${PROJECT_ROOT}/src/repos" -name "*.toml" | wc -l)
record_test "repos.toml_validation" "pass" "All ${total} valid"
fi
# ============================================================================
# TEST SUITE 4: Toolchain Scripts
# ============================================================================
echo -e "\n${BOLD}=== Test Suite 4: Toolchain Scripts ===${NC}\n"
# Dependency resolution check (all deps must resolve within the repo tree)
if command -v python3 >/dev/null 2>&1; then
DEP_OUTPUT=$(PROJECT_ROOT="${PROJECT_ROOT}" python3 << 'PYEOF' 2>/dev/null
import os, re, sys
base = os.environ.get("PROJECT_ROOT", ".") + "/src/repos"
known = set()
for repo in ['core','extra','desktop','gaming']:
d = os.path.join(base, repo)
if not os.path.isdir(d): continue
for p in os.listdir(d):
if os.path.isdir(os.path.join(d,p)) and os.path.exists(os.path.join(d,p,f"{p}.toml")):
known.add(p)
missing = set()
for repo in ['core','extra','desktop','gaming']:
d = os.path.join(base, repo)
if not os.path.isdir(d): continue
for p in os.listdir(d):
tf = os.path.join(d,p,f"{p}.toml")
if not os.path.exists(tf): continue
with open(tf) as f:
content = f.read()
for m in re.finditer(r'(?:run|build)\s*=\s*\[(.*?)\]', content):
for dm in re.finditer(r'"([\w][\w.-]*)"', m.group(1)):
if dm.group(1) not in known:
missing.add(dm.group(1))
if missing:
print(f"MISSING:{','.join(sorted(missing))}")
sys.exit(1)
else:
print(f"OK:{len(known)}")
PYEOF
)
if [ $? -eq 0 ]; then
record_test "repos.deps_resolve" "pass" "All dependencies resolve"
else
record_test "repos.deps_resolve" "fail" "${DEP_OUTPUT}"
fi
else
record_test "repos.deps_resolve" "skip" "python3 not available"
fi
# Check all scripts exist and are executable
# ============================================================================
# TEST SUITE 4: Script Validation
# ============================================================================
echo -e "\n${BOLD}=== Test Suite 4: Script Validation ===${NC}\n"
# Toolchain scripts — all executable
MISSING_SCRIPTS=0
for script in ${PROJECT_ROOT}/toolchain/scripts/*.sh; do
if [ ! -x "$script" ]; then
@@ -239,7 +396,7 @@ if [ "$MISSING_SCRIPTS" -eq 0 ]; then
record_test "toolchain.all_executable" "pass" "${count} scripts"
fi
# Syntax check all bash scripts
# Toolchain scripts — syntax check
SYNTAX_ERRORS=0
for script in ${PROJECT_ROOT}/toolchain/scripts/*.sh; do
if ! bash -n "$script" 2>/dev/null; then
@@ -251,6 +408,39 @@ if [ "$SYNTAX_ERRORS" -eq 0 ]; then
record_test "toolchain.bash_syntax" "pass"
fi
# Init/rc.d scripts — individual syntax checks
for script in "${PROJECT_ROOT}"/configs/rc.d/*; do
[ -f "$script" ] || continue
name=$(basename "$script")
if bash -n "$script" 2>/dev/null; then
record_test "scripts.init.${name}" "pass"
else
record_test "scripts.init.${name}" "fail" "Syntax error"
fi
done
# Installer scripts — syntax checks
for script in "${PROJECT_ROOT}"/src/install/*.sh "${PROJECT_ROOT}"/src/install/modules/*.sh; do
[ -f "$script" ] || continue
name=$(basename "$script" .sh)
if bash -n "$script" 2>/dev/null; then
record_test "scripts.install.${name}" "pass"
else
record_test "scripts.install.${name}" "fail" "Syntax error"
fi
done
# ISO builder scripts — syntax checks
for script in "${PROJECT_ROOT}"/src/iso/*.sh; do
[ -f "$script" ] || continue
name=$(basename "$script" .sh)
if bash -n "$script" 2>/dev/null; then
record_test "scripts.iso.${name}" "pass"
else
record_test "scripts.iso.${name}" "fail" "Syntax error"
fi
done
# ============================================================================
# TEST SUITE 5: Kernel Config
# ============================================================================
@@ -260,8 +450,8 @@ KCONFIG="${PROJECT_ROOT}/kernel/config"
if [ -f "$KCONFIG" ]; then
record_test "kernel.config_exists" "pass"
# Check critical options
for opt in CONFIG_EFI_STUB CONFIG_BLK_DEV_NVME CONFIG_PREEMPT CONFIG_R8169 CONFIG_EXT4_FS CONFIG_MODULES; do
# Check critical enabled options
for opt in CONFIG_EFI_STUB CONFIG_BLK_DEV_NVME CONFIG_PREEMPT CONFIG_R8169 CONFIG_EXT4_FS CONFIG_MODULES CONFIG_SMP CONFIG_AMD_IOMMU; do
if grep -q "^${opt}=y" "$KCONFIG"; then
record_test "kernel.${opt}" "pass"
else
@@ -282,7 +472,7 @@ else
fi
# ============================================================================
# TEST SUITE 6: Init System
# TEST SUITE 6: Init System Files
# ============================================================================
echo -e "\n${BOLD}=== Test Suite 6: Init System ===${NC}\n"
@@ -308,45 +498,86 @@ for daemon in eudev syslog dbus dhcpcd pipewire; do
done
# ============================================================================
# TEST SUITE 7: QEMU Boot Test (skipped in quick mode)
# TEST SUITE 7: Package Signing (network test)
# ============================================================================
if [ "$QUICK_MODE" = false ] && [ -x "$DPACK" ]; then
echo -e "\n${BOLD}=== Test Suite 7: Package Signing ===${NC}\n"
ZLIB_TOML="${PROJECT_ROOT}/src/repos/core/zlib/zlib.toml"
if [ -f "$ZLIB_TOML" ]; then
cp "$ZLIB_TOML" "${ZLIB_TOML}.bak"
timed_test "sign.zlib" $DPACK sign zlib || true
# Check if it got a real hash (not the placeholder)
if grep -q 'sha256 = "aaa' "$ZLIB_TOML" 2>/dev/null; then
record_test "sign.zlib_result" "fail" "Checksum still placeholder after signing"
else
record_test "sign.zlib_result" "pass"
fi
mv "${ZLIB_TOML}.bak" "$ZLIB_TOML"
fi
else
if [ "$QUICK_MODE" = true ]; then
echo -e "\n${BOLD}=== Test Suite 7: Package Signing (SKIPPED) ===${NC}\n"
record_test "sign.zlib" "skip" "Quick mode"
fi
fi
# ============================================================================
# TEST SUITE 8: QEMU Boot Test (skipped in quick mode)
# ============================================================================
if [ "$QUICK_MODE" = false ] && [ -n "$OVMF_PATH" ]; then
echo -e "\n${BOLD}=== Test Suite 7: QEMU Boot Test ===${NC}\n"
echo -e "\n${BOLD}=== Test Suite 8: QEMU Boot Test ===${NC}\n"
ISO="${PROJECT_ROOT}/darkforge-live.iso"
if [ -f "$ISO" ]; then
echo " Testing ISO boot in QEMU (30s timeout)..."
# Create a temp disk image
echo " Testing ISO boot in QEMU (60s timeout)..."
QEMU_DISK=$(mktemp /tmp/darkforge-qemu-XXXXX.qcow2)
qemu-img create -f qcow2 "$QEMU_DISK" 20G >/dev/null 2>&1
# Boot QEMU with serial console, timeout after 30s
timeout 30 qemu-system-x86_64 \
-enable-kvm \
KVM_FLAG=""
[ -c /dev/kvm ] && KVM_FLAG="-enable-kvm"
# Build OVMF flags — split CODE/VARS files need -drive, single .fd uses -bios
OVMF_FLAGS=""
if echo "$OVMF_PATH" | grep -q "OVMF_CODE"; then
OVMF_VARS_TEMPLATE="$(dirname "$OVMF_PATH")/OVMF_VARS.fd"
# Try 4m variant first
if [ ! -f "$OVMF_VARS_TEMPLATE" ]; then
OVMF_VARS_TEMPLATE="$(dirname "$OVMF_PATH")/OVMF_VARS.4m.fd"
fi
OVMF_VARS_COPY="/tmp/darkforge-ovmf-vars.fd"
cp "$OVMF_VARS_TEMPLATE" "$OVMF_VARS_COPY" 2>/dev/null || dd if=/dev/zero of="$OVMF_VARS_COPY" bs=256K count=1 2>/dev/null
OVMF_FLAGS="-drive if=pflash,format=raw,readonly=on,file=${OVMF_PATH} -drive if=pflash,format=raw,file=${OVMF_VARS_COPY}"
else
OVMF_FLAGS="-bios ${OVMF_PATH}"
fi
timeout 60 qemu-system-x86_64 \
${KVM_FLAG} \
-m 2G \
-bios "$OVMF_PATH" \
-smp 2 \
${OVMF_FLAGS} \
-cdrom "$ISO" \
-drive file="$QEMU_DISK",format=qcow2,if=virtio \
-nographic \
-serial mon:stdio \
-no-reboot \
2>"${SCRIPT_DIR}/qemu.log" | head -100 > "${SCRIPT_DIR}/qemu-output.log" &
2>"${LOG_DIR}/qemu-stderr.log" | head -200 > "${LOG_DIR}/qemu-output.log" &
QEMU_PID=$!
sleep 30
sleep 60
kill $QEMU_PID 2>/dev/null
wait $QEMU_PID 2>/dev/null
# Check if we got kernel boot messages
if grep -q "Linux version" "${SCRIPT_DIR}/qemu-output.log" 2>/dev/null; then
if grep -qi "linux version\|darkforge\|kernel" "${LOG_DIR}/qemu-output.log" 2>/dev/null; then
record_test "qemu.kernel_boots" "pass"
else
record_test "qemu.kernel_boots" "fail" "No kernel boot messages in serial output"
fi
# Check if we got to userspace
if grep -q "login:" "${SCRIPT_DIR}/qemu-output.log" 2>/dev/null || \
grep -q "DarkForge" "${SCRIPT_DIR}/qemu-output.log" 2>/dev/null; then
if grep -qi "login:\|installer\|welcome\|darkforge" "${LOG_DIR}/qemu-output.log" 2>/dev/null; then
record_test "qemu.reaches_userspace" "pass"
else
record_test "qemu.reaches_userspace" "fail" "Did not reach login prompt"
@@ -359,7 +590,7 @@ if [ "$QUICK_MODE" = false ] && [ -n "$OVMF_PATH" ]; then
record_test "qemu.reaches_userspace" "skip" "No ISO"
fi
else
echo -e "\n${BOLD}=== Test Suite 7: QEMU Boot Test (SKIPPED) ===${NC}\n"
echo -e "\n${BOLD}=== Test Suite 8: QEMU Boot Test (SKIPPED) ===${NC}\n"
record_test "qemu.kernel_boots" "skip" "Quick mode or no OVMF"
fi