Test Suites

# Start runner for suite execution
keystone start --proxy --api-key your-api-key

# Suites are managed in Keystone Studio
# Execute suites against local environment
# Full visibility into suite progression
# Individual test results and timing

# 1. Start development environment
npm run dev

# 2. Start CLI runner with debug output
keystone start --proxy --debug

# 3. From Keystone Studio:
#    - Select suite to run (e.g., "Auth Flow")
#    - Configure to run against localhost:3000
#    - Execute suite with local browser visibility
#    - Monitor individual test results

# Development suite
# - Tests against localhost
# - Includes debugging and development features
# - Fast feedback for development changes

# Staging suite  
# - Tests against staging environment
# - Comprehensive feature validation
# - Pre-production verification

# Production suite
# - Health checks and monitoring
# - Critical path validation
# - Minimal invasive testing

# Authentication suite
tests/auth/
├── login-flow.json
├── registration.json
├── password-reset.json
└── session-management.json

# E-commerce suite
tests/ecommerce/
├── product-search.json
├── cart-management.json
├── checkout-flow.json
└── payment-processing.json

# Admin suite
tests/admin/
├── user-management.json
├── content-moderation.json
└── analytics-dashboard.json

# Smoke test suite (fast, critical paths)
# - Run on every commit
# - 5-10 tests, under 5 minutes
# - Basic functionality validation

# Regression suite (comprehensive)
# - Run nightly or before releases
# - Full feature coverage
# - Edge cases and error scenarios

# Performance suite (specialized)
# - Run on performance-critical changes
# - Load testing and timing validation
# - Resource usage monitoring

# Execute suite with full debugging
keystone start --proxy --debug --headed

# Benefits:
# - See each test execute in real browser
# - Debug failures immediately
# - Inspect element selectors
# - Monitor network requests
# - Access browser dev tools

# Test development cycle
# 1. Modify application code
# 2. Run relevant test suite
# 3. Debug any failures immediately
# 4. Iterate quickly on fixes
# 5. Verify changes before commit

# Test against local changes
git checkout feature/new-checkout
npm run dev
keystone start --proxy

# Execute checkout suite against localhost
# Validate feature works before deployment
# Catch integration issues early

# Development configuration
export NODE_ENV=development
export BASE_URL=http://localhost:3000
export API_URL=http://localhost:8000
keystone start --proxy

# Staging configuration  
export NODE_ENV=staging
export BASE_URL=https://staging.example.com
export API_URL=https://api-staging.example.com
keystone start --proxy

# Authentication suite setup
setup_auth_data() {
  npm run db:seed:users
  npm run cache:clear:sessions
}

# E-commerce suite setup
setup_ecommerce_data() {
  npm run db:seed:products
  npm run db:seed:orders
  npm run stripe:setup:test
}

# Run setup before suite execution
setup_auth_data
keystone start --proxy
# Execute authentication suite from Studio

# Monitor suite performance locally
keystone start --proxy --debug

# Captured metrics:
# - Total suite execution time
# - Individual test timing
# - Browser resource usage
# - Network request performance
# - Memory consumption patterns

# Optimize for faster suite execution
keystone start --proxy --headless --fast-mode

# Benefits:
# - Reduced browser overhead
# - Faster test execution
# - Lower resource usage
# - Parallel execution where possible

# Monitor resource usage during suite execution
# Memory usage
ps aux | grep keystone | awk '{print $4}'

# CPU usage  
top -p $(pgrep keystone)

# Disk usage for recordings
du -sh ~/.keystone/recordings/

# Detailed failure logs
tail -f ~/.keystone/logs/suite-execution.log

# Individual test failure details
grep "FAILED" ~/.keystone/logs/execution.log

# Screenshots of failure states
ls ~/.keystone/screenshots/failed/

# Run failing test in isolation
# 1. Identify failing test from suite
# 2. Execute single test with full debugging
keystone start --proxy --debug --headed
# 3. Debug specific failure
# 4. Fix and re-run full suite

# Debug suite incrementally
# 1. Run first few tests only
# 2. Verify they pass
# 3. Gradually add more tests
# 4. Isolate problematic tests
# 5. Fix issues and build up full suite

# Test CI configuration locally
export CI=true
export KEYSTONE_HEADLESS=true
keystone start --proxy --headless

# Run same suites that will run in CI
# Validate they pass in headless mode
# Debug any CI-specific issues locally

# Ensure local matches CI environment
# - Same Node.js version
# - Same browser version
# - Same environment variables
# - Same test data setup

# Use Docker for consistency
docker run -it node:18-slim
npm install -g @keystone/cli
keystone start --proxy --headless

# Pre-commit suite validation
git add .
npm run test:suite:critical
git commit -m "Feature: new checkout flow"

# Feature development validation
git checkout -b feature/user-profiles
npm run dev
keystone start --proxy
# Run user management suite against localhost

# Increase timeout for slow suites
export KEYSTONE_SUITE_TIMEOUT=3600000  # 1 hour

# Or optimize individual tests
keystone start --proxy --debug
# Identify slow tests and optimize

# Monitor memory usage during suite execution
# Restart runner between large suites
keystone start --proxy --memory-limit 2048

# Use more stable browser configuration
keystone start --proxy --browser-args "--disable-dev-shm-usage --no-sandbox"

# Comprehensive debugging for suite issues
keystone start --proxy --debug --headed --slow-motion 500

# Step through suite execution
# Identify exactly where failures occur
# Analyze browser state at failure points
# Check network requests and responses

# Execute different suites based on environment
if [ "$NODE_ENV" = "development" ]; then
  # Run development suite
  echo "Running development test suite"
elif [ "$NODE_ENV" = "staging" ]; then
  # Run staging suite  
  echo "Running staging test suite"
fi

# Run multiple suites in parallel (planned feature)
# Current: single suite execution
# Future: parallel execution for independent suites

# Ensure prerequisite suites pass first
# 1. Run authentication suite
# 2. If successful, run e-commerce suite
# 3. If successful, run admin suite
# Fail fast if any prerequisite fails