OpenEvidence CI Integration

Overview

Integrate OpenEvidence testing and validation into CI/CD pipelines for healthcare applications.

Prerequisites

• GitHub, GitLab, or other CI platform
• OpenEvidence sandbox API credentials
• Test suite configured
• Secret management in CI platform

Instructions

Step 1: GitHub Actions Workflow

# .github/workflows/openevidence-ci.yml
name: OpenEvidence CI

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]

env:
  NODE_VERSION: '20'

jobs:
  lint-and-typecheck:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Lint
        run: npm run lint

      - name: Type check
        run: npm run typecheck

  unit-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run unit tests
        run: npm run test:unit -- --coverage

      - name: Upload coverage
        uses: codecov/codecov-action@v4
        with:
          files: ./coverage/lcov.info
          fail_ci_if_error: false

  integration-tests:
    runs-on: ubuntu-latest
    # Only run on main branch or explicit trigger
    if: github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'run-integration')
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run integration tests
        env:
          OPENEVIDENCE_API_KEY: ${{ secrets.OPENEVIDENCE_SANDBOX_API_KEY }}
          OPENEVIDENCE_ORG_ID: ${{ secrets.OPENEVIDENCE_SANDBOX_ORG_ID }}
          OPENEVIDENCE_BASE_URL: https://api.sandbox.openevidence.com
        run: npm run test:integration
        timeout-minutes: 10

  clinical-validation:
    runs-on: ubuntu-latest
    needs: [unit-tests]
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run clinical validation tests
        env:
          OPENEVIDENCE_API_KEY: ${{ secrets.OPENEVIDENCE_SANDBOX_API_KEY }}
          OPENEVIDENCE_ORG_ID: ${{ secrets.OPENEVIDENCE_SANDBOX_ORG_ID }}
        run: npm run test:clinical-validation

      - name: Upload validation report
        uses: actions/upload-artifact@v4
        with:
          name: clinical-validation-report
          path: reports/clinical-validation.json

Step 2: Test Configuration

// vitest.config.ts
import { defineConfig } from 'vitest/config';

export default defineConfig({
  test: {
    globals: true,
    environment: 'node',
    include: ['tests/**/*.test.ts'],
    exclude: ['tests/integration/**', 'tests/clinical-validation/**'],
    coverage: {
      provider: 'v8',
      reporter: ['text', 'lcov'],
      exclude: ['tests/**', 'node_modules/**'],
    },
    testTimeout: 10000,
  },
});

// vitest.config.integration.ts
import { defineConfig, mergeConfig } from 'vitest/config';
import baseConfig from './vitest.config';

export default mergeConfig(baseConfig, defineConfig({
  test: {
    include: ['tests/integration/**/*.test.ts'],
    exclude: [],
    testTimeout: 60000, // 60s for API calls
    retry: 2, // Retry flaky integration tests
    maxConcurrency: 1, // Avoid rate limits
  },
}));

Step 3: Unit Tests with Mocks

// tests/unit/clinical-query.test.ts
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { queryClinicalEvidence } from '../../src/services/clinical-query';

// Mock the OpenEvidence SDK
vi.mock('@openevidence/sdk', () => ({
  OpenEvidenceClient: vi.fn().mockImplementation(() => ({
    query: vi.fn().mockResolvedValue({
      id: 'test-query-123',
      answer: 'Mock clinical answer for testing purposes.',
      citations: [
        { source: 'Test Journal', title: 'Test Article', year: 2025 }
      ],
      confidence: 0.95,
      lastUpdated: '2025-01-01',
    }),
  })),
}));

describe('Clinical Query Service', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  it('should return formatted clinical response', async () => {
    const result = await queryClinicalEvidence('Test question');

    expect(result.answer).toBeDefined();
    expect(result.citations).toHaveLength(1);
    expect(result.confidence).toBeGreaterThan(0);
  });

  it('should handle query errors gracefully', async () => {
    const { OpenEvidenceClient } = await import('@openevidence/sdk');
    vi.mocked(OpenEvidenceClient).mockImplementation(() => ({
      query: vi.fn().mockRejectedValue(new Error('API Error')),
    } as any));

    await expect(queryClinicalEvidence('Test question'))
      .rejects.toThrow('API Error');
  });
});

Step 4: Integration Tests

// tests/integration/openevidence-api.test.ts
import { describe, it, expect, beforeAll } from 'vitest';
import { OpenEvidenceClient } from '@openevidence/sdk';

describe('OpenEvidence API Integration', () => {
  let client: OpenEvidenceClient;

  beforeAll(() => {
    // Ensure we're using sandbox credentials
    const baseUrl = process.env.OPENEVIDENCE_BASE_URL;
    if (!baseUrl?.includes('sandbox')) {
      throw new Error('Integration tests must use sandbox environment');
    }

    client = new OpenEvidenceClient({
      apiKey: process.env.OPENEVIDENCE_API_KEY!,
      orgId: process.env.OPENEVIDENCE_ORG_ID!,
      baseUrl,
    });
  });

  it('should successfully query clinical evidence', async () => {
    const response = await client.query({
      question: 'What is the half-life of aspirin?',
      context: {
        specialty: 'pharmacology',
        urgency: 'routine',
      },
    });

    expect(response.id).toBeDefined();
    expect(response.answer).toBeDefined();
    expect(response.answer.length).toBeGreaterThan(50);
    expect(response.citations.length).toBeGreaterThan(0);
    expect(response.confidence).toBeGreaterThan(0.5);
  }, 30000);

  it('should handle invalid queries appropriately', async () => {
    await expect(client.query({
      question: '',
      context: { specialty: 'internal-medicine', urgency: 'routine' },
    })).rejects.toThrow();
  });

  it('should return citations with required fields', async () => {
    const response = await client.query({
      question: 'What are the first-line treatments for hypertension?',
      context: { specialty: 'cardiology', urgency: 'routine' },
    });

    response.citations.forEach(citation => {
      expect(citation.source).toBeDefined();
      expect(citation.title).toBeDefined();
      expect(citation.year).toBeGreaterThan(2000);
    });
  }, 30000);
});

Step 5: Clinical Validation Tests

// tests/clinical-validation/known-answers.test.ts
import { describe, it, expect } from 'vitest';
import { OpenEvidenceClient } from '@openevidence/sdk';

interface ClinicalValidationCase {
  question: string;
  specialty: string;
  expectedKeywords: string[];
  mustNotContain?: string[];
}

const VALIDATION_CASES: ClinicalValidationCase[] = [
  {
    question: 'What is the first-line treatment for type 2 diabetes in adults?',
    specialty: 'endocrinology',
    expectedKeywords: ['metformin', 'lifestyle', 'HbA1c'],
    mustNotContain: ['insulin-first'],
  },
  {
    question: 'What are the contraindications for aspirin?',
    specialty: 'internal-medicine',
    expectedKeywords: ['bleeding', 'allergy', 'ulcer'],
  },
  {
    question: 'What is the target LDL for secondary prevention of cardiovascular disease?',
    specialty: 'cardiology',
    expectedKeywords: ['LDL', 'mg/dL', 'statin'],
  },
];

describe('Clinical Validation - Known Answers', () => {
  const client = new OpenEvidenceClient({
    apiKey: process.env.OPENEVIDENCE_API_KEY!,
    orgId: process.env.OPENEVIDENCE_ORG_ID!,
  });

  const results: any[] = [];

  VALIDATION_CASES.forEach((testCase, index) => {
    it(`should answer correctly: ${testCase.question.substring(0, 50)}...`, async () => {
      const response = await client.query({
        question: testCase.question,
        context: {
          specialty: testCase.specialty,
          urgency: 'routine',
        },
      });

      const answerLower = response.answer.toLowerCase();

      // Check expected keywords
      const foundKeywords = testCase.expectedKeywords.filter(
        kw => answerLower.includes(kw.toLowerCase())
      );

      // Check must-not-contain
      const forbiddenFound = testCase.mustNotContain?.filter(
        kw => answerLower.includes(kw.toLowerCase())
      ) || [];

      // Record results for report
      results.push({
        question: testCase.question,
        expectedKeywords: testCase.expectedKeywords,
        foundKeywords,
        forbiddenFound,
        confidence: response.confidence,
        citationCount: response.citations.length,
        passed: foundKeywords.length >= testCase.expectedKeywords.length / 2 &&
                forbiddenFound.length === 0,
      });

      // Assert
      expect(foundKeywords.length).toBeGreaterThanOrEqual(
        Math.ceil(testCase.expectedKeywords.length / 2),
        `Missing keywords: ${testCase.expectedKeywords.filter(k => !foundKeywords.includes(k)).join(', ')}`
      );
      expect(forbiddenFound).toHaveLength(0);
      expect(response.confidence).toBeGreaterThan(0.7);
    }, 60000);
  });

  afterAll(async () => {
    // Write validation report
    const fs = await import('fs/promises');
    await fs.mkdir('reports', { recursive: true });
    await fs.writeFile(
      'reports/clinical-validation.json',
      JSON.stringify({
        timestamp: new Date().toISOString(),
        totalTests: results.length,
        passed: results.filter(r => r.passed).length,
        failed: results.filter(r => !r.passed).length,
        results,
      }, null, 2)
    );
  });
});

Step 6: Secret Management

# Store these as GitHub Secrets:
# - OPENEVIDENCE_SANDBOX_API_KEY
# - OPENEVIDENCE_SANDBOX_ORG_ID
# - OPENEVIDENCE_PROD_API_KEY (for deployment only)
# - OPENEVIDENCE_PROD_ORG_ID (for deployment only)

# For local development, use .env.test:
# OPENEVIDENCE_API_KEY=oe_sandbox_***
# OPENEVIDENCE_ORG_ID=org_sandbox_***
# OPENEVIDENCE_BASE_URL=https://api.sandbox.openevidence.com

Output

• CI workflow running on push and PR
• Unit tests with mocked SDK
• Integration tests against sandbox
• Clinical validation test suite
• Coverage reports

Package.json Scripts

{
  "scripts": {
    "test": "vitest",
    "test:unit": "vitest run -c vitest.config.ts",
    "test:integration": "vitest run -c vitest.config.integration.ts",
    "test:clinical-validation": "vitest run tests/clinical-validation",
    "test:ci": "vitest run --coverage",
    "typecheck": "tsc --noEmit",
    "lint": "eslint src tests"
  }
}

Error Handling

| CI Issue | Cause | Solution | |----------|-------|----------| | Integration test timeout | API slow or rate limited | Increase timeout, add retry | | Secret not found | Missing GitHub secret | Add secret in repo settings | | Flaky tests | Network variability | Add retries, improve assertions | | Coverage drop | New code untested | Add tests, adjust thresholds |

Resources

• GitHub Actions Documentation
• Vitest Documentation

Next Steps

For deployment integration, see openevidence-deploy-integration.