GitHub Actions + AI: Automate CI/CD with Copilot and LLMs

┌──────────────┐     ┌──────────────┐     ┌──────────────┐
│   Trigger    │────►│  Traditional │────►│   AI Layer   │
│  (push, PR,  │     │  CI Steps    │     │  (review,    │
│   schedule)  │     │  (lint,test) │     │   triage,    │
│              │     │              │     │   suggest)   │
└──────────────┘     └──────────────┘     └──────────────┘
                                                │
                                                ▼
                                          ┌──────────────┐
                                          │   Output     │
                                          │  (PR comment, │
                                          │   new PR,    │
                                          │   report)    │
                                          └──────────────┘

# .github/copilot-setup-steps.yml
name: "Copilot Setup"
on: workflow_dispatch

jobs:
  setup:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: "20"
      - run: npm ci
      - run: npm run build

## Code Style
- Use TypeScript strict mode
- Prefer functional components with hooks
- All new functions must have JSDoc comments

## Testing
- Write tests using Vitest
- Place test files next to source files with `.test.ts` suffix
- Minimum 80% branch coverage for new code

## PR Guidelines
- Keep PRs under 300 lines when possible
- Reference the issue number in commit messages

---
triggers:
  - schedule: "0 9 * * 1-5"  # Weekdays at 9 AM
permissions:
  issues: read
  pull-requests: read
  contents: read
outputs:
  - type: issue_comment
agent: copilot
---

# Daily Repository Health Check

Review the repository and create a status summary.

## Tasks
- Count open issues by label (bug, feature, docs)
- List PRs that have been open for more than 7 days
- Identify any CI workflows that failed in the last 24 hours
- Summarize in a concise report with actionable recommendations

## Format
Use a Markdown table for issue counts. Keep the report under 500 words.
Link to specific issues and PRs.

name: AI Code Review
on:
  pull_request:
    types: [opened, synchronize]

jobs:
  review:
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
      contents: read
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Get PR diff
        id: diff
        run: |
          DIFF=$(git diff origin/${{ github.base_ref }}...HEAD)
          echo "diff<<EODIFF" >> $GITHUB_OUTPUT
          echo "$DIFF" >> $GITHUB_OUTPUT
          echo "EODIFF" >> $GITHUB_OUTPUT

      - name: AI Review
        id: review
        uses: appleboy/LLM-action@v1
        with:
          api_key: ${{ secrets.OPENAI_API_KEY }}
          model: "gpt-4o"
          system_prompt: |
            You are a senior code reviewer. Review the following diff.
            Focus on: bugs, security issues, performance problems, and
            readability. Be specific. Reference line numbers.
            If the code looks good, say so briefly.
          input_prompt: |
            Review this pull request diff:

            ${{ steps.diff.outputs.diff }}

      - name: Post review comment
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: `## 🤖 AI Code Review\n\n${{ steps.review.outputs.response }}`
            })

- name: AI Review (Self-Hosted)
  uses: appleboy/LLM-action@v1
  with:
    api_url: "https://your-ollama-server.internal:11434/v1"
    api_key: ${{ secrets.INTERNAL_API_KEY }}
    model: "llama3.3:70b"
    input_prompt: "Review this diff for security issues: ${{ steps.diff.outputs.diff }}"

- name: Analyze commit
  id: analysis
  uses: appleboy/LLM-action@v1
  with:
    api_key: ${{ secrets.OPENAI_API_KEY }}
    model: "gpt-4o"
    input_prompt: "Analyze this commit message and diff: ${{ github.event.head_commit.message }}"
    tool_schema: |
      {
        "name": "commit_analysis",
        "description": "Structured commit analysis",
        "parameters": {
          "type": "object",
          "properties": {
            "risk_level": {
              "type": "string",
              "enum": ["low", "medium", "high"]
            },
            "category": {
              "type": "string",
              "enum": ["feature", "bugfix", "refactor", "docs", "chore"]
            },
            "needs_review": {
              "type": "boolean"
            },
            "summary": {
              "type": "string"
            }
          },
          "required": ["risk_level", "category", "needs_review", "summary"]
        }
      }

name: Prompt Evaluation
on:
  pull_request:
    paths:
      - "prompts/**"
      - "promptfooconfig.yaml"

jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/cache@v4
        with:
          path: |
            ~/.promptfoo/cache
            .promptfoo-cache
          key: promptfoo-${{ hashFiles('promptfooconfig.yaml') }}

      - uses: promptfoo/promptfoo-action@v1
        with:
          openai-api-key: ${{ secrets.OPENAI_API_KEY }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
          prompts: "prompts/"
          config: "promptfooconfig.yaml"

# promptfooconfig.yaml
prompts:
  - file://prompts/customer-support.txt
  - file://prompts/customer-support-v2.txt

providers:
  - openai:gpt-4o
  - openai:gpt-4o-mini

tests:
  - vars:
      customer_name: "Alex"
      question: "How do I reset my password?"
    assert:
      - type: llm-rubric
        value: "Response should include step-by-step instructions"
      - type: contains
        value: "password"
      - type: javascript
        value: "output.length < 500"

  - vars:
      customer_name: "Jordan"
      question: "I want to cancel my subscription"
    assert:
      - type: llm-rubric
        value: "Response should be empathetic and offer retention options"
      - type: not-contains
        value: "sorry to see you go"

name: LLM Quality Check
on:
  push:
    branches: [main]

jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: pip install evidently openai

      - name: Run LLM evaluation
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: python scripts/evaluate_llm.py

      - name: Check quality gate
        run: |
          SCORE=$(cat results/quality_score.json | jq '.overall_score')
          if (( $(echo "$SCORE < 0.85" | bc -l) )); then
            echo "Quality score $SCORE is below threshold 0.85"
            exit 1
          fi

name: Smart Deployment
on:
  push:
    tags: ["v*"]

jobs:
  analyze:
    runs-on: ubuntu-latest
    outputs:
      strategy: ${{ steps.decide.outputs.strategy }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 50

      - name: Generate changelog
        id: changelog
        run: |
          LOG=$(git log --oneline $(git describe --tags --abbrev=0 HEAD~1)..HEAD)
          echo "log<<EOF" >> $GITHUB_OUTPUT
          echo "$LOG" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: AI deployment decision
        id: decide
        uses: appleboy/LLM-action@v1
        with:
          api_key: ${{ secrets.OPENAI_API_KEY }}
          model: "gpt-4o"
          input_prompt: |
            Analyze these commits and decide on a deployment strategy.
            Commits: ${{ steps.changelog.outputs.log }}
          tool_schema: |
            {
              "name": "deployment_decision",
              "parameters": {
                "type": "object",
                "properties": {
                  "strategy": {
                    "type": "string",
                    "enum": ["immediate", "canary", "staged"]
                  },
                  "reason": { "type": "string" }
                },
                "required": ["strategy", "reason"]
              }
            }

  deploy:
    needs: analyze
    runs-on: ubuntu-latest
    steps:
      - name: Deploy with strategy
        run: |
          echo "Deploying with strategy: ${{ needs.analyze.outputs.strategy }}"
          # Your actual deployment logic here

name: Auto-Fix Lint
on:
  pull_request:
    types: [opened, synchronize]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          token: ${{ secrets.PAT_TOKEN }}

      - uses: actions/setup-node@v4
        with:
          node-version: "20"

      - run: npm ci

      - name: Run linter
        id: lint
        run: |
          npx eslint . --format json --output-file lint-results.json || true
          ERRORS=$(cat lint-results.json | jq '[.[].errorCount] | add')
          echo "errors=$ERRORS" >> $GITHUB_OUTPUT

      - name: AI fix
        if: steps.lint.outputs.errors > 0
        uses: appleboy/LLM-action@v1
        with:
          api_key: ${{ secrets.OPENAI_API_KEY }}
          model: "gpt-4o"
          system_prompt: "You are a code fixer. Output ONLY the corrected code, no explanations."
          input_prompt: |
            Fix the linting errors in these files:
            $(cat lint-results.json | jq -r '.[] | select(.errorCount > 0) | .filePath')

      # In practice, you'd parse the AI output and apply fixes
      # This is a simplified example — see the troubleshooting section

name: Docs Freshness Check
on:
  pull_request:
    paths:
      - "src/**"

jobs:
  check-docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Get changed files
        id: changes
        run: |
          FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD | grep '^src/')
          echo "files=$FILES" >> $GITHUB_OUTPUT

      - name: Check if docs need updating
        uses: appleboy/LLM-action@v1
        with:
          api_key: ${{ secrets.OPENAI_API_KEY }}
          model: "gpt-4o"
          input_prompt: |
            These source files changed in a PR:
            ${{ steps.changes.outputs.files }}

            Does the documentation in docs/ need updating?
            Consider: API changes, new features, removed features,
            configuration changes.
          tool_schema: |
            {
              "name": "docs_check",
              "parameters": {
                "type": "object",
                "properties": {
                  "needs_update": { "type": "boolean" },
                  "affected_docs": {
                    "type": "array",
                    "items": { "type": "string" }
                  },
                  "summary": { "type": "string" }
                },
                "required": ["needs_update", "summary"]
              }
            }

permissions:
  pull-requests: write
  contents: read
  issues: write

# Instead of sending the full diff, filter to changed files only
git diff --stat origin/main...HEAD
git diff origin/main...HEAD -- "*.py" "*.ts"