"""Tests for the ingestion API endpoints."""
from __future__ import annotations
import io
import json
import sys
import time
import zipfile
from pathlib import Path
from unittest.mock import patch
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from fastapi import FastAPI
from fastapi.testclient import TestClient
from ingestion.api import router, _jobs
@pytest.fixture(autouse=True)
def clear_jobs():
"""Clear the job store between tests."""
_jobs.clear()
yield
_jobs.clear()
@pytest.fixture
def app():
app = FastAPI()
app.include_router(router)
return app
@pytest.fixture
def client(app):
return TestClient(app)
@pytest.fixture
def input_dir_with_files(tmp_path: Path) -> Path:
d = tmp_path / "test_input"
d.mkdir()
(d / "hello.txt").write_text("Hello world")
(d / "data.csv").write_text("a,b\n1,2\n")
return d
# ---------------------------------------------------------------------------
# POST /ingestion/start
# ---------------------------------------------------------------------------
class TestStartIngestion:
def test_start_valid_path(self, client, input_dir_with_files):
resp = client.post("/ingestion/start", json={
"input_dir": str(input_dir_with_files),
"output_dir": str(input_dir_with_files.parent / "output"),
"workers": 2,
})
assert resp.status_code == 200
data = resp.json()
assert "job_id" in data
assert data["status"] in ("pending", "running")
def test_start_invalid_path(self, client):
resp = client.post("/ingestion/start", json={
"input_dir": "/nonexistent/path/that/does/not/exist",
})
assert resp.status_code == 400
assert "not found" in resp.json()["detail"].lower()
def test_start_returns_unique_job_ids(self, client, input_dir_with_files):
r1 = client.post("/ingestion/start", json={"input_dir": str(input_dir_with_files)})
r2 = client.post("/ingestion/start", json={"input_dir": str(input_dir_with_files)})
assert r1.json()["job_id"] != r2.json()["job_id"]
# ---------------------------------------------------------------------------
# POST /ingestion/upload
# ---------------------------------------------------------------------------
class TestUploadIngestion:
def test_upload_single_file(self, client, tmp_path: Path):
output_dir = tmp_path / "upload_output"
output_dir.mkdir()
resp = client.post(
"/ingestion/upload",
files=[("files", ("test.txt", b"Hello from upload", "text/plain"))],
data={"output_dir": str(output_dir), "workers": "1"},
)
assert resp.status_code == 200
data = resp.json()
assert data["uploaded_files"] >= 1
assert "job_id" in data
def test_upload_zip_file(self, client, tmp_path: Path):
output_dir = tmp_path / "zip_output"
output_dir.mkdir()
# Create a zip in memory
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w") as zf:
zf.writestr("doc1.txt", "First document")
zf.writestr("subdir/doc2.txt", "Second document")
buf.seek(0)
resp = client.post(
"/ingestion/upload",
files=[("files", ("archive.zip", buf.getvalue(), "application/zip"))],
data={"output_dir": str(output_dir), "workers": "1"},
)
assert resp.status_code == 200
data = resp.json()
assert data["uploaded_files"] >= 2
def test_upload_multiple_files(self, client, tmp_path: Path):
output_dir = tmp_path / "multi_output"
output_dir.mkdir()
resp = client.post(
"/ingestion/upload",
files=[
("files", ("a.txt", b"File A content", "text/plain")),
("files", ("b.csv", b"x,y\n1,2\n", "text/csv")),
],
data={"output_dir": str(output_dir), "workers": "1"},
)
assert resp.status_code == 200
assert resp.json()["uploaded_files"] == 2
# ---------------------------------------------------------------------------
# GET /ingestion/{job_id}/status
# ---------------------------------------------------------------------------
class TestJobStatus:
def test_status_of_running_job(self, client, input_dir_with_files):
start_resp = client.post("/ingestion/start", json={
"input_dir": str(input_dir_with_files),
"output_dir": str(input_dir_with_files.parent / "output"),
"workers": 1,
})
job_id = start_resp.json()["job_id"]
# Give it a moment to start
time.sleep(0.2)
status_resp = client.get(f"/ingestion/{job_id}/status")
assert status_resp.status_code == 200
data = status_resp.json()
assert data["job_id"] == job_id
assert data["status"] in ("pending", "running", "completed")
def test_status_unknown_job(self, client):
resp = client.get("/ingestion/nonexistent123/status")
assert resp.status_code == 404
# ---------------------------------------------------------------------------
# POST /ingestion/{job_id}/cancel
# ---------------------------------------------------------------------------
class TestCancelJob:
def test_cancel_running_job(self, client, input_dir_with_files):
# Create many files so the job takes a moment
for i in range(50):
(input_dir_with_files / f"file_{i:03d}.txt").write_text(f"Content {i}")
start_resp = client.post("/ingestion/start", json={
"input_dir": str(input_dir_with_files),
"output_dir": str(input_dir_with_files.parent / "output"),
"workers": 1,
})
job_id = start_resp.json()["job_id"]
cancel_resp = client.post(f"/ingestion/{job_id}/cancel")
assert cancel_resp.status_code == 200
assert cancel_resp.json()["status"] == "cancelling"
def test_cancel_unknown_job(self, client):
resp = client.post("/ingestion/nonexistent123/cancel")
assert resp.status_code == 404
# ---------------------------------------------------------------------------
# GET /ingestion/{job_id}/stream (SSE)
# ---------------------------------------------------------------------------
class TestSSEStream:
def test_stream_receives_events(self, client, input_dir_with_files):
start_resp = client.post("/ingestion/start", json={
"input_dir": str(input_dir_with_files),
"output_dir": str(input_dir_with_files.parent / "output"),
"workers": 1,
})
job_id = start_resp.json()["job_id"]
# Read the SSE stream
with client.stream("GET", f"/ingestion/{job_id}/stream") as resp:
assert resp.status_code == 200
assert "text/event-stream" in resp.headers.get("content-type", "")
events = []
for line in resp.iter_lines():
if line.startswith("event:"):
events.append(line)
if "ingestion_completed" in line:
break
assert len(events) >= 1
def test_stream_unknown_job(self, client):
resp = client.get("/ingestion/nonexistent123/stream")
assert resp.status_code == 404